1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2025, Christoph Hellwig. 4 * Copyright (c) 2025, Western Digital Corporation or its affiliates. 5 * 6 * Zoned Loop Device driver - exports a zoned block device using one file per 7 * zone as backing storage. 8 */ 9 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 10 11 #include <linux/module.h> 12 #include <linux/blk-mq.h> 13 #include <linux/blkzoned.h> 14 #include <linux/pagemap.h> 15 #include <linux/miscdevice.h> 16 #include <linux/falloc.h> 17 #include <linux/mutex.h> 18 #include <linux/parser.h> 19 #include <linux/seq_file.h> 20 #include <linux/xattr.h> 21 22 /* 23 * Options for adding (and removing) a device. 24 */ 25 enum { 26 ZLOOP_OPT_ERR = 0, 27 ZLOOP_OPT_ID = (1 << 0), 28 ZLOOP_OPT_CAPACITY = (1 << 1), 29 ZLOOP_OPT_ZONE_SIZE = (1 << 2), 30 ZLOOP_OPT_ZONE_CAPACITY = (1 << 3), 31 ZLOOP_OPT_NR_CONV_ZONES = (1 << 4), 32 ZLOOP_OPT_BASE_DIR = (1 << 5), 33 ZLOOP_OPT_NR_QUEUES = (1 << 6), 34 ZLOOP_OPT_QUEUE_DEPTH = (1 << 7), 35 ZLOOP_OPT_BUFFERED_IO = (1 << 8), 36 ZLOOP_OPT_ZONE_APPEND = (1 << 9), 37 ZLOOP_OPT_ORDERED_ZONE_APPEND = (1 << 10), 38 ZLOOP_OPT_DISCARD_WRITE_CACHE = (1 << 11), 39 ZLOOP_OPT_MAX_OPEN_ZONES = (1 << 12), 40 }; 41 42 static const match_table_t zloop_opt_tokens = { 43 { ZLOOP_OPT_ID, "id=%d" }, 44 { ZLOOP_OPT_CAPACITY, "capacity_mb=%u" }, 45 { ZLOOP_OPT_ZONE_SIZE, "zone_size_mb=%u" }, 46 { ZLOOP_OPT_ZONE_CAPACITY, "zone_capacity_mb=%u" }, 47 { ZLOOP_OPT_NR_CONV_ZONES, "conv_zones=%u" }, 48 { ZLOOP_OPT_BASE_DIR, "base_dir=%s" }, 49 { ZLOOP_OPT_NR_QUEUES, "nr_queues=%u" }, 50 { ZLOOP_OPT_QUEUE_DEPTH, "queue_depth=%u" }, 51 { ZLOOP_OPT_BUFFERED_IO, "buffered_io" }, 52 { ZLOOP_OPT_ZONE_APPEND, "zone_append=%u" }, 53 { ZLOOP_OPT_ORDERED_ZONE_APPEND, "ordered_zone_append" }, 54 { ZLOOP_OPT_DISCARD_WRITE_CACHE, "discard_write_cache" }, 55 { ZLOOP_OPT_MAX_OPEN_ZONES, "max_open_zones=%u" }, 56 { ZLOOP_OPT_ERR, NULL } 57 }; 58 59 /* Default values for the "add" operation. */ 60 #define ZLOOP_DEF_ID -1 61 #define ZLOOP_DEF_ZONE_SIZE ((256ULL * SZ_1M) >> SECTOR_SHIFT) 62 #define ZLOOP_DEF_NR_ZONES 64 63 #define ZLOOP_DEF_NR_CONV_ZONES 8 64 #define ZLOOP_DEF_MAX_OPEN_ZONES 0 65 #define ZLOOP_DEF_BASE_DIR "/var/local/zloop" 66 #define ZLOOP_DEF_NR_QUEUES 1 67 #define ZLOOP_DEF_QUEUE_DEPTH 128 68 #define ZLOOP_DEF_BUFFERED_IO false 69 #define ZLOOP_DEF_ZONE_APPEND true 70 #define ZLOOP_DEF_ORDERED_ZONE_APPEND false 71 72 /* Arbitrary limit on the zone size (16GB). */ 73 #define ZLOOP_MAX_ZONE_SIZE_MB 16384 74 75 struct zloop_options { 76 unsigned int mask; 77 int id; 78 sector_t capacity; 79 sector_t zone_size; 80 sector_t zone_capacity; 81 unsigned int nr_conv_zones; 82 unsigned int max_open_zones; 83 char *base_dir; 84 unsigned int nr_queues; 85 unsigned int queue_depth; 86 bool buffered_io; 87 bool zone_append; 88 bool ordered_zone_append; 89 bool discard_write_cache; 90 }; 91 92 /* 93 * Device states. 94 */ 95 enum { 96 Zlo_creating = 0, 97 Zlo_live, 98 Zlo_deleting, 99 }; 100 101 enum zloop_zone_flags { 102 ZLOOP_ZONE_CONV = 0, 103 ZLOOP_ZONE_SEQ_ERROR, 104 }; 105 106 /* 107 * Zone descriptor. 108 * Locking order: z.lock -> z.wp_lock -> zlo.open_zones_lock 109 */ 110 struct zloop_zone { 111 struct list_head open_zone_entry; 112 struct file *file; 113 114 unsigned long flags; 115 struct mutex lock; 116 spinlock_t wp_lock; 117 enum blk_zone_cond cond; 118 sector_t start; 119 sector_t wp; 120 121 gfp_t old_gfp_mask; 122 }; 123 124 struct zloop_device { 125 unsigned int id; 126 unsigned int state; 127 128 struct blk_mq_tag_set tag_set; 129 struct gendisk *disk; 130 131 struct workqueue_struct *workqueue; 132 bool buffered_io; 133 bool zone_append; 134 bool ordered_zone_append; 135 bool discard_write_cache; 136 137 const char *base_dir; 138 struct file *data_dir; 139 140 unsigned int zone_shift; 141 sector_t zone_size; 142 sector_t zone_capacity; 143 unsigned int nr_zones; 144 unsigned int nr_conv_zones; 145 unsigned int max_open_zones; 146 unsigned int block_size; 147 unsigned int dio_mem_align; 148 149 spinlock_t open_zones_lock; 150 struct list_head open_zones_lru_list; 151 unsigned int nr_open_zones; 152 153 struct zloop_zone zones[] __counted_by(nr_zones); 154 }; 155 156 struct zloop_cmd { 157 struct work_struct work; 158 atomic_t ref; 159 sector_t sector; 160 sector_t nr_sectors; 161 long ret; 162 struct kiocb iocb; 163 struct bio_vec *bvec; 164 }; 165 166 static DEFINE_IDR(zloop_index_idr); 167 static DEFINE_MUTEX(zloop_ctl_mutex); 168 169 static unsigned int rq_zone_no(struct request *rq) 170 { 171 struct zloop_device *zlo = rq->q->queuedata; 172 173 return blk_rq_pos(rq) >> zlo->zone_shift; 174 } 175 176 /* 177 * Open an already open zone. This is mostly a no-op, except for the imp open -> 178 * exp open condition change that may happen. We also move a zone at the tail of 179 * the list of open zones so that if we need to 180 * implicitly close one open zone, we can do so in LRU order. 181 */ 182 static inline void zloop_lru_rotate_open_zone(struct zloop_device *zlo, 183 struct zloop_zone *zone) 184 { 185 if (zlo->max_open_zones) { 186 spin_lock(&zlo->open_zones_lock); 187 list_move_tail(&zone->open_zone_entry, 188 &zlo->open_zones_lru_list); 189 spin_unlock(&zlo->open_zones_lock); 190 } 191 } 192 193 static inline void zloop_lru_remove_open_zone(struct zloop_device *zlo, 194 struct zloop_zone *zone) 195 { 196 if (zone->cond == BLK_ZONE_COND_IMP_OPEN || 197 zone->cond == BLK_ZONE_COND_EXP_OPEN) { 198 spin_lock(&zlo->open_zones_lock); 199 list_del_init(&zone->open_zone_entry); 200 zlo->nr_open_zones--; 201 spin_unlock(&zlo->open_zones_lock); 202 } 203 } 204 205 static inline bool zloop_can_open_zone(struct zloop_device *zlo) 206 { 207 return !zlo->max_open_zones || zlo->nr_open_zones < zlo->max_open_zones; 208 } 209 210 /* 211 * If we have reached the maximum open zones limit, attempt to close an 212 * implicitly open zone (if we have any) so that we can implicitly open another 213 * zone without exceeding the maximum number of open zones. 214 */ 215 static bool zloop_close_imp_open_zone(struct zloop_device *zlo) 216 { 217 struct zloop_zone *zone; 218 219 lockdep_assert_held(&zlo->open_zones_lock); 220 221 if (zloop_can_open_zone(zlo)) 222 return true; 223 224 list_for_each_entry(zone, &zlo->open_zones_lru_list, open_zone_entry) { 225 if (zone->cond == BLK_ZONE_COND_IMP_OPEN) { 226 zone->cond = BLK_ZONE_COND_CLOSED; 227 list_del_init(&zone->open_zone_entry); 228 zlo->nr_open_zones--; 229 return true; 230 } 231 } 232 233 return false; 234 } 235 236 static bool zloop_open_closed_or_empty_zone(struct zloop_device *zlo, 237 struct zloop_zone *zone, 238 bool explicit) 239 { 240 spin_lock(&zlo->open_zones_lock); 241 242 if (explicit) { 243 /* 244 * Explicit open: we cannot allow this if we have reached the 245 * maximum open zones limit. 246 */ 247 if (!zloop_can_open_zone(zlo)) 248 goto fail; 249 zone->cond = BLK_ZONE_COND_EXP_OPEN; 250 } else { 251 /* 252 * Implicit open case: if we have reached the maximum open zones 253 * limit, try to close an implicitly open zone first. 254 */ 255 if (!zloop_close_imp_open_zone(zlo)) 256 goto fail; 257 zone->cond = BLK_ZONE_COND_IMP_OPEN; 258 } 259 260 zlo->nr_open_zones++; 261 list_add_tail(&zone->open_zone_entry, 262 &zlo->open_zones_lru_list); 263 264 spin_unlock(&zlo->open_zones_lock); 265 266 return true; 267 268 fail: 269 spin_unlock(&zlo->open_zones_lock); 270 271 return false; 272 } 273 274 static bool zloop_do_open_zone(struct zloop_device *zlo, 275 struct zloop_zone *zone, bool explicit) 276 { 277 switch (zone->cond) { 278 case BLK_ZONE_COND_IMP_OPEN: 279 case BLK_ZONE_COND_EXP_OPEN: 280 if (explicit) 281 zone->cond = BLK_ZONE_COND_EXP_OPEN; 282 zloop_lru_rotate_open_zone(zlo, zone); 283 return true; 284 case BLK_ZONE_COND_EMPTY: 285 case BLK_ZONE_COND_CLOSED: 286 return zloop_open_closed_or_empty_zone(zlo, zone, explicit); 287 default: 288 return false; 289 } 290 } 291 292 static void zloop_mark_full(struct zloop_device *zlo, struct zloop_zone *zone) 293 { 294 lockdep_assert_held(&zone->wp_lock); 295 296 zloop_lru_remove_open_zone(zlo, zone); 297 zone->cond = BLK_ZONE_COND_FULL; 298 zone->wp = ULLONG_MAX; 299 } 300 301 static void zloop_mark_empty(struct zloop_device *zlo, struct zloop_zone *zone) 302 { 303 lockdep_assert_held(&zone->wp_lock); 304 305 zloop_lru_remove_open_zone(zlo, zone); 306 zone->cond = BLK_ZONE_COND_EMPTY; 307 zone->wp = zone->start; 308 } 309 310 static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no) 311 { 312 struct zloop_zone *zone = &zlo->zones[zone_no]; 313 struct kstat stat; 314 sector_t file_sectors; 315 int ret; 316 317 lockdep_assert_held(&zone->lock); 318 319 ret = vfs_getattr(&zone->file->f_path, &stat, STATX_SIZE, 0); 320 if (ret < 0) { 321 pr_err("Failed to get zone %u file stat (err=%d)\n", 322 zone_no, ret); 323 set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); 324 return ret; 325 } 326 327 file_sectors = stat.size >> SECTOR_SHIFT; 328 if (file_sectors > zlo->zone_capacity) { 329 pr_err("Zone %u file too large (%llu sectors > %llu)\n", 330 zone_no, file_sectors, zlo->zone_capacity); 331 return -EINVAL; 332 } 333 334 if (!IS_ALIGNED(stat.size, zlo->block_size)) { 335 pr_err("Zone %u file size (%llu) not aligned to block size %u\n", 336 zone_no, stat.size, zlo->block_size); 337 return -EINVAL; 338 } 339 340 spin_lock(&zone->wp_lock); 341 if (!file_sectors) { 342 zloop_mark_empty(zlo, zone); 343 } else if (file_sectors == zlo->zone_capacity) { 344 zloop_mark_full(zlo, zone); 345 } else { 346 if (zone->cond != BLK_ZONE_COND_IMP_OPEN && 347 zone->cond != BLK_ZONE_COND_EXP_OPEN) 348 zone->cond = BLK_ZONE_COND_CLOSED; 349 zone->wp = zone->start + file_sectors; 350 } 351 spin_unlock(&zone->wp_lock); 352 353 return 0; 354 } 355 356 static int zloop_open_zone(struct zloop_device *zlo, unsigned int zone_no) 357 { 358 struct zloop_zone *zone = &zlo->zones[zone_no]; 359 int ret = 0; 360 361 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) 362 return -EIO; 363 364 mutex_lock(&zone->lock); 365 366 if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { 367 ret = zloop_update_seq_zone(zlo, zone_no); 368 if (ret) 369 goto unlock; 370 } 371 372 if (!zloop_do_open_zone(zlo, zone, true)) 373 ret = -EIO; 374 375 unlock: 376 mutex_unlock(&zone->lock); 377 378 return ret; 379 } 380 381 static int zloop_close_zone(struct zloop_device *zlo, unsigned int zone_no) 382 { 383 struct zloop_zone *zone = &zlo->zones[zone_no]; 384 int ret = 0; 385 386 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) 387 return -EIO; 388 389 mutex_lock(&zone->lock); 390 391 if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { 392 ret = zloop_update_seq_zone(zlo, zone_no); 393 if (ret) 394 goto unlock; 395 } 396 397 switch (zone->cond) { 398 case BLK_ZONE_COND_CLOSED: 399 break; 400 case BLK_ZONE_COND_IMP_OPEN: 401 case BLK_ZONE_COND_EXP_OPEN: 402 spin_lock(&zone->wp_lock); 403 zloop_lru_remove_open_zone(zlo, zone); 404 if (zone->wp == zone->start) 405 zone->cond = BLK_ZONE_COND_EMPTY; 406 else 407 zone->cond = BLK_ZONE_COND_CLOSED; 408 spin_unlock(&zone->wp_lock); 409 break; 410 case BLK_ZONE_COND_EMPTY: 411 case BLK_ZONE_COND_FULL: 412 default: 413 ret = -EIO; 414 break; 415 } 416 417 unlock: 418 mutex_unlock(&zone->lock); 419 420 return ret; 421 } 422 423 static int zloop_reset_zone(struct zloop_device *zlo, unsigned int zone_no) 424 { 425 struct zloop_zone *zone = &zlo->zones[zone_no]; 426 int ret = 0; 427 428 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) 429 return -EIO; 430 431 mutex_lock(&zone->lock); 432 433 if (!test_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags) && 434 zone->cond == BLK_ZONE_COND_EMPTY) 435 goto unlock; 436 437 if (vfs_truncate(&zone->file->f_path, 0)) { 438 set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); 439 ret = -EIO; 440 goto unlock; 441 } 442 443 spin_lock(&zone->wp_lock); 444 zloop_mark_empty(zlo, zone); 445 clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); 446 spin_unlock(&zone->wp_lock); 447 448 unlock: 449 mutex_unlock(&zone->lock); 450 451 return ret; 452 } 453 454 static int zloop_reset_all_zones(struct zloop_device *zlo) 455 { 456 unsigned int i; 457 int ret; 458 459 for (i = zlo->nr_conv_zones; i < zlo->nr_zones; i++) { 460 ret = zloop_reset_zone(zlo, i); 461 if (ret) 462 return ret; 463 } 464 465 return 0; 466 } 467 468 static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no) 469 { 470 struct zloop_zone *zone = &zlo->zones[zone_no]; 471 int ret = 0; 472 473 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) 474 return -EIO; 475 476 mutex_lock(&zone->lock); 477 478 if (!test_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags) && 479 zone->cond == BLK_ZONE_COND_FULL) 480 goto unlock; 481 482 if (vfs_truncate(&zone->file->f_path, 483 zlo->zone_capacity << SECTOR_SHIFT)) { 484 set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); 485 ret = -EIO; 486 goto unlock; 487 } 488 489 spin_lock(&zone->wp_lock); 490 zloop_mark_full(zlo, zone); 491 clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); 492 spin_unlock(&zone->wp_lock); 493 494 unlock: 495 mutex_unlock(&zone->lock); 496 497 return ret; 498 } 499 500 static void zloop_put_cmd(struct zloop_cmd *cmd) 501 { 502 struct request *rq = blk_mq_rq_from_pdu(cmd); 503 504 if (!atomic_dec_and_test(&cmd->ref)) 505 return; 506 kfree(cmd->bvec); 507 cmd->bvec = NULL; 508 if (likely(!blk_should_fake_timeout(rq->q))) 509 blk_mq_complete_request(rq); 510 } 511 512 static void zloop_rw_complete(struct kiocb *iocb, long ret) 513 { 514 struct zloop_cmd *cmd = container_of(iocb, struct zloop_cmd, iocb); 515 516 cmd->ret = ret; 517 zloop_put_cmd(cmd); 518 } 519 520 static int zloop_do_rw(struct zloop_cmd *cmd) 521 { 522 struct request *rq = blk_mq_rq_from_pdu(cmd); 523 int rw = req_op(rq) == REQ_OP_READ ? ITER_DEST : ITER_SOURCE; 524 unsigned int nr_bvec = blk_rq_nr_bvec(rq); 525 struct zloop_device *zlo = rq->q->queuedata; 526 struct zloop_zone *zone = &zlo->zones[rq_zone_no(rq)]; 527 struct req_iterator rq_iter; 528 struct iov_iter iter; 529 530 if (rq->bio != rq->biotail) { 531 struct bio_vec tmp, *bvec; 532 533 cmd->bvec = kmalloc_objs(*cmd->bvec, nr_bvec, GFP_NOIO); 534 if (!cmd->bvec) 535 return -EIO; 536 537 /* 538 * The bios of the request may be started from the middle of 539 * the 'bvec' because of bio splitting, so we can't directly 540 * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec 541 * API will take care of all details for us. 542 */ 543 bvec = cmd->bvec; 544 rq_for_each_bvec(tmp, rq, rq_iter) { 545 *bvec = tmp; 546 bvec++; 547 } 548 iov_iter_bvec(&iter, rw, cmd->bvec, nr_bvec, blk_rq_bytes(rq)); 549 } else { 550 /* 551 * Same here, this bio may be started from the middle of the 552 * 'bvec' because of bio splitting, so offset from the bvec 553 * must be passed to iov iterator 554 */ 555 iov_iter_bvec(&iter, rw, 556 __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter), 557 nr_bvec, blk_rq_bytes(rq)); 558 iter.iov_offset = rq->bio->bi_iter.bi_offset; 559 } 560 561 cmd->iocb.ki_pos = (cmd->sector - zone->start) << SECTOR_SHIFT; 562 cmd->iocb.ki_filp = zone->file; 563 cmd->iocb.ki_complete = zloop_rw_complete; 564 if (!zlo->buffered_io) 565 cmd->iocb.ki_flags = IOCB_DIRECT; 566 cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0); 567 568 if (rw == ITER_SOURCE) 569 return zone->file->f_op->write_iter(&cmd->iocb, &iter); 570 return zone->file->f_op->read_iter(&cmd->iocb, &iter); 571 } 572 573 static int zloop_seq_write_prep(struct zloop_cmd *cmd) 574 { 575 struct request *rq = blk_mq_rq_from_pdu(cmd); 576 struct zloop_device *zlo = rq->q->queuedata; 577 unsigned int zone_no = rq_zone_no(rq); 578 sector_t nr_sectors = blk_rq_sectors(rq); 579 bool is_append = req_op(rq) == REQ_OP_ZONE_APPEND; 580 struct zloop_zone *zone = &zlo->zones[zone_no]; 581 sector_t zone_end = zone->start + zlo->zone_capacity; 582 int ret = 0; 583 584 spin_lock(&zone->wp_lock); 585 586 /* 587 * Zone append operations always go at the current write pointer, but 588 * regular write operations must already be aligned to the write pointer 589 * when submitted. 590 */ 591 if (is_append) { 592 /* 593 * If ordered zone append is in use, we already checked and set 594 * the target sector in zloop_queue_rq(). 595 */ 596 if (!zlo->ordered_zone_append) { 597 if (zone->cond == BLK_ZONE_COND_FULL || 598 zone->wp + nr_sectors > zone_end) { 599 ret = -EIO; 600 goto out_unlock; 601 } 602 cmd->sector = zone->wp; 603 } 604 } else { 605 if (cmd->sector != zone->wp) { 606 pr_err("Zone %u: unaligned write: sect %llu, wp %llu\n", 607 zone_no, cmd->sector, zone->wp); 608 ret = -EIO; 609 goto out_unlock; 610 } 611 } 612 613 /* Implicitly open the target zone. */ 614 if (!zloop_do_open_zone(zlo, zone, false)) { 615 ret = -EIO; 616 goto out_unlock; 617 } 618 619 /* 620 * Advance the write pointer, unless ordered zone append is in use. If 621 * the write fails, the write pointer position will be corrected when 622 * the next I/O starts execution. 623 */ 624 if (!is_append || !zlo->ordered_zone_append) { 625 zone->wp += nr_sectors; 626 if (zone->wp == zone_end) 627 zloop_mark_full(zlo, zone); 628 } 629 out_unlock: 630 spin_unlock(&zone->wp_lock); 631 return ret; 632 } 633 634 static void zloop_rw(struct zloop_cmd *cmd) 635 { 636 struct request *rq = blk_mq_rq_from_pdu(cmd); 637 struct zloop_device *zlo = rq->q->queuedata; 638 unsigned int zone_no = rq_zone_no(rq); 639 sector_t nr_sectors = blk_rq_sectors(rq); 640 bool is_append = req_op(rq) == REQ_OP_ZONE_APPEND; 641 bool is_write = req_op(rq) == REQ_OP_WRITE || is_append; 642 struct zloop_zone *zone; 643 int ret = -EIO; 644 645 atomic_set(&cmd->ref, 2); 646 cmd->sector = blk_rq_pos(rq); 647 cmd->nr_sectors = nr_sectors; 648 cmd->ret = 0; 649 650 if (WARN_ON_ONCE(is_append && !zlo->zone_append)) 651 goto out; 652 653 /* We should never get an I/O beyond the device capacity. */ 654 if (WARN_ON_ONCE(zone_no >= zlo->nr_zones)) 655 goto out; 656 657 zone = &zlo->zones[zone_no]; 658 659 /* 660 * The block layer should never send requests that are not fully 661 * contained within the zone. 662 */ 663 if (WARN_ON_ONCE(cmd->sector + nr_sectors > 664 zone->start + zlo->zone_size)) 665 goto out; 666 667 if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { 668 mutex_lock(&zone->lock); 669 ret = zloop_update_seq_zone(zlo, zone_no); 670 mutex_unlock(&zone->lock); 671 if (ret) 672 goto out; 673 } 674 675 if (!test_bit(ZLOOP_ZONE_CONV, &zone->flags) && is_write) { 676 mutex_lock(&zone->lock); 677 ret = zloop_seq_write_prep(cmd); 678 if (!ret) 679 ret = zloop_do_rw(cmd); 680 mutex_unlock(&zone->lock); 681 } else { 682 ret = zloop_do_rw(cmd); 683 } 684 out: 685 if (ret != -EIOCBQUEUED) 686 zloop_rw_complete(&cmd->iocb, ret); 687 zloop_put_cmd(cmd); 688 } 689 690 static inline bool zloop_zone_is_active(struct zloop_zone *zone) 691 { 692 switch (zone->cond) { 693 case BLK_ZONE_COND_EXP_OPEN: 694 case BLK_ZONE_COND_IMP_OPEN: 695 case BLK_ZONE_COND_CLOSED: 696 return true; 697 default: 698 return false; 699 } 700 } 701 702 static int zloop_record_safe_wps(struct zloop_device *zlo) 703 { 704 unsigned int i; 705 int ret; 706 707 for (i = 0; i < zlo->nr_zones; i++) { 708 struct zloop_zone *zone = &zlo->zones[i]; 709 struct file *file = zone->file; 710 711 if (!zloop_zone_is_active(zone)) 712 continue; 713 ret = vfs_setxattr(file_mnt_idmap(file), file_dentry(file), 714 "user.zloop.wp", &zone->wp, sizeof(zone->wp), 0); 715 if (ret) { 716 pr_err("%pg: failed to record write pointer (%d)\n", 717 zlo->disk->part0, ret); 718 return ret; 719 } 720 } 721 722 return 0; 723 } 724 725 /* 726 * Sync the entire FS containing the zone files instead of walking all files. 727 */ 728 static int zloop_flush(struct zloop_device *zlo) 729 { 730 struct super_block *sb = file_inode(zlo->data_dir)->i_sb; 731 int ret; 732 733 if (zlo->discard_write_cache) { 734 ret = zloop_record_safe_wps(zlo); 735 if (ret) 736 return ret; 737 } 738 739 down_read(&sb->s_umount); 740 ret = sync_filesystem(sb); 741 up_read(&sb->s_umount); 742 743 return ret; 744 } 745 746 static void zloop_handle_cmd(struct zloop_cmd *cmd) 747 { 748 struct request *rq = blk_mq_rq_from_pdu(cmd); 749 struct zloop_device *zlo = rq->q->queuedata; 750 751 /* We can block in this context, so ignore REQ_NOWAIT. */ 752 if (rq->cmd_flags & REQ_NOWAIT) 753 rq->cmd_flags &= ~REQ_NOWAIT; 754 755 switch (req_op(rq)) { 756 case REQ_OP_READ: 757 case REQ_OP_WRITE: 758 case REQ_OP_ZONE_APPEND: 759 /* 760 * zloop_rw() always executes asynchronously or completes 761 * directly. 762 */ 763 zloop_rw(cmd); 764 return; 765 case REQ_OP_FLUSH: 766 cmd->ret = zloop_flush(zlo); 767 break; 768 case REQ_OP_ZONE_RESET: 769 cmd->ret = zloop_reset_zone(zlo, rq_zone_no(rq)); 770 break; 771 case REQ_OP_ZONE_RESET_ALL: 772 cmd->ret = zloop_reset_all_zones(zlo); 773 break; 774 case REQ_OP_ZONE_FINISH: 775 cmd->ret = zloop_finish_zone(zlo, rq_zone_no(rq)); 776 break; 777 case REQ_OP_ZONE_OPEN: 778 cmd->ret = zloop_open_zone(zlo, rq_zone_no(rq)); 779 break; 780 case REQ_OP_ZONE_CLOSE: 781 cmd->ret = zloop_close_zone(zlo, rq_zone_no(rq)); 782 break; 783 default: 784 WARN_ON_ONCE(1); 785 pr_err("Unsupported operation %d\n", req_op(rq)); 786 cmd->ret = -EOPNOTSUPP; 787 break; 788 } 789 790 blk_mq_complete_request(rq); 791 } 792 793 static void zloop_cmd_workfn(struct work_struct *work) 794 { 795 struct zloop_cmd *cmd = container_of(work, struct zloop_cmd, work); 796 int orig_flags = current->flags; 797 798 current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; 799 zloop_handle_cmd(cmd); 800 current->flags = orig_flags; 801 } 802 803 static void zloop_complete_rq(struct request *rq) 804 { 805 struct zloop_cmd *cmd = blk_mq_rq_to_pdu(rq); 806 struct zloop_device *zlo = rq->q->queuedata; 807 unsigned int zone_no = cmd->sector >> zlo->zone_shift; 808 struct zloop_zone *zone = &zlo->zones[zone_no]; 809 blk_status_t sts = BLK_STS_OK; 810 811 switch (req_op(rq)) { 812 case REQ_OP_READ: 813 if (cmd->ret < 0) 814 pr_err("Zone %u: failed read sector %llu, %llu sectors\n", 815 zone_no, cmd->sector, cmd->nr_sectors); 816 817 if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq)) { 818 /* short read */ 819 struct bio *bio; 820 821 __rq_for_each_bio(bio, rq) 822 zero_fill_bio(bio); 823 } 824 break; 825 case REQ_OP_WRITE: 826 case REQ_OP_ZONE_APPEND: 827 if (cmd->ret < 0) 828 pr_err("Zone %u: failed %swrite sector %llu, %llu sectors\n", 829 zone_no, 830 req_op(rq) == REQ_OP_WRITE ? "" : "append ", 831 cmd->sector, cmd->nr_sectors); 832 833 if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq)) { 834 pr_err("Zone %u: partial write %ld/%u B\n", 835 zone_no, cmd->ret, blk_rq_bytes(rq)); 836 cmd->ret = -EIO; 837 } 838 839 if (cmd->ret < 0 && !test_bit(ZLOOP_ZONE_CONV, &zone->flags)) { 840 /* 841 * A write to a sequential zone file failed: mark the 842 * zone as having an error. This will be corrected and 843 * cleared when the next IO is submitted. 844 */ 845 set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); 846 break; 847 } 848 if (req_op(rq) == REQ_OP_ZONE_APPEND) 849 rq->__sector = cmd->sector; 850 851 break; 852 default: 853 break; 854 } 855 856 if (cmd->ret < 0) 857 sts = errno_to_blk_status(cmd->ret); 858 blk_mq_end_request(rq, sts); 859 } 860 861 static bool zloop_set_zone_append_sector(struct request *rq) 862 { 863 struct zloop_device *zlo = rq->q->queuedata; 864 unsigned int zone_no = rq_zone_no(rq); 865 struct zloop_zone *zone = &zlo->zones[zone_no]; 866 sector_t zone_end = zone->start + zlo->zone_capacity; 867 sector_t nr_sectors = blk_rq_sectors(rq); 868 869 spin_lock(&zone->wp_lock); 870 871 if (zone->cond == BLK_ZONE_COND_FULL || 872 zone->wp + nr_sectors > zone_end) { 873 spin_unlock(&zone->wp_lock); 874 return false; 875 } 876 877 rq->__sector = zone->wp; 878 zone->wp += blk_rq_sectors(rq); 879 if (zone->wp >= zone_end) 880 zloop_mark_full(zlo, zone); 881 882 spin_unlock(&zone->wp_lock); 883 884 return true; 885 } 886 887 static blk_status_t zloop_queue_rq(struct blk_mq_hw_ctx *hctx, 888 const struct blk_mq_queue_data *bd) 889 { 890 struct request *rq = bd->rq; 891 struct zloop_cmd *cmd = blk_mq_rq_to_pdu(rq); 892 struct zloop_device *zlo = rq->q->queuedata; 893 894 if (data_race(READ_ONCE(zlo->state)) == Zlo_deleting) { 895 rq->rq_flags |= RQF_QUIET; 896 return BLK_STS_IOERR; 897 } 898 899 /* 900 * If we need to strongly order zone append operations, set the request 901 * sector to the zone write pointer location now instead of when the 902 * command work runs. 903 */ 904 if (zlo->ordered_zone_append && req_op(rq) == REQ_OP_ZONE_APPEND) { 905 if (!zloop_set_zone_append_sector(rq)) 906 return BLK_STS_IOERR; 907 } 908 909 blk_mq_start_request(rq); 910 911 INIT_WORK(&cmd->work, zloop_cmd_workfn); 912 queue_work(zlo->workqueue, &cmd->work); 913 914 return BLK_STS_OK; 915 } 916 917 static const struct blk_mq_ops zloop_mq_ops = { 918 .queue_rq = zloop_queue_rq, 919 .complete = zloop_complete_rq, 920 }; 921 922 static int zloop_open(struct gendisk *disk, blk_mode_t mode) 923 { 924 struct zloop_device *zlo = disk->private_data; 925 int ret; 926 927 ret = mutex_lock_killable(&zloop_ctl_mutex); 928 if (ret) 929 return ret; 930 931 if (zlo->state != Zlo_live) 932 ret = -ENXIO; 933 mutex_unlock(&zloop_ctl_mutex); 934 return ret; 935 } 936 937 static int zloop_report_zones(struct gendisk *disk, sector_t sector, 938 unsigned int nr_zones, struct blk_report_zones_args *args) 939 { 940 struct zloop_device *zlo = disk->private_data; 941 struct blk_zone blkz = {}; 942 unsigned int first, i; 943 int ret; 944 945 first = disk_zone_no(disk, sector); 946 if (first >= zlo->nr_zones) 947 return 0; 948 nr_zones = min(nr_zones, zlo->nr_zones - first); 949 950 for (i = 0; i < nr_zones; i++) { 951 unsigned int zone_no = first + i; 952 struct zloop_zone *zone = &zlo->zones[zone_no]; 953 954 mutex_lock(&zone->lock); 955 956 if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { 957 ret = zloop_update_seq_zone(zlo, zone_no); 958 if (ret) { 959 mutex_unlock(&zone->lock); 960 return ret; 961 } 962 } 963 964 blkz.start = zone->start; 965 blkz.len = zlo->zone_size; 966 spin_lock(&zone->wp_lock); 967 blkz.wp = zone->wp; 968 spin_unlock(&zone->wp_lock); 969 blkz.cond = zone->cond; 970 if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) { 971 blkz.type = BLK_ZONE_TYPE_CONVENTIONAL; 972 blkz.capacity = zlo->zone_size; 973 } else { 974 blkz.type = BLK_ZONE_TYPE_SEQWRITE_REQ; 975 blkz.capacity = zlo->zone_capacity; 976 } 977 978 mutex_unlock(&zone->lock); 979 980 ret = disk_report_zone(disk, &blkz, i, args); 981 if (ret) 982 return ret; 983 } 984 985 return nr_zones; 986 } 987 988 static void zloop_free_disk(struct gendisk *disk) 989 { 990 struct zloop_device *zlo = disk->private_data; 991 unsigned int i; 992 993 blk_mq_free_tag_set(&zlo->tag_set); 994 995 for (i = 0; i < zlo->nr_zones; i++) { 996 struct zloop_zone *zone = &zlo->zones[i]; 997 998 mapping_set_gfp_mask(zone->file->f_mapping, 999 zone->old_gfp_mask); 1000 fput(zone->file); 1001 } 1002 1003 fput(zlo->data_dir); 1004 destroy_workqueue(zlo->workqueue); 1005 kfree(zlo->base_dir); 1006 kvfree(zlo); 1007 } 1008 1009 static const struct block_device_operations zloop_fops = { 1010 .owner = THIS_MODULE, 1011 .open = zloop_open, 1012 .report_zones = zloop_report_zones, 1013 .free_disk = zloop_free_disk, 1014 }; 1015 1016 __printf(3, 4) 1017 static struct file *zloop_filp_open_fmt(int oflags, umode_t mode, 1018 const char *fmt, ...) 1019 { 1020 struct file *file; 1021 va_list ap; 1022 char *p; 1023 1024 va_start(ap, fmt); 1025 p = kvasprintf(GFP_KERNEL, fmt, ap); 1026 va_end(ap); 1027 1028 if (!p) 1029 return ERR_PTR(-ENOMEM); 1030 file = filp_open(p, oflags, mode); 1031 kfree(p); 1032 return file; 1033 } 1034 1035 static int zloop_get_block_size(struct zloop_device *zlo, 1036 struct zloop_zone *zone) 1037 { 1038 struct block_device *sb_bdev = zone->file->f_mapping->host->i_sb->s_bdev; 1039 struct kstat st; 1040 1041 /* 1042 * Use the dio alignment of the file system if provided. The incoming 1043 * request's bio_vec is forwarded to the backing file unchanged, so its 1044 * required memory alignment becomes the device's dma_alignment when 1045 * used for direct-io. The file system reports zeroed alignments if the 1046 * file can't be used for direct-io at all, so fall back to the block 1047 * device limits in that case. 1048 */ 1049 if (!vfs_getattr(&zone->file->f_path, &st, STATX_DIOALIGN, 0) && 1050 (st.result_mask & STATX_DIOALIGN) && st.dio_mem_align) { 1051 zlo->block_size = st.dio_offset_align; 1052 zlo->dio_mem_align = min(st.dio_mem_align - 1, PAGE_SIZE - 1); 1053 } else if (sb_bdev) { 1054 zlo->block_size = bdev_physical_block_size(sb_bdev); 1055 zlo->dio_mem_align = bdev_dma_alignment(sb_bdev); 1056 } else { 1057 zlo->block_size = SECTOR_SIZE; 1058 zlo->dio_mem_align = SECTOR_SIZE - 1; 1059 } 1060 1061 /* 1062 * Prefer the FS block size for the device block size when it is no 1063 * larger than 4K; otherwise keep the direct I/O / physical block size 1064 * selected above. 1065 */ 1066 if (file_inode(zone->file)->i_sb->s_blocksize <= SZ_4K) 1067 zlo->block_size = file_inode(zone->file)->i_sb->s_blocksize; 1068 1069 if (zlo->zone_capacity & ((zlo->block_size >> SECTOR_SHIFT) - 1)) { 1070 pr_err("Zone capacity is not aligned to block size %u\n", 1071 zlo->block_size); 1072 return -EINVAL; 1073 } 1074 1075 return 0; 1076 } 1077 1078 static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts, 1079 unsigned int zone_no, bool restore) 1080 { 1081 struct zloop_zone *zone = &zlo->zones[zone_no]; 1082 int oflags = O_RDWR; 1083 struct kstat stat; 1084 sector_t file_sectors; 1085 int ret; 1086 1087 mutex_init(&zone->lock); 1088 INIT_LIST_HEAD(&zone->open_zone_entry); 1089 spin_lock_init(&zone->wp_lock); 1090 zone->start = (sector_t)zone_no << zlo->zone_shift; 1091 1092 if (!restore) 1093 oflags |= O_CREAT; 1094 1095 if (!opts->buffered_io) 1096 oflags |= O_DIRECT; 1097 1098 if (zone_no < zlo->nr_conv_zones) { 1099 /* Conventional zone file. */ 1100 set_bit(ZLOOP_ZONE_CONV, &zone->flags); 1101 zone->cond = BLK_ZONE_COND_NOT_WP; 1102 zone->wp = U64_MAX; 1103 1104 zone->file = zloop_filp_open_fmt(oflags, 0600, "%s/%u/cnv-%06u", 1105 zlo->base_dir, zlo->id, zone_no); 1106 if (IS_ERR(zone->file)) { 1107 pr_err("Failed to open zone %u file %s/%u/cnv-%06u (err=%ld)", 1108 zone_no, zlo->base_dir, zlo->id, zone_no, 1109 PTR_ERR(zone->file)); 1110 return PTR_ERR(zone->file); 1111 } 1112 1113 if (!zlo->block_size) { 1114 ret = zloop_get_block_size(zlo, zone); 1115 if (ret) 1116 return ret; 1117 } 1118 1119 ret = vfs_getattr(&zone->file->f_path, &stat, STATX_SIZE, 0); 1120 if (ret < 0) { 1121 pr_err("Failed to get zone %u file stat\n", zone_no); 1122 return ret; 1123 } 1124 file_sectors = stat.size >> SECTOR_SHIFT; 1125 1126 if (restore && file_sectors != zlo->zone_size) { 1127 pr_err("Invalid conventional zone %u file size (%llu sectors != %llu)\n", 1128 zone_no, file_sectors, zlo->zone_capacity); 1129 return ret; 1130 } 1131 1132 ret = vfs_truncate(&zone->file->f_path, 1133 zlo->zone_size << SECTOR_SHIFT); 1134 if (ret < 0) { 1135 pr_err("Failed to truncate zone %u file (err=%d)\n", 1136 zone_no, ret); 1137 return ret; 1138 } 1139 1140 return 0; 1141 } 1142 1143 /* Sequential zone file. */ 1144 zone->file = zloop_filp_open_fmt(oflags, 0600, "%s/%u/seq-%06u", 1145 zlo->base_dir, zlo->id, zone_no); 1146 if (IS_ERR(zone->file)) { 1147 pr_err("Failed to open zone %u file %s/%u/seq-%06u (err=%ld)", 1148 zone_no, zlo->base_dir, zlo->id, zone_no, 1149 PTR_ERR(zone->file)); 1150 return PTR_ERR(zone->file); 1151 } 1152 1153 if (!zlo->block_size) { 1154 ret = zloop_get_block_size(zlo, zone); 1155 if (ret) 1156 return ret; 1157 } 1158 1159 zloop_get_block_size(zlo, zone); 1160 1161 mutex_lock(&zone->lock); 1162 ret = zloop_update_seq_zone(zlo, zone_no); 1163 mutex_unlock(&zone->lock); 1164 1165 return ret; 1166 } 1167 1168 static bool zloop_dev_exists(struct zloop_device *zlo) 1169 { 1170 struct file *cnv, *seq; 1171 bool exists; 1172 1173 cnv = zloop_filp_open_fmt(O_RDONLY, 0600, "%s/%u/cnv-%06u", 1174 zlo->base_dir, zlo->id, 0); 1175 seq = zloop_filp_open_fmt(O_RDONLY, 0600, "%s/%u/seq-%06u", 1176 zlo->base_dir, zlo->id, 0); 1177 exists = !IS_ERR(cnv) || !IS_ERR(seq); 1178 1179 if (!IS_ERR(cnv)) 1180 fput(cnv); 1181 if (!IS_ERR(seq)) 1182 fput(seq); 1183 1184 return exists; 1185 } 1186 1187 static int zloop_ctl_add(struct zloop_options *opts) 1188 { 1189 struct queue_limits lim = { 1190 .max_hw_sectors = SZ_1M >> SECTOR_SHIFT, 1191 .chunk_sectors = opts->zone_size, 1192 .features = BLK_FEAT_ZONED | BLK_FEAT_WRITE_CACHE, 1193 1194 }; 1195 unsigned int nr_zones, i, j; 1196 struct zloop_device *zlo; 1197 int ret = -EINVAL; 1198 bool restore; 1199 1200 __module_get(THIS_MODULE); 1201 1202 nr_zones = opts->capacity >> ilog2(opts->zone_size); 1203 if (opts->nr_conv_zones >= nr_zones) { 1204 pr_err("Invalid number of conventional zones %u\n", 1205 opts->nr_conv_zones); 1206 goto out; 1207 } 1208 1209 if (opts->max_open_zones > nr_zones - opts->nr_conv_zones) { 1210 pr_err("Invalid maximum number of open zones %u\n", 1211 opts->max_open_zones); 1212 goto out; 1213 } 1214 1215 zlo = kvzalloc_flex(*zlo, zones, nr_zones); 1216 if (!zlo) { 1217 ret = -ENOMEM; 1218 goto out; 1219 } 1220 WRITE_ONCE(zlo->state, Zlo_creating); 1221 spin_lock_init(&zlo->open_zones_lock); 1222 INIT_LIST_HEAD(&zlo->open_zones_lru_list); 1223 1224 ret = mutex_lock_killable(&zloop_ctl_mutex); 1225 if (ret) 1226 goto out_free_dev; 1227 1228 /* Allocate id, if @opts->id >= 0, we're requesting that specific id */ 1229 if (opts->id >= 0) { 1230 ret = idr_alloc(&zloop_index_idr, zlo, 1231 opts->id, opts->id + 1, GFP_KERNEL); 1232 if (ret == -ENOSPC) 1233 ret = -EEXIST; 1234 } else { 1235 ret = idr_alloc(&zloop_index_idr, zlo, 0, 0, GFP_KERNEL); 1236 } 1237 mutex_unlock(&zloop_ctl_mutex); 1238 if (ret < 0) 1239 goto out_free_dev; 1240 1241 zlo->id = ret; 1242 zlo->zone_shift = ilog2(opts->zone_size); 1243 zlo->zone_size = opts->zone_size; 1244 if (opts->zone_capacity) 1245 zlo->zone_capacity = opts->zone_capacity; 1246 else 1247 zlo->zone_capacity = zlo->zone_size; 1248 zlo->nr_zones = nr_zones; 1249 zlo->nr_conv_zones = opts->nr_conv_zones; 1250 zlo->max_open_zones = opts->max_open_zones; 1251 zlo->buffered_io = opts->buffered_io; 1252 zlo->zone_append = opts->zone_append; 1253 if (zlo->zone_append) 1254 zlo->ordered_zone_append = opts->ordered_zone_append; 1255 zlo->discard_write_cache = opts->discard_write_cache; 1256 1257 zlo->workqueue = alloc_workqueue("zloop%d", WQ_UNBOUND | WQ_FREEZABLE, 1258 opts->nr_queues * opts->queue_depth, zlo->id); 1259 if (!zlo->workqueue) { 1260 ret = -ENOMEM; 1261 goto out_free_idr; 1262 } 1263 1264 if (opts->base_dir) 1265 zlo->base_dir = kstrdup(opts->base_dir, GFP_KERNEL); 1266 else 1267 zlo->base_dir = kstrdup(ZLOOP_DEF_BASE_DIR, GFP_KERNEL); 1268 if (!zlo->base_dir) { 1269 ret = -ENOMEM; 1270 goto out_destroy_workqueue; 1271 } 1272 1273 zlo->data_dir = zloop_filp_open_fmt(O_RDONLY | O_DIRECTORY, 0, "%s/%u", 1274 zlo->base_dir, zlo->id); 1275 if (IS_ERR(zlo->data_dir)) { 1276 ret = PTR_ERR(zlo->data_dir); 1277 pr_warn("Failed to open directory %s/%u (err=%d)\n", 1278 zlo->base_dir, zlo->id, ret); 1279 goto out_free_base_dir; 1280 } 1281 1282 /* 1283 * If we already have zone files, we are restoring a device created by a 1284 * previous add operation. In this case, zloop_init_zone() will check 1285 * that the zone files are consistent with the zone configuration given. 1286 */ 1287 restore = zloop_dev_exists(zlo); 1288 for (i = 0; i < nr_zones; i++) { 1289 ret = zloop_init_zone(zlo, opts, i, restore); 1290 if (ret) 1291 goto out_close_files; 1292 } 1293 1294 lim.physical_block_size = zlo->block_size; 1295 lim.logical_block_size = zlo->block_size; 1296 /* Direct I/O forwards the request pages to the backing files as-is. */ 1297 if (!opts->buffered_io) 1298 lim.dma_alignment = max_t(unsigned int, zlo->dio_mem_align, 1299 SECTOR_SIZE - 1); 1300 if (zlo->zone_append) 1301 lim.max_hw_zone_append_sectors = lim.max_hw_sectors; 1302 lim.max_open_zones = zlo->max_open_zones; 1303 1304 zlo->tag_set.ops = &zloop_mq_ops; 1305 zlo->tag_set.nr_hw_queues = opts->nr_queues; 1306 zlo->tag_set.queue_depth = opts->queue_depth; 1307 zlo->tag_set.numa_node = NUMA_NO_NODE; 1308 zlo->tag_set.cmd_size = sizeof(struct zloop_cmd); 1309 zlo->tag_set.driver_data = zlo; 1310 1311 ret = blk_mq_alloc_tag_set(&zlo->tag_set); 1312 if (ret) { 1313 pr_err("blk_mq_alloc_tag_set failed (err=%d)\n", ret); 1314 goto out_close_files; 1315 } 1316 1317 zlo->disk = blk_mq_alloc_disk(&zlo->tag_set, &lim, zlo); 1318 if (IS_ERR(zlo->disk)) { 1319 pr_err("blk_mq_alloc_disk failed (err=%d)\n", ret); 1320 ret = PTR_ERR(zlo->disk); 1321 goto out_cleanup_tags; 1322 } 1323 zlo->disk->flags = GENHD_FL_NO_PART; 1324 zlo->disk->fops = &zloop_fops; 1325 zlo->disk->private_data = zlo; 1326 sprintf(zlo->disk->disk_name, "zloop%d", zlo->id); 1327 set_capacity(zlo->disk, (u64)lim.chunk_sectors * zlo->nr_zones); 1328 1329 ret = blk_revalidate_disk_zones(zlo->disk); 1330 if (ret) 1331 goto out_cleanup_disk; 1332 1333 ret = add_disk(zlo->disk); 1334 if (ret) { 1335 pr_err("add_disk failed (err=%d)\n", ret); 1336 goto out_cleanup_disk; 1337 } 1338 1339 mutex_lock(&zloop_ctl_mutex); 1340 WRITE_ONCE(zlo->state, Zlo_live); 1341 mutex_unlock(&zloop_ctl_mutex); 1342 1343 pr_info("zloop: device %d, %u zones of %llu MiB, %u B block size\n", 1344 zlo->id, zlo->nr_zones, 1345 ((sector_t)zlo->zone_size << SECTOR_SHIFT) >> 20, 1346 zlo->block_size); 1347 pr_info("zloop%d: using %s%s zone append\n", 1348 zlo->id, 1349 zlo->ordered_zone_append ? "ordered " : "", 1350 zlo->zone_append ? "native" : "emulated"); 1351 1352 return 0; 1353 1354 out_cleanup_disk: 1355 put_disk(zlo->disk); 1356 out_cleanup_tags: 1357 blk_mq_free_tag_set(&zlo->tag_set); 1358 out_close_files: 1359 for (j = 0; j < i; j++) { 1360 struct zloop_zone *zone = &zlo->zones[j]; 1361 1362 if (!IS_ERR_OR_NULL(zone->file)) 1363 fput(zone->file); 1364 } 1365 fput(zlo->data_dir); 1366 out_free_base_dir: 1367 kfree(zlo->base_dir); 1368 out_destroy_workqueue: 1369 destroy_workqueue(zlo->workqueue); 1370 out_free_idr: 1371 mutex_lock(&zloop_ctl_mutex); 1372 idr_remove(&zloop_index_idr, zlo->id); 1373 mutex_unlock(&zloop_ctl_mutex); 1374 out_free_dev: 1375 kvfree(zlo); 1376 out: 1377 module_put(THIS_MODULE); 1378 if (ret == -ENOENT) 1379 ret = -EINVAL; 1380 return ret; 1381 } 1382 1383 static void zloop_forget_cache(struct zloop_device *zlo) 1384 { 1385 unsigned int i; 1386 int ret; 1387 1388 pr_info("%pg: discarding volatile write cache\n", zlo->disk->part0); 1389 1390 for (i = 0; i < zlo->nr_zones; i++) { 1391 struct zloop_zone *zone = &zlo->zones[i]; 1392 struct file *file = zone->file; 1393 sector_t old_wp; 1394 1395 if (!zloop_zone_is_active(zone)) 1396 continue; 1397 1398 ret = vfs_getxattr(file_mnt_idmap(file), file_dentry(file), 1399 "user.zloop.wp", &old_wp, sizeof(old_wp)); 1400 if (ret == -ENODATA) { 1401 old_wp = 0; 1402 } else if (ret != sizeof(old_wp)) { 1403 pr_err("%pg: failed to retrieve write pointer (%d)\n", 1404 zlo->disk->part0, ret); 1405 continue; 1406 } 1407 1408 if (old_wp > zone->wp) 1409 continue; 1410 /* 1411 * This should not happen, if we recored a full zone, it can't 1412 * be active. 1413 */ 1414 if (WARN_ON_ONCE(old_wp == ULLONG_MAX)) 1415 continue; 1416 1417 vfs_truncate(&file->f_path, 1418 (old_wp - zone->start) << SECTOR_SHIFT); 1419 } 1420 } 1421 1422 static int zloop_ctl_remove(struct zloop_options *opts) 1423 { 1424 struct zloop_device *zlo; 1425 int ret; 1426 1427 if (!(opts->mask & ZLOOP_OPT_ID)) { 1428 pr_err("No ID specified for remove\n"); 1429 return -EINVAL; 1430 } 1431 1432 if (opts->mask & ~ZLOOP_OPT_ID) { 1433 pr_err("Invalid option specified for remove\n"); 1434 return -EINVAL; 1435 } 1436 1437 ret = mutex_lock_killable(&zloop_ctl_mutex); 1438 if (ret) 1439 return ret; 1440 1441 zlo = idr_find(&zloop_index_idr, opts->id); 1442 if (!zlo || zlo->state == Zlo_creating) { 1443 ret = -ENODEV; 1444 } else if (zlo->state == Zlo_deleting) { 1445 ret = -EINVAL; 1446 } else { 1447 idr_remove(&zloop_index_idr, zlo->id); 1448 WRITE_ONCE(zlo->state, Zlo_deleting); 1449 } 1450 1451 mutex_unlock(&zloop_ctl_mutex); 1452 if (ret) 1453 return ret; 1454 1455 del_gendisk(zlo->disk); 1456 1457 if (zlo->discard_write_cache) 1458 zloop_forget_cache(zlo); 1459 1460 put_disk(zlo->disk); 1461 1462 pr_info("Removed device %d\n", opts->id); 1463 1464 module_put(THIS_MODULE); 1465 1466 return 0; 1467 } 1468 1469 static int zloop_parse_options(struct zloop_options *opts, const char *buf) 1470 { 1471 substring_t args[MAX_OPT_ARGS]; 1472 char *options, *o, *p; 1473 unsigned int token; 1474 int ret = 0; 1475 1476 /* Set defaults. */ 1477 opts->mask = 0; 1478 opts->id = ZLOOP_DEF_ID; 1479 opts->capacity = ZLOOP_DEF_ZONE_SIZE * ZLOOP_DEF_NR_ZONES; 1480 opts->zone_size = ZLOOP_DEF_ZONE_SIZE; 1481 opts->nr_conv_zones = ZLOOP_DEF_NR_CONV_ZONES; 1482 opts->max_open_zones = ZLOOP_DEF_MAX_OPEN_ZONES; 1483 opts->nr_queues = ZLOOP_DEF_NR_QUEUES; 1484 opts->queue_depth = ZLOOP_DEF_QUEUE_DEPTH; 1485 opts->buffered_io = ZLOOP_DEF_BUFFERED_IO; 1486 opts->zone_append = ZLOOP_DEF_ZONE_APPEND; 1487 opts->ordered_zone_append = ZLOOP_DEF_ORDERED_ZONE_APPEND; 1488 1489 if (!buf) 1490 return 0; 1491 1492 /* Skip leading spaces before the options. */ 1493 while (isspace(*buf)) 1494 buf++; 1495 1496 options = o = kstrdup(buf, GFP_KERNEL); 1497 if (!options) 1498 return -ENOMEM; 1499 1500 /* Parse the options, doing only some light invalid value checks. */ 1501 while ((p = strsep(&o, ",\n")) != NULL) { 1502 if (!*p) 1503 continue; 1504 1505 token = match_token(p, zloop_opt_tokens, args); 1506 opts->mask |= token; 1507 switch (token) { 1508 case ZLOOP_OPT_ID: 1509 if (match_int(args, &opts->id)) { 1510 ret = -EINVAL; 1511 goto out; 1512 } 1513 break; 1514 case ZLOOP_OPT_CAPACITY: 1515 if (match_uint(args, &token)) { 1516 ret = -EINVAL; 1517 goto out; 1518 } 1519 if (!token) { 1520 pr_err("Invalid capacity\n"); 1521 ret = -EINVAL; 1522 goto out; 1523 } 1524 opts->capacity = 1525 ((sector_t)token * SZ_1M) >> SECTOR_SHIFT; 1526 break; 1527 case ZLOOP_OPT_ZONE_SIZE: 1528 if (match_uint(args, &token)) { 1529 ret = -EINVAL; 1530 goto out; 1531 } 1532 if (!token || token > ZLOOP_MAX_ZONE_SIZE_MB || 1533 !is_power_of_2(token)) { 1534 pr_err("Invalid zone size %u\n", token); 1535 ret = -EINVAL; 1536 goto out; 1537 } 1538 opts->zone_size = 1539 ((sector_t)token * SZ_1M) >> SECTOR_SHIFT; 1540 break; 1541 case ZLOOP_OPT_ZONE_CAPACITY: 1542 if (match_uint(args, &token)) { 1543 ret = -EINVAL; 1544 goto out; 1545 } 1546 if (!token) { 1547 pr_err("Invalid zone capacity\n"); 1548 ret = -EINVAL; 1549 goto out; 1550 } 1551 opts->zone_capacity = 1552 ((sector_t)token * SZ_1M) >> SECTOR_SHIFT; 1553 break; 1554 case ZLOOP_OPT_NR_CONV_ZONES: 1555 if (match_uint(args, &token)) { 1556 ret = -EINVAL; 1557 goto out; 1558 } 1559 opts->nr_conv_zones = token; 1560 break; 1561 case ZLOOP_OPT_MAX_OPEN_ZONES: 1562 if (match_uint(args, &token)) { 1563 ret = -EINVAL; 1564 goto out; 1565 } 1566 opts->max_open_zones = token; 1567 break; 1568 case ZLOOP_OPT_BASE_DIR: 1569 p = match_strdup(args); 1570 if (!p) { 1571 ret = -ENOMEM; 1572 goto out; 1573 } 1574 kfree(opts->base_dir); 1575 opts->base_dir = p; 1576 break; 1577 case ZLOOP_OPT_NR_QUEUES: 1578 if (match_uint(args, &token)) { 1579 ret = -EINVAL; 1580 goto out; 1581 } 1582 if (!token) { 1583 pr_err("Invalid number of queues\n"); 1584 ret = -EINVAL; 1585 goto out; 1586 } 1587 opts->nr_queues = min(token, num_online_cpus()); 1588 break; 1589 case ZLOOP_OPT_QUEUE_DEPTH: 1590 if (match_uint(args, &token)) { 1591 ret = -EINVAL; 1592 goto out; 1593 } 1594 if (!token) { 1595 pr_err("Invalid queue depth\n"); 1596 ret = -EINVAL; 1597 goto out; 1598 } 1599 opts->queue_depth = token; 1600 break; 1601 case ZLOOP_OPT_BUFFERED_IO: 1602 opts->buffered_io = true; 1603 break; 1604 case ZLOOP_OPT_ZONE_APPEND: 1605 if (match_uint(args, &token)) { 1606 ret = -EINVAL; 1607 goto out; 1608 } 1609 if (token != 0 && token != 1) { 1610 pr_err("Invalid zone_append value\n"); 1611 ret = -EINVAL; 1612 goto out; 1613 } 1614 opts->zone_append = token; 1615 break; 1616 case ZLOOP_OPT_ORDERED_ZONE_APPEND: 1617 opts->ordered_zone_append = true; 1618 break; 1619 case ZLOOP_OPT_DISCARD_WRITE_CACHE: 1620 opts->discard_write_cache = true; 1621 break; 1622 case ZLOOP_OPT_ERR: 1623 default: 1624 pr_warn("unknown parameter or missing value '%s'\n", p); 1625 ret = -EINVAL; 1626 goto out; 1627 } 1628 } 1629 1630 ret = -EINVAL; 1631 if (opts->capacity <= opts->zone_size) { 1632 pr_err("Invalid capacity\n"); 1633 goto out; 1634 } 1635 1636 if (opts->zone_capacity > opts->zone_size) { 1637 pr_err("Invalid zone capacity\n"); 1638 goto out; 1639 } 1640 1641 ret = 0; 1642 out: 1643 kfree(options); 1644 return ret; 1645 } 1646 1647 enum { 1648 ZLOOP_CTL_ADD, 1649 ZLOOP_CTL_REMOVE, 1650 }; 1651 1652 static struct zloop_ctl_op { 1653 int code; 1654 const char *name; 1655 } zloop_ctl_ops[] = { 1656 { ZLOOP_CTL_ADD, "add" }, 1657 { ZLOOP_CTL_REMOVE, "remove" }, 1658 { -1, NULL }, 1659 }; 1660 1661 static ssize_t zloop_ctl_write(struct file *file, const char __user *ubuf, 1662 size_t count, loff_t *pos) 1663 { 1664 struct zloop_options opts = { }; 1665 struct zloop_ctl_op *op; 1666 const char *buf, *opts_buf; 1667 int i, ret; 1668 1669 if (count > PAGE_SIZE) 1670 return -ENOMEM; 1671 1672 buf = memdup_user_nul(ubuf, count); 1673 if (IS_ERR(buf)) 1674 return PTR_ERR(buf); 1675 1676 for (i = 0; i < ARRAY_SIZE(zloop_ctl_ops); i++) { 1677 op = &zloop_ctl_ops[i]; 1678 if (!op->name) { 1679 pr_err("Invalid operation\n"); 1680 ret = -EINVAL; 1681 goto out; 1682 } 1683 if (!strncmp(buf, op->name, strlen(op->name))) 1684 break; 1685 } 1686 1687 if (count <= strlen(op->name)) 1688 opts_buf = NULL; 1689 else 1690 opts_buf = buf + strlen(op->name); 1691 1692 ret = zloop_parse_options(&opts, opts_buf); 1693 if (ret) { 1694 pr_err("Failed to parse options\n"); 1695 goto out; 1696 } 1697 1698 switch (op->code) { 1699 case ZLOOP_CTL_ADD: 1700 ret = zloop_ctl_add(&opts); 1701 break; 1702 case ZLOOP_CTL_REMOVE: 1703 ret = zloop_ctl_remove(&opts); 1704 break; 1705 default: 1706 pr_err("Invalid operation\n"); 1707 ret = -EINVAL; 1708 goto out; 1709 } 1710 1711 out: 1712 kfree(opts.base_dir); 1713 kfree(buf); 1714 return ret ? ret : count; 1715 } 1716 1717 static int zloop_ctl_show(struct seq_file *seq_file, void *private) 1718 { 1719 const struct match_token *tok; 1720 int i; 1721 1722 /* Add operation */ 1723 seq_printf(seq_file, "%s ", zloop_ctl_ops[0].name); 1724 for (i = 0; i < ARRAY_SIZE(zloop_opt_tokens); i++) { 1725 tok = &zloop_opt_tokens[i]; 1726 if (!tok->pattern) 1727 break; 1728 if (i) 1729 seq_putc(seq_file, ','); 1730 seq_puts(seq_file, tok->pattern); 1731 } 1732 seq_putc(seq_file, '\n'); 1733 1734 /* Remove operation */ 1735 seq_puts(seq_file, zloop_ctl_ops[1].name); 1736 seq_puts(seq_file, " id=%d\n"); 1737 1738 return 0; 1739 } 1740 1741 static int zloop_ctl_open(struct inode *inode, struct file *file) 1742 { 1743 file->private_data = NULL; 1744 return single_open(file, zloop_ctl_show, NULL); 1745 } 1746 1747 static int zloop_ctl_release(struct inode *inode, struct file *file) 1748 { 1749 return single_release(inode, file); 1750 } 1751 1752 static const struct file_operations zloop_ctl_fops = { 1753 .owner = THIS_MODULE, 1754 .open = zloop_ctl_open, 1755 .release = zloop_ctl_release, 1756 .write = zloop_ctl_write, 1757 .read = seq_read, 1758 }; 1759 1760 static struct miscdevice zloop_misc = { 1761 .minor = MISC_DYNAMIC_MINOR, 1762 .name = "zloop-control", 1763 .fops = &zloop_ctl_fops, 1764 }; 1765 1766 static int __init zloop_init(void) 1767 { 1768 int ret; 1769 1770 ret = misc_register(&zloop_misc); 1771 if (ret) { 1772 pr_err("Failed to register misc device: %d\n", ret); 1773 return ret; 1774 } 1775 pr_info("Module loaded\n"); 1776 1777 return 0; 1778 } 1779 1780 static void __exit zloop_exit(void) 1781 { 1782 misc_deregister(&zloop_misc); 1783 idr_destroy(&zloop_index_idr); 1784 } 1785 1786 module_init(zloop_init); 1787 module_exit(zloop_exit); 1788 1789 MODULE_DESCRIPTION("Zoned loopback device"); 1790 MODULE_LICENSE("GPL"); 1791