1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/blkdev.h> 4 #include <linux/wait.h> 5 #include <linux/rbtree.h> 6 #include <linux/kthread.h> 7 #include <linux/backing-dev.h> 8 #include <linux/blk-cgroup.h> 9 #include <linux/freezer.h> 10 #include <linux/fs.h> 11 #include <linux/pagemap.h> 12 #include <linux/mm.h> 13 #include <linux/sched/mm.h> 14 #include <linux/sched.h> 15 #include <linux/module.h> 16 #include <linux/writeback.h> 17 #include <linux/device.h> 18 #include <trace/events/writeback.h> 19 20 struct backing_dev_info noop_backing_dev_info; 21 EXPORT_SYMBOL_GPL(noop_backing_dev_info); 22 23 static const char *bdi_unknown_name = "(unknown)"; 24 25 /* 26 * bdi_lock protects bdi_tree and updates to bdi_list. bdi_list has RCU 27 * reader side locking. 28 */ 29 DEFINE_SPINLOCK(bdi_lock); 30 static u64 bdi_id_cursor; 31 static struct rb_root bdi_tree = RB_ROOT; 32 LIST_HEAD(bdi_list); 33 34 /* bdi_wq serves all asynchronous writeback tasks */ 35 struct workqueue_struct *bdi_wq; 36 37 #define K(x) ((x) << (PAGE_SHIFT - 10)) 38 39 #ifdef CONFIG_DEBUG_FS 40 #include <linux/debugfs.h> 41 #include <linux/seq_file.h> 42 43 static struct dentry *bdi_debug_root; 44 45 static void bdi_debug_init(void) 46 { 47 bdi_debug_root = debugfs_create_dir("bdi", NULL); 48 } 49 50 static int bdi_debug_stats_show(struct seq_file *m, void *v) 51 { 52 struct backing_dev_info *bdi = m->private; 53 struct bdi_writeback *wb = &bdi->wb; 54 unsigned long background_thresh; 55 unsigned long dirty_thresh; 56 unsigned long wb_thresh; 57 unsigned long nr_dirty, nr_io, nr_more_io, nr_dirty_time; 58 struct inode *inode; 59 60 nr_dirty = nr_io = nr_more_io = nr_dirty_time = 0; 61 spin_lock(&wb->list_lock); 62 list_for_each_entry(inode, &wb->b_dirty, i_io_list) 63 nr_dirty++; 64 list_for_each_entry(inode, &wb->b_io, i_io_list) 65 nr_io++; 66 list_for_each_entry(inode, &wb->b_more_io, i_io_list) 67 nr_more_io++; 68 list_for_each_entry(inode, &wb->b_dirty_time, i_io_list) 69 if (inode->i_state & I_DIRTY_TIME) 70 nr_dirty_time++; 71 spin_unlock(&wb->list_lock); 72 73 global_dirty_limits(&background_thresh, &dirty_thresh); 74 wb_thresh = wb_calc_thresh(wb, dirty_thresh); 75 76 seq_printf(m, 77 "BdiWriteback: %10lu kB\n" 78 "BdiReclaimable: %10lu kB\n" 79 "BdiDirtyThresh: %10lu kB\n" 80 "DirtyThresh: %10lu kB\n" 81 "BackgroundThresh: %10lu kB\n" 82 "BdiDirtied: %10lu kB\n" 83 "BdiWritten: %10lu kB\n" 84 "BdiWriteBandwidth: %10lu kBps\n" 85 "b_dirty: %10lu\n" 86 "b_io: %10lu\n" 87 "b_more_io: %10lu\n" 88 "b_dirty_time: %10lu\n" 89 "bdi_list: %10u\n" 90 "state: %10lx\n", 91 (unsigned long) K(wb_stat(wb, WB_WRITEBACK)), 92 (unsigned long) K(wb_stat(wb, WB_RECLAIMABLE)), 93 K(wb_thresh), 94 K(dirty_thresh), 95 K(background_thresh), 96 (unsigned long) K(wb_stat(wb, WB_DIRTIED)), 97 (unsigned long) K(wb_stat(wb, WB_WRITTEN)), 98 (unsigned long) K(wb->write_bandwidth), 99 nr_dirty, 100 nr_io, 101 nr_more_io, 102 nr_dirty_time, 103 !list_empty(&bdi->bdi_list), bdi->wb.state); 104 105 return 0; 106 } 107 DEFINE_SHOW_ATTRIBUTE(bdi_debug_stats); 108 109 static void bdi_debug_register(struct backing_dev_info *bdi, const char *name) 110 { 111 bdi->debug_dir = debugfs_create_dir(name, bdi_debug_root); 112 113 debugfs_create_file("stats", 0444, bdi->debug_dir, bdi, 114 &bdi_debug_stats_fops); 115 } 116 117 static void bdi_debug_unregister(struct backing_dev_info *bdi) 118 { 119 debugfs_remove_recursive(bdi->debug_dir); 120 } 121 #else 122 static inline void bdi_debug_init(void) 123 { 124 } 125 static inline void bdi_debug_register(struct backing_dev_info *bdi, 126 const char *name) 127 { 128 } 129 static inline void bdi_debug_unregister(struct backing_dev_info *bdi) 130 { 131 } 132 #endif 133 134 static ssize_t read_ahead_kb_store(struct device *dev, 135 struct device_attribute *attr, 136 const char *buf, size_t count) 137 { 138 struct backing_dev_info *bdi = dev_get_drvdata(dev); 139 unsigned long read_ahead_kb; 140 ssize_t ret; 141 142 ret = kstrtoul(buf, 10, &read_ahead_kb); 143 if (ret < 0) 144 return ret; 145 146 bdi->ra_pages = read_ahead_kb >> (PAGE_SHIFT - 10); 147 148 return count; 149 } 150 151 #define BDI_SHOW(name, expr) \ 152 static ssize_t name##_show(struct device *dev, \ 153 struct device_attribute *attr, char *buf) \ 154 { \ 155 struct backing_dev_info *bdi = dev_get_drvdata(dev); \ 156 \ 157 return sysfs_emit(buf, "%lld\n", (long long)expr); \ 158 } \ 159 static DEVICE_ATTR_RW(name); 160 161 BDI_SHOW(read_ahead_kb, K(bdi->ra_pages)) 162 163 static ssize_t min_ratio_store(struct device *dev, 164 struct device_attribute *attr, const char *buf, size_t count) 165 { 166 struct backing_dev_info *bdi = dev_get_drvdata(dev); 167 unsigned int ratio; 168 ssize_t ret; 169 170 ret = kstrtouint(buf, 10, &ratio); 171 if (ret < 0) 172 return ret; 173 174 ret = bdi_set_min_ratio(bdi, ratio); 175 if (!ret) 176 ret = count; 177 178 return ret; 179 } 180 BDI_SHOW(min_ratio, bdi->min_ratio / BDI_RATIO_SCALE) 181 182 static ssize_t min_ratio_fine_store(struct device *dev, 183 struct device_attribute *attr, const char *buf, size_t count) 184 { 185 struct backing_dev_info *bdi = dev_get_drvdata(dev); 186 unsigned int ratio; 187 ssize_t ret; 188 189 ret = kstrtouint(buf, 10, &ratio); 190 if (ret < 0) 191 return ret; 192 193 ret = bdi_set_min_ratio_no_scale(bdi, ratio); 194 if (!ret) 195 ret = count; 196 197 return ret; 198 } 199 BDI_SHOW(min_ratio_fine, bdi->min_ratio) 200 201 static ssize_t max_ratio_store(struct device *dev, 202 struct device_attribute *attr, const char *buf, size_t count) 203 { 204 struct backing_dev_info *bdi = dev_get_drvdata(dev); 205 unsigned int ratio; 206 ssize_t ret; 207 208 ret = kstrtouint(buf, 10, &ratio); 209 if (ret < 0) 210 return ret; 211 212 ret = bdi_set_max_ratio(bdi, ratio); 213 if (!ret) 214 ret = count; 215 216 return ret; 217 } 218 BDI_SHOW(max_ratio, bdi->max_ratio / BDI_RATIO_SCALE) 219 220 static ssize_t max_ratio_fine_store(struct device *dev, 221 struct device_attribute *attr, const char *buf, size_t count) 222 { 223 struct backing_dev_info *bdi = dev_get_drvdata(dev); 224 unsigned int ratio; 225 ssize_t ret; 226 227 ret = kstrtouint(buf, 10, &ratio); 228 if (ret < 0) 229 return ret; 230 231 ret = bdi_set_max_ratio_no_scale(bdi, ratio); 232 if (!ret) 233 ret = count; 234 235 return ret; 236 } 237 BDI_SHOW(max_ratio_fine, bdi->max_ratio) 238 239 static ssize_t min_bytes_show(struct device *dev, 240 struct device_attribute *attr, 241 char *buf) 242 { 243 struct backing_dev_info *bdi = dev_get_drvdata(dev); 244 245 return sysfs_emit(buf, "%llu\n", bdi_get_min_bytes(bdi)); 246 } 247 248 static ssize_t min_bytes_store(struct device *dev, 249 struct device_attribute *attr, const char *buf, size_t count) 250 { 251 struct backing_dev_info *bdi = dev_get_drvdata(dev); 252 u64 bytes; 253 ssize_t ret; 254 255 ret = kstrtoull(buf, 10, &bytes); 256 if (ret < 0) 257 return ret; 258 259 ret = bdi_set_min_bytes(bdi, bytes); 260 if (!ret) 261 ret = count; 262 263 return ret; 264 } 265 static DEVICE_ATTR_RW(min_bytes); 266 267 static ssize_t max_bytes_show(struct device *dev, 268 struct device_attribute *attr, 269 char *buf) 270 { 271 struct backing_dev_info *bdi = dev_get_drvdata(dev); 272 273 return sysfs_emit(buf, "%llu\n", bdi_get_max_bytes(bdi)); 274 } 275 276 static ssize_t max_bytes_store(struct device *dev, 277 struct device_attribute *attr, const char *buf, size_t count) 278 { 279 struct backing_dev_info *bdi = dev_get_drvdata(dev); 280 u64 bytes; 281 ssize_t ret; 282 283 ret = kstrtoull(buf, 10, &bytes); 284 if (ret < 0) 285 return ret; 286 287 ret = bdi_set_max_bytes(bdi, bytes); 288 if (!ret) 289 ret = count; 290 291 return ret; 292 } 293 static DEVICE_ATTR_RW(max_bytes); 294 295 static ssize_t stable_pages_required_show(struct device *dev, 296 struct device_attribute *attr, 297 char *buf) 298 { 299 dev_warn_once(dev, 300 "the stable_pages_required attribute has been removed. Use the stable_writes queue attribute instead.\n"); 301 return sysfs_emit(buf, "%d\n", 0); 302 } 303 static DEVICE_ATTR_RO(stable_pages_required); 304 305 static ssize_t strict_limit_store(struct device *dev, 306 struct device_attribute *attr, const char *buf, size_t count) 307 { 308 struct backing_dev_info *bdi = dev_get_drvdata(dev); 309 unsigned int strict_limit; 310 ssize_t ret; 311 312 ret = kstrtouint(buf, 10, &strict_limit); 313 if (ret < 0) 314 return ret; 315 316 ret = bdi_set_strict_limit(bdi, strict_limit); 317 if (!ret) 318 ret = count; 319 320 return ret; 321 } 322 323 static ssize_t strict_limit_show(struct device *dev, 324 struct device_attribute *attr, char *buf) 325 { 326 struct backing_dev_info *bdi = dev_get_drvdata(dev); 327 328 return sysfs_emit(buf, "%d\n", 329 !!(bdi->capabilities & BDI_CAP_STRICTLIMIT)); 330 } 331 static DEVICE_ATTR_RW(strict_limit); 332 333 static struct attribute *bdi_dev_attrs[] = { 334 &dev_attr_read_ahead_kb.attr, 335 &dev_attr_min_ratio.attr, 336 &dev_attr_min_ratio_fine.attr, 337 &dev_attr_max_ratio.attr, 338 &dev_attr_max_ratio_fine.attr, 339 &dev_attr_min_bytes.attr, 340 &dev_attr_max_bytes.attr, 341 &dev_attr_stable_pages_required.attr, 342 &dev_attr_strict_limit.attr, 343 NULL, 344 }; 345 ATTRIBUTE_GROUPS(bdi_dev); 346 347 static const struct class bdi_class = { 348 .name = "bdi", 349 .dev_groups = bdi_dev_groups, 350 }; 351 352 static __init int bdi_class_init(void) 353 { 354 int ret; 355 356 ret = class_register(&bdi_class); 357 if (ret) 358 return ret; 359 360 bdi_debug_init(); 361 362 return 0; 363 } 364 postcore_initcall(bdi_class_init); 365 366 static int __init default_bdi_init(void) 367 { 368 bdi_wq = alloc_workqueue("writeback", WQ_MEM_RECLAIM | WQ_UNBOUND | 369 WQ_SYSFS, 0); 370 if (!bdi_wq) 371 return -ENOMEM; 372 return 0; 373 } 374 subsys_initcall(default_bdi_init); 375 376 /* 377 * This function is used when the first inode for this wb is marked dirty. It 378 * wakes-up the corresponding bdi thread which should then take care of the 379 * periodic background write-out of dirty inodes. Since the write-out would 380 * starts only 'dirty_writeback_interval' centisecs from now anyway, we just 381 * set up a timer which wakes the bdi thread up later. 382 * 383 * Note, we wouldn't bother setting up the timer, but this function is on the 384 * fast-path (used by '__mark_inode_dirty()'), so we save few context switches 385 * by delaying the wake-up. 386 * 387 * We have to be careful not to postpone flush work if it is scheduled for 388 * earlier. Thus we use queue_delayed_work(). 389 */ 390 void wb_wakeup_delayed(struct bdi_writeback *wb) 391 { 392 unsigned long timeout; 393 394 timeout = msecs_to_jiffies(dirty_writeback_interval * 10); 395 spin_lock_irq(&wb->work_lock); 396 if (test_bit(WB_registered, &wb->state)) 397 queue_delayed_work(bdi_wq, &wb->dwork, timeout); 398 spin_unlock_irq(&wb->work_lock); 399 } 400 401 static void wb_update_bandwidth_workfn(struct work_struct *work) 402 { 403 struct bdi_writeback *wb = container_of(to_delayed_work(work), 404 struct bdi_writeback, bw_dwork); 405 406 wb_update_bandwidth(wb); 407 } 408 409 /* 410 * Initial write bandwidth: 100 MB/s 411 */ 412 #define INIT_BW (100 << (20 - PAGE_SHIFT)) 413 414 static int wb_init(struct bdi_writeback *wb, struct backing_dev_info *bdi, 415 gfp_t gfp) 416 { 417 int i, err; 418 419 memset(wb, 0, sizeof(*wb)); 420 421 wb->bdi = bdi; 422 wb->last_old_flush = jiffies; 423 INIT_LIST_HEAD(&wb->b_dirty); 424 INIT_LIST_HEAD(&wb->b_io); 425 INIT_LIST_HEAD(&wb->b_more_io); 426 INIT_LIST_HEAD(&wb->b_dirty_time); 427 spin_lock_init(&wb->list_lock); 428 429 atomic_set(&wb->writeback_inodes, 0); 430 wb->bw_time_stamp = jiffies; 431 wb->balanced_dirty_ratelimit = INIT_BW; 432 wb->dirty_ratelimit = INIT_BW; 433 wb->write_bandwidth = INIT_BW; 434 wb->avg_write_bandwidth = INIT_BW; 435 436 spin_lock_init(&wb->work_lock); 437 INIT_LIST_HEAD(&wb->work_list); 438 INIT_DELAYED_WORK(&wb->dwork, wb_workfn); 439 INIT_DELAYED_WORK(&wb->bw_dwork, wb_update_bandwidth_workfn); 440 wb->dirty_sleep = jiffies; 441 442 err = fprop_local_init_percpu(&wb->completions, gfp); 443 if (err) 444 return err; 445 446 for (i = 0; i < NR_WB_STAT_ITEMS; i++) { 447 err = percpu_counter_init(&wb->stat[i], 0, gfp); 448 if (err) 449 goto out_destroy_stat; 450 } 451 452 return 0; 453 454 out_destroy_stat: 455 while (i--) 456 percpu_counter_destroy(&wb->stat[i]); 457 fprop_local_destroy_percpu(&wb->completions); 458 return err; 459 } 460 461 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb); 462 463 /* 464 * Remove bdi from the global list and shutdown any threads we have running 465 */ 466 static void wb_shutdown(struct bdi_writeback *wb) 467 { 468 /* Make sure nobody queues further work */ 469 spin_lock_irq(&wb->work_lock); 470 if (!test_and_clear_bit(WB_registered, &wb->state)) { 471 spin_unlock_irq(&wb->work_lock); 472 return; 473 } 474 spin_unlock_irq(&wb->work_lock); 475 476 cgwb_remove_from_bdi_list(wb); 477 /* 478 * Drain work list and shutdown the delayed_work. !WB_registered 479 * tells wb_workfn() that @wb is dying and its work_list needs to 480 * be drained no matter what. 481 */ 482 mod_delayed_work(bdi_wq, &wb->dwork, 0); 483 flush_delayed_work(&wb->dwork); 484 WARN_ON(!list_empty(&wb->work_list)); 485 flush_delayed_work(&wb->bw_dwork); 486 } 487 488 static void wb_exit(struct bdi_writeback *wb) 489 { 490 int i; 491 492 WARN_ON(delayed_work_pending(&wb->dwork)); 493 494 for (i = 0; i < NR_WB_STAT_ITEMS; i++) 495 percpu_counter_destroy(&wb->stat[i]); 496 497 fprop_local_destroy_percpu(&wb->completions); 498 } 499 500 #ifdef CONFIG_CGROUP_WRITEBACK 501 502 #include <linux/memcontrol.h> 503 504 /* 505 * cgwb_lock protects bdi->cgwb_tree, blkcg->cgwb_list, offline_cgwbs and 506 * memcg->cgwb_list. bdi->cgwb_tree is also RCU protected. 507 */ 508 static DEFINE_SPINLOCK(cgwb_lock); 509 static struct workqueue_struct *cgwb_release_wq; 510 511 static LIST_HEAD(offline_cgwbs); 512 static void cleanup_offline_cgwbs_workfn(struct work_struct *work); 513 static DECLARE_WORK(cleanup_offline_cgwbs_work, cleanup_offline_cgwbs_workfn); 514 515 static void cgwb_free_rcu(struct rcu_head *rcu_head) 516 { 517 struct bdi_writeback *wb = container_of(rcu_head, 518 struct bdi_writeback, rcu); 519 520 percpu_ref_exit(&wb->refcnt); 521 kfree(wb); 522 } 523 524 static void cgwb_release_workfn(struct work_struct *work) 525 { 526 struct bdi_writeback *wb = container_of(work, struct bdi_writeback, 527 release_work); 528 struct backing_dev_info *bdi = wb->bdi; 529 530 mutex_lock(&wb->bdi->cgwb_release_mutex); 531 wb_shutdown(wb); 532 533 css_put(wb->memcg_css); 534 css_put(wb->blkcg_css); 535 mutex_unlock(&wb->bdi->cgwb_release_mutex); 536 537 /* triggers blkg destruction if no online users left */ 538 blkcg_unpin_online(wb->blkcg_css); 539 540 fprop_local_destroy_percpu(&wb->memcg_completions); 541 542 spin_lock_irq(&cgwb_lock); 543 list_del(&wb->offline_node); 544 spin_unlock_irq(&cgwb_lock); 545 546 wb_exit(wb); 547 bdi_put(bdi); 548 WARN_ON_ONCE(!list_empty(&wb->b_attached)); 549 call_rcu(&wb->rcu, cgwb_free_rcu); 550 } 551 552 static void cgwb_release(struct percpu_ref *refcnt) 553 { 554 struct bdi_writeback *wb = container_of(refcnt, struct bdi_writeback, 555 refcnt); 556 queue_work(cgwb_release_wq, &wb->release_work); 557 } 558 559 static void cgwb_kill(struct bdi_writeback *wb) 560 { 561 lockdep_assert_held(&cgwb_lock); 562 563 WARN_ON(!radix_tree_delete(&wb->bdi->cgwb_tree, wb->memcg_css->id)); 564 list_del(&wb->memcg_node); 565 list_del(&wb->blkcg_node); 566 list_add(&wb->offline_node, &offline_cgwbs); 567 percpu_ref_kill(&wb->refcnt); 568 } 569 570 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb) 571 { 572 spin_lock_irq(&cgwb_lock); 573 list_del_rcu(&wb->bdi_node); 574 spin_unlock_irq(&cgwb_lock); 575 } 576 577 static int cgwb_create(struct backing_dev_info *bdi, 578 struct cgroup_subsys_state *memcg_css, gfp_t gfp) 579 { 580 struct mem_cgroup *memcg; 581 struct cgroup_subsys_state *blkcg_css; 582 struct list_head *memcg_cgwb_list, *blkcg_cgwb_list; 583 struct bdi_writeback *wb; 584 unsigned long flags; 585 int ret = 0; 586 587 memcg = mem_cgroup_from_css(memcg_css); 588 blkcg_css = cgroup_get_e_css(memcg_css->cgroup, &io_cgrp_subsys); 589 memcg_cgwb_list = &memcg->cgwb_list; 590 blkcg_cgwb_list = blkcg_get_cgwb_list(blkcg_css); 591 592 /* look up again under lock and discard on blkcg mismatch */ 593 spin_lock_irqsave(&cgwb_lock, flags); 594 wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id); 595 if (wb && wb->blkcg_css != blkcg_css) { 596 cgwb_kill(wb); 597 wb = NULL; 598 } 599 spin_unlock_irqrestore(&cgwb_lock, flags); 600 if (wb) 601 goto out_put; 602 603 /* need to create a new one */ 604 wb = kmalloc(sizeof(*wb), gfp); 605 if (!wb) { 606 ret = -ENOMEM; 607 goto out_put; 608 } 609 610 ret = wb_init(wb, bdi, gfp); 611 if (ret) 612 goto err_free; 613 614 ret = percpu_ref_init(&wb->refcnt, cgwb_release, 0, gfp); 615 if (ret) 616 goto err_wb_exit; 617 618 ret = fprop_local_init_percpu(&wb->memcg_completions, gfp); 619 if (ret) 620 goto err_ref_exit; 621 622 wb->memcg_css = memcg_css; 623 wb->blkcg_css = blkcg_css; 624 INIT_LIST_HEAD(&wb->b_attached); 625 INIT_WORK(&wb->release_work, cgwb_release_workfn); 626 set_bit(WB_registered, &wb->state); 627 bdi_get(bdi); 628 629 /* 630 * The root wb determines the registered state of the whole bdi and 631 * memcg_cgwb_list and blkcg_cgwb_list's next pointers indicate 632 * whether they're still online. Don't link @wb if any is dead. 633 * See wb_memcg_offline() and wb_blkcg_offline(). 634 */ 635 ret = -ENODEV; 636 spin_lock_irqsave(&cgwb_lock, flags); 637 if (test_bit(WB_registered, &bdi->wb.state) && 638 blkcg_cgwb_list->next && memcg_cgwb_list->next) { 639 /* we might have raced another instance of this function */ 640 ret = radix_tree_insert(&bdi->cgwb_tree, memcg_css->id, wb); 641 if (!ret) { 642 list_add_tail_rcu(&wb->bdi_node, &bdi->wb_list); 643 list_add(&wb->memcg_node, memcg_cgwb_list); 644 list_add(&wb->blkcg_node, blkcg_cgwb_list); 645 blkcg_pin_online(blkcg_css); 646 css_get(memcg_css); 647 css_get(blkcg_css); 648 } 649 } 650 spin_unlock_irqrestore(&cgwb_lock, flags); 651 if (ret) { 652 if (ret == -EEXIST) 653 ret = 0; 654 goto err_fprop_exit; 655 } 656 goto out_put; 657 658 err_fprop_exit: 659 bdi_put(bdi); 660 fprop_local_destroy_percpu(&wb->memcg_completions); 661 err_ref_exit: 662 percpu_ref_exit(&wb->refcnt); 663 err_wb_exit: 664 wb_exit(wb); 665 err_free: 666 kfree(wb); 667 out_put: 668 css_put(blkcg_css); 669 return ret; 670 } 671 672 /** 673 * wb_get_lookup - get wb for a given memcg 674 * @bdi: target bdi 675 * @memcg_css: cgroup_subsys_state of the target memcg (must have positive ref) 676 * 677 * Try to get the wb for @memcg_css on @bdi. The returned wb has its 678 * refcount incremented. 679 * 680 * This function uses css_get() on @memcg_css and thus expects its refcnt 681 * to be positive on invocation. IOW, rcu_read_lock() protection on 682 * @memcg_css isn't enough. try_get it before calling this function. 683 * 684 * A wb is keyed by its associated memcg. As blkcg implicitly enables 685 * memcg on the default hierarchy, memcg association is guaranteed to be 686 * more specific (equal or descendant to the associated blkcg) and thus can 687 * identify both the memcg and blkcg associations. 688 * 689 * Because the blkcg associated with a memcg may change as blkcg is enabled 690 * and disabled closer to root in the hierarchy, each wb keeps track of 691 * both the memcg and blkcg associated with it and verifies the blkcg on 692 * each lookup. On mismatch, the existing wb is discarded and a new one is 693 * created. 694 */ 695 struct bdi_writeback *wb_get_lookup(struct backing_dev_info *bdi, 696 struct cgroup_subsys_state *memcg_css) 697 { 698 struct bdi_writeback *wb; 699 700 if (!memcg_css->parent) 701 return &bdi->wb; 702 703 rcu_read_lock(); 704 wb = radix_tree_lookup(&bdi->cgwb_tree, memcg_css->id); 705 if (wb) { 706 struct cgroup_subsys_state *blkcg_css; 707 708 /* see whether the blkcg association has changed */ 709 blkcg_css = cgroup_get_e_css(memcg_css->cgroup, &io_cgrp_subsys); 710 if (unlikely(wb->blkcg_css != blkcg_css || !wb_tryget(wb))) 711 wb = NULL; 712 css_put(blkcg_css); 713 } 714 rcu_read_unlock(); 715 716 return wb; 717 } 718 719 /** 720 * wb_get_create - get wb for a given memcg, create if necessary 721 * @bdi: target bdi 722 * @memcg_css: cgroup_subsys_state of the target memcg (must have positive ref) 723 * @gfp: allocation mask to use 724 * 725 * Try to get the wb for @memcg_css on @bdi. If it doesn't exist, try to 726 * create one. See wb_get_lookup() for more details. 727 */ 728 struct bdi_writeback *wb_get_create(struct backing_dev_info *bdi, 729 struct cgroup_subsys_state *memcg_css, 730 gfp_t gfp) 731 { 732 struct bdi_writeback *wb; 733 734 might_alloc(gfp); 735 736 if (!memcg_css->parent) 737 return &bdi->wb; 738 739 do { 740 wb = wb_get_lookup(bdi, memcg_css); 741 } while (!wb && !cgwb_create(bdi, memcg_css, gfp)); 742 743 return wb; 744 } 745 746 static int cgwb_bdi_init(struct backing_dev_info *bdi) 747 { 748 int ret; 749 750 INIT_RADIX_TREE(&bdi->cgwb_tree, GFP_ATOMIC); 751 mutex_init(&bdi->cgwb_release_mutex); 752 init_rwsem(&bdi->wb_switch_rwsem); 753 754 ret = wb_init(&bdi->wb, bdi, GFP_KERNEL); 755 if (!ret) { 756 bdi->wb.memcg_css = &root_mem_cgroup->css; 757 bdi->wb.blkcg_css = blkcg_root_css; 758 } 759 return ret; 760 } 761 762 static void cgwb_bdi_unregister(struct backing_dev_info *bdi) 763 { 764 struct radix_tree_iter iter; 765 void **slot; 766 struct bdi_writeback *wb; 767 768 WARN_ON(test_bit(WB_registered, &bdi->wb.state)); 769 770 spin_lock_irq(&cgwb_lock); 771 radix_tree_for_each_slot(slot, &bdi->cgwb_tree, &iter, 0) 772 cgwb_kill(*slot); 773 spin_unlock_irq(&cgwb_lock); 774 775 mutex_lock(&bdi->cgwb_release_mutex); 776 spin_lock_irq(&cgwb_lock); 777 while (!list_empty(&bdi->wb_list)) { 778 wb = list_first_entry(&bdi->wb_list, struct bdi_writeback, 779 bdi_node); 780 spin_unlock_irq(&cgwb_lock); 781 wb_shutdown(wb); 782 spin_lock_irq(&cgwb_lock); 783 } 784 spin_unlock_irq(&cgwb_lock); 785 mutex_unlock(&bdi->cgwb_release_mutex); 786 } 787 788 /* 789 * cleanup_offline_cgwbs_workfn - try to release dying cgwbs 790 * 791 * Try to release dying cgwbs by switching attached inodes to the nearest 792 * living ancestor's writeback. Processed wbs are placed at the end 793 * of the list to guarantee the forward progress. 794 */ 795 static void cleanup_offline_cgwbs_workfn(struct work_struct *work) 796 { 797 struct bdi_writeback *wb; 798 LIST_HEAD(processed); 799 800 spin_lock_irq(&cgwb_lock); 801 802 while (!list_empty(&offline_cgwbs)) { 803 wb = list_first_entry(&offline_cgwbs, struct bdi_writeback, 804 offline_node); 805 list_move(&wb->offline_node, &processed); 806 807 /* 808 * If wb is dirty, cleaning up the writeback by switching 809 * attached inodes will result in an effective removal of any 810 * bandwidth restrictions, which isn't the goal. Instead, 811 * it can be postponed until the next time, when all io 812 * will be likely completed. If in the meantime some inodes 813 * will get re-dirtied, they should be eventually switched to 814 * a new cgwb. 815 */ 816 if (wb_has_dirty_io(wb)) 817 continue; 818 819 if (!wb_tryget(wb)) 820 continue; 821 822 spin_unlock_irq(&cgwb_lock); 823 while (cleanup_offline_cgwb(wb)) 824 cond_resched(); 825 spin_lock_irq(&cgwb_lock); 826 827 wb_put(wb); 828 } 829 830 if (!list_empty(&processed)) 831 list_splice_tail(&processed, &offline_cgwbs); 832 833 spin_unlock_irq(&cgwb_lock); 834 } 835 836 /** 837 * wb_memcg_offline - kill all wb's associated with a memcg being offlined 838 * @memcg: memcg being offlined 839 * 840 * Also prevents creation of any new wb's associated with @memcg. 841 */ 842 void wb_memcg_offline(struct mem_cgroup *memcg) 843 { 844 struct list_head *memcg_cgwb_list = &memcg->cgwb_list; 845 struct bdi_writeback *wb, *next; 846 847 spin_lock_irq(&cgwb_lock); 848 list_for_each_entry_safe(wb, next, memcg_cgwb_list, memcg_node) 849 cgwb_kill(wb); 850 memcg_cgwb_list->next = NULL; /* prevent new wb's */ 851 spin_unlock_irq(&cgwb_lock); 852 853 queue_work(system_unbound_wq, &cleanup_offline_cgwbs_work); 854 } 855 856 /** 857 * wb_blkcg_offline - kill all wb's associated with a blkcg being offlined 858 * @css: blkcg being offlined 859 * 860 * Also prevents creation of any new wb's associated with @blkcg. 861 */ 862 void wb_blkcg_offline(struct cgroup_subsys_state *css) 863 { 864 struct bdi_writeback *wb, *next; 865 struct list_head *list = blkcg_get_cgwb_list(css); 866 867 spin_lock_irq(&cgwb_lock); 868 list_for_each_entry_safe(wb, next, list, blkcg_node) 869 cgwb_kill(wb); 870 list->next = NULL; /* prevent new wb's */ 871 spin_unlock_irq(&cgwb_lock); 872 } 873 874 static void cgwb_bdi_register(struct backing_dev_info *bdi) 875 { 876 spin_lock_irq(&cgwb_lock); 877 list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list); 878 spin_unlock_irq(&cgwb_lock); 879 } 880 881 static int __init cgwb_init(void) 882 { 883 /* 884 * There can be many concurrent release work items overwhelming 885 * system_wq. Put them in a separate wq and limit concurrency. 886 * There's no point in executing many of these in parallel. 887 */ 888 cgwb_release_wq = alloc_workqueue("cgwb_release", 0, 1); 889 if (!cgwb_release_wq) 890 return -ENOMEM; 891 892 return 0; 893 } 894 subsys_initcall(cgwb_init); 895 896 #else /* CONFIG_CGROUP_WRITEBACK */ 897 898 static int cgwb_bdi_init(struct backing_dev_info *bdi) 899 { 900 return wb_init(&bdi->wb, bdi, GFP_KERNEL); 901 } 902 903 static void cgwb_bdi_unregister(struct backing_dev_info *bdi) { } 904 905 static void cgwb_bdi_register(struct backing_dev_info *bdi) 906 { 907 list_add_tail_rcu(&bdi->wb.bdi_node, &bdi->wb_list); 908 } 909 910 static void cgwb_remove_from_bdi_list(struct bdi_writeback *wb) 911 { 912 list_del_rcu(&wb->bdi_node); 913 } 914 915 #endif /* CONFIG_CGROUP_WRITEBACK */ 916 917 int bdi_init(struct backing_dev_info *bdi) 918 { 919 bdi->dev = NULL; 920 921 kref_init(&bdi->refcnt); 922 bdi->min_ratio = 0; 923 bdi->max_ratio = 100 * BDI_RATIO_SCALE; 924 bdi->max_prop_frac = FPROP_FRAC_BASE; 925 INIT_LIST_HEAD(&bdi->bdi_list); 926 INIT_LIST_HEAD(&bdi->wb_list); 927 init_waitqueue_head(&bdi->wb_waitq); 928 929 return cgwb_bdi_init(bdi); 930 } 931 932 struct backing_dev_info *bdi_alloc(int node_id) 933 { 934 struct backing_dev_info *bdi; 935 936 bdi = kzalloc_node(sizeof(*bdi), GFP_KERNEL, node_id); 937 if (!bdi) 938 return NULL; 939 940 if (bdi_init(bdi)) { 941 kfree(bdi); 942 return NULL; 943 } 944 bdi->capabilities = BDI_CAP_WRITEBACK | BDI_CAP_WRITEBACK_ACCT; 945 bdi->ra_pages = VM_READAHEAD_PAGES; 946 bdi->io_pages = VM_READAHEAD_PAGES; 947 timer_setup(&bdi->laptop_mode_wb_timer, laptop_mode_timer_fn, 0); 948 return bdi; 949 } 950 EXPORT_SYMBOL(bdi_alloc); 951 952 static struct rb_node **bdi_lookup_rb_node(u64 id, struct rb_node **parentp) 953 { 954 struct rb_node **p = &bdi_tree.rb_node; 955 struct rb_node *parent = NULL; 956 struct backing_dev_info *bdi; 957 958 lockdep_assert_held(&bdi_lock); 959 960 while (*p) { 961 parent = *p; 962 bdi = rb_entry(parent, struct backing_dev_info, rb_node); 963 964 if (bdi->id > id) 965 p = &(*p)->rb_left; 966 else if (bdi->id < id) 967 p = &(*p)->rb_right; 968 else 969 break; 970 } 971 972 if (parentp) 973 *parentp = parent; 974 return p; 975 } 976 977 /** 978 * bdi_get_by_id - lookup and get bdi from its id 979 * @id: bdi id to lookup 980 * 981 * Find bdi matching @id and get it. Returns NULL if the matching bdi 982 * doesn't exist or is already unregistered. 983 */ 984 struct backing_dev_info *bdi_get_by_id(u64 id) 985 { 986 struct backing_dev_info *bdi = NULL; 987 struct rb_node **p; 988 989 spin_lock_bh(&bdi_lock); 990 p = bdi_lookup_rb_node(id, NULL); 991 if (*p) { 992 bdi = rb_entry(*p, struct backing_dev_info, rb_node); 993 bdi_get(bdi); 994 } 995 spin_unlock_bh(&bdi_lock); 996 997 return bdi; 998 } 999 1000 int bdi_register_va(struct backing_dev_info *bdi, const char *fmt, va_list args) 1001 { 1002 struct device *dev; 1003 struct rb_node *parent, **p; 1004 1005 if (bdi->dev) /* The driver needs to use separate queues per device */ 1006 return 0; 1007 1008 vsnprintf(bdi->dev_name, sizeof(bdi->dev_name), fmt, args); 1009 dev = device_create(&bdi_class, NULL, MKDEV(0, 0), bdi, bdi->dev_name); 1010 if (IS_ERR(dev)) 1011 return PTR_ERR(dev); 1012 1013 cgwb_bdi_register(bdi); 1014 bdi->dev = dev; 1015 1016 bdi_debug_register(bdi, dev_name(dev)); 1017 set_bit(WB_registered, &bdi->wb.state); 1018 1019 spin_lock_bh(&bdi_lock); 1020 1021 bdi->id = ++bdi_id_cursor; 1022 1023 p = bdi_lookup_rb_node(bdi->id, &parent); 1024 rb_link_node(&bdi->rb_node, parent, p); 1025 rb_insert_color(&bdi->rb_node, &bdi_tree); 1026 1027 list_add_tail_rcu(&bdi->bdi_list, &bdi_list); 1028 1029 spin_unlock_bh(&bdi_lock); 1030 1031 trace_writeback_bdi_register(bdi); 1032 return 0; 1033 } 1034 1035 int bdi_register(struct backing_dev_info *bdi, const char *fmt, ...) 1036 { 1037 va_list args; 1038 int ret; 1039 1040 va_start(args, fmt); 1041 ret = bdi_register_va(bdi, fmt, args); 1042 va_end(args); 1043 return ret; 1044 } 1045 EXPORT_SYMBOL(bdi_register); 1046 1047 void bdi_set_owner(struct backing_dev_info *bdi, struct device *owner) 1048 { 1049 WARN_ON_ONCE(bdi->owner); 1050 bdi->owner = owner; 1051 get_device(owner); 1052 } 1053 1054 /* 1055 * Remove bdi from bdi_list, and ensure that it is no longer visible 1056 */ 1057 static void bdi_remove_from_list(struct backing_dev_info *bdi) 1058 { 1059 spin_lock_bh(&bdi_lock); 1060 rb_erase(&bdi->rb_node, &bdi_tree); 1061 list_del_rcu(&bdi->bdi_list); 1062 spin_unlock_bh(&bdi_lock); 1063 1064 synchronize_rcu_expedited(); 1065 } 1066 1067 void bdi_unregister(struct backing_dev_info *bdi) 1068 { 1069 del_timer_sync(&bdi->laptop_mode_wb_timer); 1070 1071 /* make sure nobody finds us on the bdi_list anymore */ 1072 bdi_remove_from_list(bdi); 1073 wb_shutdown(&bdi->wb); 1074 cgwb_bdi_unregister(bdi); 1075 1076 /* 1077 * If this BDI's min ratio has been set, use bdi_set_min_ratio() to 1078 * update the global bdi_min_ratio. 1079 */ 1080 if (bdi->min_ratio) 1081 bdi_set_min_ratio(bdi, 0); 1082 1083 if (bdi->dev) { 1084 bdi_debug_unregister(bdi); 1085 device_unregister(bdi->dev); 1086 bdi->dev = NULL; 1087 } 1088 1089 if (bdi->owner) { 1090 put_device(bdi->owner); 1091 bdi->owner = NULL; 1092 } 1093 } 1094 EXPORT_SYMBOL(bdi_unregister); 1095 1096 static void release_bdi(struct kref *ref) 1097 { 1098 struct backing_dev_info *bdi = 1099 container_of(ref, struct backing_dev_info, refcnt); 1100 1101 WARN_ON_ONCE(test_bit(WB_registered, &bdi->wb.state)); 1102 WARN_ON_ONCE(bdi->dev); 1103 wb_exit(&bdi->wb); 1104 kfree(bdi); 1105 } 1106 1107 void bdi_put(struct backing_dev_info *bdi) 1108 { 1109 kref_put(&bdi->refcnt, release_bdi); 1110 } 1111 EXPORT_SYMBOL(bdi_put); 1112 1113 struct backing_dev_info *inode_to_bdi(struct inode *inode) 1114 { 1115 struct super_block *sb; 1116 1117 if (!inode) 1118 return &noop_backing_dev_info; 1119 1120 sb = inode->i_sb; 1121 #ifdef CONFIG_BLOCK 1122 if (sb_is_blkdev_sb(sb)) 1123 return I_BDEV(inode)->bd_disk->bdi; 1124 #endif 1125 return sb->s_bdi; 1126 } 1127 EXPORT_SYMBOL(inode_to_bdi); 1128 1129 const char *bdi_dev_name(struct backing_dev_info *bdi) 1130 { 1131 if (!bdi || !bdi->dev) 1132 return bdi_unknown_name; 1133 return bdi->dev_name; 1134 } 1135 EXPORT_SYMBOL_GPL(bdi_dev_name); 1136