page-writeback.c (664b0bae0b87f69bc9deb098f5e0158b9cf18e04) page-writeback.c (b93b016313b3ba8003c3b8bb71f569af91f19fc7)
1/*
2 * mm/page-writeback.c
3 *
4 * Copyright (C) 2002, Linus Torvalds.
5 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
6 *
7 * Contains functions related to writing back dirty pages at the
8 * address_space level.

--- 2085 unchanged lines hidden (view full) ---

2094 * all pages that have DIRTY tag set with a special TOWRITE tag. The idea is
2095 * that write_cache_pages (or whoever calls this function) will then use
2096 * TOWRITE tag to identify pages eligible for writeback. This mechanism is
2097 * used to avoid livelocking of writeback by a process steadily creating new
2098 * dirty pages in the file (thus it is important for this function to be quick
2099 * so that it can tag pages faster than a dirtying process can create them).
2100 */
2101/*
1/*
2 * mm/page-writeback.c
3 *
4 * Copyright (C) 2002, Linus Torvalds.
5 * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
6 *
7 * Contains functions related to writing back dirty pages at the
8 * address_space level.

--- 2085 unchanged lines hidden (view full) ---

2094 * all pages that have DIRTY tag set with a special TOWRITE tag. The idea is
2095 * that write_cache_pages (or whoever calls this function) will then use
2096 * TOWRITE tag to identify pages eligible for writeback. This mechanism is
2097 * used to avoid livelocking of writeback by a process steadily creating new
2098 * dirty pages in the file (thus it is important for this function to be quick
2099 * so that it can tag pages faster than a dirtying process can create them).
2100 */
2101/*
2102 * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce tree_lock latency.
2102 * We tag pages in batches of WRITEBACK_TAG_BATCH to reduce the i_pages lock
2103 * latency.
2103 */
2104void tag_pages_for_writeback(struct address_space *mapping,
2105 pgoff_t start, pgoff_t end)
2106{
2107#define WRITEBACK_TAG_BATCH 4096
2108 unsigned long tagged = 0;
2109 struct radix_tree_iter iter;
2110 void **slot;
2111
2104 */
2105void tag_pages_for_writeback(struct address_space *mapping,
2106 pgoff_t start, pgoff_t end)
2107{
2108#define WRITEBACK_TAG_BATCH 4096
2109 unsigned long tagged = 0;
2110 struct radix_tree_iter iter;
2111 void **slot;
2112
2112 spin_lock_irq(&mapping->tree_lock);
2113 radix_tree_for_each_tagged(slot, &mapping->page_tree, &iter, start,
2113 xa_lock_irq(&mapping->i_pages);
2114 radix_tree_for_each_tagged(slot, &mapping->i_pages, &iter, start,
2114 PAGECACHE_TAG_DIRTY) {
2115 if (iter.index > end)
2116 break;
2115 PAGECACHE_TAG_DIRTY) {
2116 if (iter.index > end)
2117 break;
2117 radix_tree_iter_tag_set(&mapping->page_tree, &iter,
2118 radix_tree_iter_tag_set(&mapping->i_pages, &iter,
2118 PAGECACHE_TAG_TOWRITE);
2119 tagged++;
2120 if ((tagged % WRITEBACK_TAG_BATCH) != 0)
2121 continue;
2122 slot = radix_tree_iter_resume(slot, &iter);
2119 PAGECACHE_TAG_TOWRITE);
2120 tagged++;
2121 if ((tagged % WRITEBACK_TAG_BATCH) != 0)
2122 continue;
2123 slot = radix_tree_iter_resume(slot, &iter);
2123 spin_unlock_irq(&mapping->tree_lock);
2124 xa_unlock_irq(&mapping->i_pages);
2124 cond_resched();
2125 cond_resched();
2125 spin_lock_irq(&mapping->tree_lock);
2126 xa_lock_irq(&mapping->i_pages);
2126 }
2127 }
2127 spin_unlock_irq(&mapping->tree_lock);
2128 xa_unlock_irq(&mapping->i_pages);
2128}
2129EXPORT_SYMBOL(tag_pages_for_writeback);
2130
2131/**
2132 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
2133 * @mapping: address space structure to write
2134 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
2135 * @writepage: function called for each page

--- 326 unchanged lines hidden (view full) ---

2462 struct address_space *mapping = page_mapping(page);
2463 unsigned long flags;
2464
2465 if (!mapping) {
2466 unlock_page_memcg(page);
2467 return 1;
2468 }
2469
2129}
2130EXPORT_SYMBOL(tag_pages_for_writeback);
2131
2132/**
2133 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
2134 * @mapping: address space structure to write
2135 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
2136 * @writepage: function called for each page

--- 326 unchanged lines hidden (view full) ---

2463 struct address_space *mapping = page_mapping(page);
2464 unsigned long flags;
2465
2466 if (!mapping) {
2467 unlock_page_memcg(page);
2468 return 1;
2469 }
2470
2470 spin_lock_irqsave(&mapping->tree_lock, flags);
2471 xa_lock_irqsave(&mapping->i_pages, flags);
2471 BUG_ON(page_mapping(page) != mapping);
2472 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
2473 account_page_dirtied(page, mapping);
2472 BUG_ON(page_mapping(page) != mapping);
2473 WARN_ON_ONCE(!PagePrivate(page) && !PageUptodate(page));
2474 account_page_dirtied(page, mapping);
2474 radix_tree_tag_set(&mapping->page_tree, page_index(page),
2475 radix_tree_tag_set(&mapping->i_pages, page_index(page),
2475 PAGECACHE_TAG_DIRTY);
2476 PAGECACHE_TAG_DIRTY);
2476 spin_unlock_irqrestore(&mapping->tree_lock, flags);
2477 xa_unlock_irqrestore(&mapping->i_pages, flags);
2477 unlock_page_memcg(page);
2478
2479 if (mapping->host) {
2480 /* !PageAnon && !swapper_space */
2481 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2482 }
2483 return 1;
2484 }

--- 228 unchanged lines hidden (view full) ---

2713
2714 memcg = lock_page_memcg(page);
2715 lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
2716 if (mapping && mapping_use_writeback_tags(mapping)) {
2717 struct inode *inode = mapping->host;
2718 struct backing_dev_info *bdi = inode_to_bdi(inode);
2719 unsigned long flags;
2720
2478 unlock_page_memcg(page);
2479
2480 if (mapping->host) {
2481 /* !PageAnon && !swapper_space */
2482 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2483 }
2484 return 1;
2485 }

--- 228 unchanged lines hidden (view full) ---

2714
2715 memcg = lock_page_memcg(page);
2716 lruvec = mem_cgroup_page_lruvec(page, page_pgdat(page));
2717 if (mapping && mapping_use_writeback_tags(mapping)) {
2718 struct inode *inode = mapping->host;
2719 struct backing_dev_info *bdi = inode_to_bdi(inode);
2720 unsigned long flags;
2721
2721 spin_lock_irqsave(&mapping->tree_lock, flags);
2722 xa_lock_irqsave(&mapping->i_pages, flags);
2722 ret = TestClearPageWriteback(page);
2723 if (ret) {
2723 ret = TestClearPageWriteback(page);
2724 if (ret) {
2724 radix_tree_tag_clear(&mapping->page_tree,
2725 page_index(page),
2725 radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2726 PAGECACHE_TAG_WRITEBACK);
2727 if (bdi_cap_account_writeback(bdi)) {
2728 struct bdi_writeback *wb = inode_to_wb(inode);
2729
2730 dec_wb_stat(wb, WB_WRITEBACK);
2731 __wb_writeout_inc(wb);
2732 }
2733 }
2734
2735 if (mapping->host && !mapping_tagged(mapping,
2736 PAGECACHE_TAG_WRITEBACK))
2737 sb_clear_inode_writeback(mapping->host);
2738
2726 PAGECACHE_TAG_WRITEBACK);
2727 if (bdi_cap_account_writeback(bdi)) {
2728 struct bdi_writeback *wb = inode_to_wb(inode);
2729
2730 dec_wb_stat(wb, WB_WRITEBACK);
2731 __wb_writeout_inc(wb);
2732 }
2733 }
2734
2735 if (mapping->host && !mapping_tagged(mapping,
2736 PAGECACHE_TAG_WRITEBACK))
2737 sb_clear_inode_writeback(mapping->host);
2738
2739 spin_unlock_irqrestore(&mapping->tree_lock, flags);
2739 xa_unlock_irqrestore(&mapping->i_pages, flags);
2740 } else {
2741 ret = TestClearPageWriteback(page);
2742 }
2743 /*
2744 * NOTE: Page might be free now! Writeback doesn't hold a page
2745 * reference on its own, it relies on truncation to wait for
2746 * the clearing of PG_writeback. The below can only access
2747 * page state that is static across allocation cycles.

--- 13 unchanged lines hidden (view full) ---

2761 int ret;
2762
2763 lock_page_memcg(page);
2764 if (mapping && mapping_use_writeback_tags(mapping)) {
2765 struct inode *inode = mapping->host;
2766 struct backing_dev_info *bdi = inode_to_bdi(inode);
2767 unsigned long flags;
2768
2740 } else {
2741 ret = TestClearPageWriteback(page);
2742 }
2743 /*
2744 * NOTE: Page might be free now! Writeback doesn't hold a page
2745 * reference on its own, it relies on truncation to wait for
2746 * the clearing of PG_writeback. The below can only access
2747 * page state that is static across allocation cycles.

--- 13 unchanged lines hidden (view full) ---

2761 int ret;
2762
2763 lock_page_memcg(page);
2764 if (mapping && mapping_use_writeback_tags(mapping)) {
2765 struct inode *inode = mapping->host;
2766 struct backing_dev_info *bdi = inode_to_bdi(inode);
2767 unsigned long flags;
2768
2769 spin_lock_irqsave(&mapping->tree_lock, flags);
2769 xa_lock_irqsave(&mapping->i_pages, flags);
2770 ret = TestSetPageWriteback(page);
2771 if (!ret) {
2772 bool on_wblist;
2773
2774 on_wblist = mapping_tagged(mapping,
2775 PAGECACHE_TAG_WRITEBACK);
2776
2770 ret = TestSetPageWriteback(page);
2771 if (!ret) {
2772 bool on_wblist;
2773
2774 on_wblist = mapping_tagged(mapping,
2775 PAGECACHE_TAG_WRITEBACK);
2776
2777 radix_tree_tag_set(&mapping->page_tree,
2778 page_index(page),
2777 radix_tree_tag_set(&mapping->i_pages, page_index(page),
2779 PAGECACHE_TAG_WRITEBACK);
2780 if (bdi_cap_account_writeback(bdi))
2781 inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK);
2782
2783 /*
2784 * We can come through here when swapping anonymous
2785 * pages, so we don't necessarily have an inode to track
2786 * for sync.
2787 */
2788 if (mapping->host && !on_wblist)
2789 sb_mark_inode_writeback(mapping->host);
2790 }
2791 if (!PageDirty(page))
2778 PAGECACHE_TAG_WRITEBACK);
2779 if (bdi_cap_account_writeback(bdi))
2780 inc_wb_stat(inode_to_wb(inode), WB_WRITEBACK);
2781
2782 /*
2783 * We can come through here when swapping anonymous
2784 * pages, so we don't necessarily have an inode to track
2785 * for sync.
2786 */
2787 if (mapping->host && !on_wblist)
2788 sb_mark_inode_writeback(mapping->host);
2789 }
2790 if (!PageDirty(page))
2792 radix_tree_tag_clear(&mapping->page_tree,
2793 page_index(page),
2791 radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2794 PAGECACHE_TAG_DIRTY);
2795 if (!keep_write)
2792 PAGECACHE_TAG_DIRTY);
2793 if (!keep_write)
2796 radix_tree_tag_clear(&mapping->page_tree,
2797 page_index(page),
2794 radix_tree_tag_clear(&mapping->i_pages, page_index(page),
2798 PAGECACHE_TAG_TOWRITE);
2795 PAGECACHE_TAG_TOWRITE);
2799 spin_unlock_irqrestore(&mapping->tree_lock, flags);
2796 xa_unlock_irqrestore(&mapping->i_pages, flags);
2800 } else {
2801 ret = TestSetPageWriteback(page);
2802 }
2803 if (!ret) {
2804 inc_lruvec_page_state(page, NR_WRITEBACK);
2805 inc_zone_page_state(page, NR_ZONE_WRITE_PENDING);
2806 }
2807 unlock_page_memcg(page);
2808 return ret;
2809
2810}
2811EXPORT_SYMBOL(__test_set_page_writeback);
2812
2813/*
2814 * Return true if any of the pages in the mapping are marked with the
2815 * passed tag.
2816 */
2817int mapping_tagged(struct address_space *mapping, int tag)
2818{
2797 } else {
2798 ret = TestSetPageWriteback(page);
2799 }
2800 if (!ret) {
2801 inc_lruvec_page_state(page, NR_WRITEBACK);
2802 inc_zone_page_state(page, NR_ZONE_WRITE_PENDING);
2803 }
2804 unlock_page_memcg(page);
2805 return ret;
2806
2807}
2808EXPORT_SYMBOL(__test_set_page_writeback);
2809
2810/*
2811 * Return true if any of the pages in the mapping are marked with the
2812 * passed tag.
2813 */
2814int mapping_tagged(struct address_space *mapping, int tag)
2815{
2819 return radix_tree_tagged(&mapping->page_tree, tag);
2816 return radix_tree_tagged(&mapping->i_pages, tag);
2820}
2821EXPORT_SYMBOL(mapping_tagged);
2822
2823/**
2824 * wait_for_stable_page() - wait for writeback to finish, if necessary.
2825 * @page: The page to wait on.
2826 *
2827 * This function determines if the given page is related to a backing device
2828 * that requires page contents to be held stable during writeback. If so, then
2829 * it will wait for any pending writeback to complete.
2830 */
2831void wait_for_stable_page(struct page *page)
2832{
2833 if (bdi_cap_stable_pages_required(inode_to_bdi(page->mapping->host)))
2834 wait_on_page_writeback(page);
2835}
2836EXPORT_SYMBOL_GPL(wait_for_stable_page);
2817}
2818EXPORT_SYMBOL(mapping_tagged);
2819
2820/**
2821 * wait_for_stable_page() - wait for writeback to finish, if necessary.
2822 * @page: The page to wait on.
2823 *
2824 * This function determines if the given page is related to a backing device
2825 * that requires page contents to be held stable during writeback. If so, then
2826 * it will wait for any pending writeback to complete.
2827 */
2828void wait_for_stable_page(struct page *page)
2829{
2830 if (bdi_cap_stable_pages_required(inode_to_bdi(page->mapping->host)))
2831 wait_on_page_writeback(page);
2832}
2833EXPORT_SYMBOL_GPL(wait_for_stable_page);