83 void buffer_check_dirty_writeback(struct folio *folio,  in buffer_check_dirty_writeback()  argument
90 	BUG_ON(!folio_test_locked(folio));  in buffer_check_dirty_writeback()
92 	head = folio_buffers(folio);  in buffer_check_dirty_writeback()
96 	if (folio_test_writeback(folio))  in buffer_check_dirty_writeback()
225 	struct folio *folio;  in __find_get_block_slow()  local
230 	folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0);  in __find_get_block_slow()
231 	if (IS_ERR(folio))  in __find_get_block_slow()
242 		folio_lock(folio);  in __find_get_block_slow()
244 	head = folio_buffers(folio);  in __find_get_block_slow()
287 		folio_unlock(folio);  in __find_get_block_slow()
288 	folio_put(folio);  in __find_get_block_slow()
298 	struct folio *folio;  in end_buffer_async_read()  local
303 	folio = bh->b_folio;  in end_buffer_async_read()
316 	first = folio_buffers(folio);  in end_buffer_async_read()
332 	folio_end_read(folio, folio_uptodate);  in end_buffer_async_read()
405 	struct folio *folio;  in bh_end_async_write()  local
409 	folio = bh->b_folio;  in bh_end_async_write()
418 	first = folio_buffers(folio);  in bh_end_async_write()
432 	folio_end_writeback(folio);  in bh_end_async_write()
673 bool block_dirty_folio(struct address_space *mapping, struct folio *folio)  in block_dirty_folio()  argument
679 	head = folio_buffers(folio);  in block_dirty_folio()
692 	newly_dirty = !folio_test_set_dirty(folio);  in block_dirty_folio()
696 		__folio_mark_dirty(folio, mapping, 1);  in block_dirty_folio()
730 struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size,  in folio_alloc_buffers()  argument
737 	memcg = get_mem_cgroup_from_folio(folio);  in folio_alloc_buffers()
741 	offset = folio_size(folio);  in folio_alloc_buffers()
754 		folio_set_bh(bh, folio, offset);  in folio_alloc_buffers()
784 static inline void link_dev_buffers(struct folio *folio,  in link_dev_buffers()  argument
795 	folio_attach_private(folio, head);  in link_dev_buffers()
813 static sector_t folio_init_buffers(struct folio *folio,  in folio_init_buffers()  argument
816 	struct buffer_head *head = folio_buffers(folio);  in folio_init_buffers()
818 	bool uptodate = folio_test_uptodate(folio);  in folio_init_buffers()
819 	sector_t block = div_u64(folio_pos(folio), size);  in folio_init_buffers()
854 	struct folio *folio;  in grow_dev_folio()  local
858 	folio = __filemap_get_folio(mapping, index,  in grow_dev_folio()
860 	if (IS_ERR(folio))  in grow_dev_folio()
863 	bh = folio_buffers(folio);  in grow_dev_folio()
866 			end_block = folio_init_buffers(folio, bdev, size);  in grow_dev_folio()
877 		if (!try_to_free_buffers(folio)) {  in grow_dev_folio()
883 	bh = folio_alloc_buffers(folio, size, gfp | __GFP_ACCOUNT);  in grow_dev_folio()
893 	link_dev_buffers(folio, bh);  in grow_dev_folio()
894 	end_block = folio_init_buffers(folio, bdev, size);  in grow_dev_folio()
897 	folio_unlock(folio);  in grow_dev_folio()
898 	folio_put(folio);  in grow_dev_folio()
1008 		struct folio *folio = bh->b_folio;  in mark_buffer_dirty()  local
1011 		if (!folio_test_set_dirty(folio)) {  in mark_buffer_dirty()
1012 			mapping = folio->mapping;  in mark_buffer_dirty()
1014 				__folio_mark_dirty(folio, mapping, 0);  in mark_buffer_dirty()
1463 void folio_set_bh(struct buffer_head *bh, struct folio *folio,  in folio_set_bh()  argument
1466 	bh->b_folio = folio;  in folio_set_bh()
1467 	BUG_ON(offset >= folio_size(folio));  in folio_set_bh()
1468 	if (folio_test_highmem(folio))  in folio_set_bh()
1474 		bh->b_data = folio_address(folio) + offset;  in folio_set_bh()
1516 void block_invalidate_folio(struct folio *folio, size_t offset, size_t length)  in block_invalidate_folio()  argument
1522 	BUG_ON(!folio_test_locked(folio));  in block_invalidate_folio()
1527 	BUG_ON(stop > folio_size(folio) || stop < length);  in block_invalidate_folio()
1529 	head = folio_buffers(folio);  in block_invalidate_folio()
1558 	if (length == folio_size(folio))  in block_invalidate_folio()
1559 		filemap_release_folio(folio, 0);  in block_invalidate_folio()
1561 	folio_clear_mappedtodisk(folio);  in block_invalidate_folio()
1570 struct buffer_head *create_empty_buffers(struct folio *folio,  in create_empty_buffers()  argument
1576 	head = folio_alloc_buffers(folio, blocksize, gfp);  in create_empty_buffers()
1585 	spin_lock(&folio->mapping->i_private_lock);  in create_empty_buffers()
1586 	if (folio_test_uptodate(folio) || folio_test_dirty(folio)) {  in create_empty_buffers()
1589 			if (folio_test_dirty(folio))  in create_empty_buffers()
1591 			if (folio_test_uptodate(folio))  in create_empty_buffers()
1596 	folio_attach_private(folio, head);  in create_empty_buffers()
1597 	spin_unlock(&folio->mapping->i_private_lock);  in create_empty_buffers()
1639 			struct folio *folio = fbatch.folios[i];  in clean_bdev_aliases()  local
1641 			if (!folio_buffers(folio))  in clean_bdev_aliases()
1648 			folio_lock(folio);  in clean_bdev_aliases()
1650 			head = folio_buffers(folio);  in clean_bdev_aliases()
1666 			folio_unlock(folio);  in clean_bdev_aliases()
1677 static struct buffer_head *folio_create_buffers(struct folio *folio,  in folio_create_buffers()  argument
1683 	BUG_ON(!folio_test_locked(folio));  in folio_create_buffers()
1685 	bh = folio_buffers(folio);  in folio_create_buffers()
1687 		bh = create_empty_buffers(folio,  in folio_create_buffers()
1721 int __block_write_full_folio(struct inode *inode, struct folio *folio,  in __block_write_full_folio()  argument
1732 	head = folio_create_buffers(folio, inode,  in __block_write_full_folio()
1748 	block = div_u64(folio_pos(folio), blocksize);  in __block_write_full_folio()
1797 			folio_redirty_for_writepage(wbc, folio);  in __block_write_full_folio()
1811 	BUG_ON(folio_test_writeback(folio));  in __block_write_full_folio()
1812 	folio_start_writeback(folio);  in __block_write_full_folio()
1824 	folio_unlock(folio);  in __block_write_full_folio()
1834 		folio_end_writeback(folio);  in __block_write_full_folio()
1865 	BUG_ON(folio_test_writeback(folio));  in __block_write_full_folio()
1866 	mapping_set_error(folio->mapping, err);  in __block_write_full_folio()
1867 	folio_start_writeback(folio);  in __block_write_full_folio()
1879 	folio_unlock(folio);  in __block_write_full_folio()
1889 void folio_zero_new_buffers(struct folio *folio, size_t from, size_t to)  in folio_zero_new_buffers()  argument
1894 	BUG_ON(!folio_test_locked(folio));  in folio_zero_new_buffers()
1895 	head = folio_buffers(folio);  in folio_zero_new_buffers()
1906 				if (!folio_test_uptodate(folio)) {  in folio_zero_new_buffers()
1912 					folio_zero_segment(folio, start, xend);  in folio_zero_new_buffers()
1995 int __block_write_begin_int(struct folio *folio, loff_t pos, unsigned len,  in __block_write_begin_int()  argument
1998 	size_t from = offset_in_folio(folio, pos);  in __block_write_begin_int()
2000 	struct inode *inode = folio->mapping->host;  in __block_write_begin_int()
2007 	BUG_ON(!folio_test_locked(folio));  in __block_write_begin_int()
2008 	BUG_ON(to > folio_size(folio));  in __block_write_begin_int()
2011 	head = folio_create_buffers(folio, inode, 0);  in __block_write_begin_int()
2013 	block = div_u64(folio_pos(folio), blocksize);  in __block_write_begin_int()
2019 			if (folio_test_uptodate(folio)) {  in __block_write_begin_int()
2038 				if (folio_test_uptodate(folio)) {  in __block_write_begin_int()
2045 					folio_zero_segments(folio,  in __block_write_begin_int()
2051 		if (folio_test_uptodate(folio)) {  in __block_write_begin_int()
2072 		folio_zero_new_buffers(folio, from, to);  in __block_write_begin_int()
2076 int __block_write_begin(struct folio *folio, loff_t pos, unsigned len,  in __block_write_begin()  argument
2079 	return __block_write_begin_int(folio, pos, len, get_block, NULL);  in __block_write_begin()
2083 void block_commit_write(struct folio *folio, size_t from, size_t to)  in block_commit_write()  argument
2087 	bool uptodate = folio_test_uptodate(folio);  in block_commit_write()
2091 	bh = head = folio_buffers(folio);  in block_commit_write()
2122 		folio_mark_uptodate(folio);  in block_commit_write()
2133 		struct folio **foliop, get_block_t *get_block)  in block_write_begin()
2136 	struct folio *folio;  in block_write_begin()  local
2139 	folio = __filemap_get_folio(mapping, index, FGP_WRITEBEGIN,  in block_write_begin()
2141 	if (IS_ERR(folio))  in block_write_begin()
2142 		return PTR_ERR(folio);  in block_write_begin()
2144 	status = __block_write_begin_int(folio, pos, len, get_block, NULL);  in block_write_begin()
2146 		folio_unlock(folio);  in block_write_begin()
2147 		folio_put(folio);  in block_write_begin()
2148 		folio = NULL;  in block_write_begin()
2151 	*foliop = folio;  in block_write_begin()
2157 		struct folio *folio)  in block_write_end()  argument
2159 	size_t start = pos - folio_pos(folio);  in block_write_end()
2174 		if (!folio_test_uptodate(folio))  in block_write_end()
2177 		folio_zero_new_buffers(folio, start+copied, start+len);  in block_write_end()
2179 	flush_dcache_folio(folio);  in block_write_end()
2182 	block_commit_write(folio, start, start + copied);  in block_write_end()
2190 		      struct folio *folio, void *fsdata)  in generic_write_end()  argument
2196 	copied = block_write_end(pos, len, copied, folio);  in generic_write_end()
2210 	folio_unlock(folio);  in generic_write_end()
2211 	folio_put(folio);  in generic_write_end()
2234 bool block_is_partially_uptodate(struct folio *folio, size_t from, size_t count)  in block_is_partially_uptodate()  argument
2241 	head = folio_buffers(folio);  in block_is_partially_uptodate()
2245 	to = min(folio_size(folio) - from, count);  in block_is_partially_uptodate()
2247 	if (from < blocksize && to > folio_size(folio) - blocksize)  in block_is_partially_uptodate()
2277 int block_read_full_folio(struct folio *folio, get_block_t *get_block)  in block_read_full_folio()  argument
2279 	struct inode *inode = folio->mapping->host;  in block_read_full_folio()
2291 	head = folio_create_buffers(folio, inode, 0);  in block_read_full_folio()
2294 	iblock = div_u64(folio_pos(folio), blocksize);  in block_read_full_folio()
2313 				folio_zero_range(folio, bh_offset(bh),  in block_read_full_folio()
2364 		folio_set_mappedtodisk(folio);  in block_read_full_folio()
2375 		folio_end_read(folio, !page_error);  in block_read_full_folio()
2389 	struct folio *folio;  in generic_cont_expand_simple()  local
2397 	err = aops->write_begin(NULL, mapping, size, 0, &folio, &fsdata);  in generic_cont_expand_simple()
2401 	err = aops->write_end(NULL, mapping, size, 0, 0, folio, fsdata);  in generic_cont_expand_simple()
2416 	struct folio *folio;  in cont_expand_zero()  local
2435 					    &folio, &fsdata);  in cont_expand_zero()
2438 		folio_zero_range(folio, offset_in_folio(folio, curpos), len);  in cont_expand_zero()
2440 						folio, fsdata);  in cont_expand_zero()
2468 					    &folio, &fsdata);  in cont_expand_zero()
2471 		folio_zero_range(folio, offset_in_folio(folio, curpos), len);  in cont_expand_zero()
2473 						folio, fsdata);  in cont_expand_zero()
2488 		     loff_t pos, unsigned len, struct folio **foliop,  in cont_write_begin()
2531 	struct folio *folio = page_folio(vmf->page);  in block_page_mkwrite()  local
2537 	folio_lock(folio);  in block_page_mkwrite()
2539 	if ((folio->mapping != inode->i_mapping) ||  in block_page_mkwrite()
2540 	    (folio_pos(folio) >= size)) {  in block_page_mkwrite()
2546 	end = folio_size(folio);  in block_page_mkwrite()
2548 	if (folio_pos(folio) + end > size)  in block_page_mkwrite()
2549 		end = size - folio_pos(folio);  in block_page_mkwrite()
2551 	ret = __block_write_begin_int(folio, 0, end, get_block, NULL);  in block_page_mkwrite()
2555 	block_commit_write(folio, 0, end);  in block_page_mkwrite()
2557 	folio_mark_dirty(folio);  in block_page_mkwrite()
2558 	folio_wait_stable(folio);  in block_page_mkwrite()
2561 	folio_unlock(folio);  in block_page_mkwrite()
2574 	struct folio *folio;  in block_truncate_page()  local
2588 	folio = filemap_grab_folio(mapping, index);  in block_truncate_page()
2589 	if (IS_ERR(folio))  in block_truncate_page()
2590 		return PTR_ERR(folio);  in block_truncate_page()
2592 	bh = folio_buffers(folio);  in block_truncate_page()
2594 		bh = create_empty_buffers(folio, blocksize, 0);  in block_truncate_page()
2597 	offset = offset_in_folio(folio, from);  in block_truncate_page()
2616 	if (folio_test_uptodate(folio))  in block_truncate_page()
2626 	folio_zero_range(folio, offset, length);  in block_truncate_page()
2630 	folio_unlock(folio);  in block_truncate_page()
2631 	folio_put(folio);  in block_truncate_page()
2640 int block_write_full_folio(struct folio *folio, struct writeback_control *wbc,  in block_write_full_folio()  argument
2643 	struct inode * const inode = folio->mapping->host;  in block_write_full_folio()
2647 	if (folio_next_pos(folio) <= i_size)  in block_write_full_folio()
2648 		return __block_write_full_folio(inode, folio, get_block, wbc);  in block_write_full_folio()
2651 	if (folio_pos(folio) >= i_size) {  in block_write_full_folio()
2652 		folio_unlock(folio);  in block_write_full_folio()
2663 	folio_zero_segment(folio, offset_in_folio(folio, i_size),  in block_write_full_folio()
2664 			folio_size(folio));  in block_write_full_folio()
2665 	return __block_write_full_folio(inode, folio, get_block, wbc);  in block_write_full_folio()
2735 drop_buffers(struct folio *folio, struct buffer_head **buffers_to_free)  in drop_buffers()  argument
2737 	struct buffer_head *head = folio_buffers(folio);  in drop_buffers()
2754 	folio_detach_private(folio);  in drop_buffers()
2784 bool try_to_free_buffers(struct folio *folio)  in try_to_free_buffers()  argument
2786 	struct address_space * const mapping = folio->mapping;  in try_to_free_buffers()
2790 	BUG_ON(!folio_test_locked(folio));  in try_to_free_buffers()
2791 	if (folio_test_writeback(folio))  in try_to_free_buffers()
2795 	if (WARN_ON_ONCE(!folio_buffers(folio)))  in try_to_free_buffers()
2799 		ret = drop_buffers(folio, &buffers_to_free);  in try_to_free_buffers()
2804 	ret = drop_buffers(folio, &buffers_to_free);  in try_to_free_buffers()
2821 		folio_cancel_dirty(folio);  in try_to_free_buffers()