Lines Matching +full:data +full:- +full:mapping
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 1994-1999 Linus Torvalds
30 #include <linux/error-injection.h>
33 #include <linux/backing-dev.h>
72 * finished 'unifying' the page and buffer cache and SMP-threaded the
73 * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
75 * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
81 * ->i_mmap_rwsem (truncate_pagecache)
82 * ->private_lock (__free_pte->block_dirty_folio)
83 * ->swap_lock (exclusive_swap_page, others)
84 * ->i_pages lock
86 * ->i_rwsem
87 * ->invalidate_lock (acquired by fs in truncate path)
88 * ->i_mmap_rwsem (truncate->unmap_mapping_range)
90 * ->mmap_lock
91 * ->i_mmap_rwsem
92 * ->page_table_lock or pte_lock (various, mainly in memory.c)
93 * ->i_pages lock (arch-dependent flush_dcache_mmap_lock)
95 * ->mmap_lock
96 * ->invalidate_lock (filemap_fault)
97 * ->lock_page (filemap_fault, access_process_vm)
99 * ->i_rwsem (generic_perform_write)
100 * ->mmap_lock (fault_in_readable->do_page_fault)
102 * bdi->wb.list_lock
103 * sb_lock (fs/fs-writeback.c)
104 * ->i_pages lock (__sync_single_inode)
106 * ->i_mmap_rwsem
107 * ->anon_vma.lock (vma_merge)
109 * ->anon_vma.lock
110 * ->page_table_lock or pte_lock (anon_vma_prepare and various)
112 * ->page_table_lock or pte_lock
113 * ->swap_lock (try_to_unmap_one)
114 * ->private_lock (try_to_unmap_one)
115 * ->i_pages lock (try_to_unmap_one)
116 * ->lruvec->lru_lock (follow_page_mask->mark_page_accessed)
117 * ->lruvec->lru_lock (check_pte_range->folio_isolate_lru)
118 * ->private_lock (folio_remove_rmap_pte->set_page_dirty)
119 * ->i_pages lock (folio_remove_rmap_pte->set_page_dirty)
120 * bdi.wb->list_lock (folio_remove_rmap_pte->set_page_dirty)
121 * ->inode->i_lock (folio_remove_rmap_pte->set_page_dirty)
122 * ->memcg->move_lock (folio_remove_rmap_pte->folio_memcg_lock)
123 * bdi.wb->list_lock (zap_pte_range->set_page_dirty)
124 * ->inode->i_lock (zap_pte_range->set_page_dirty)
125 * ->private_lock (zap_pte_range->block_dirty_folio)
129 struct address_space *mapping)
131 if (dax_mapping(mapping) || shmem_mapping(mapping))
137 static void page_cache_delete(struct address_space *mapping,
140 XA_STATE(xas, &mapping->i_pages, folio->index);
143 mapping_set_update(&xas, mapping);
145 xas_set_order(&xas, folio->index, folio_order(folio));
153 folio->mapping = NULL;
154 /* Leave page->index set: truncation lookup relies upon it */
155 mapping->nrpages -= nr;
158 static void filemap_unaccount_folio(struct address_space *mapping,
166 current->comm, folio_pfn(folio));
167 dump_page(&folio->page, "still mapped when deleted");
171 if (mapping_exiting(mapping) && !folio_test_large(folio)) {
181 atomic_set(&folio->_mapcount, -1);
193 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
195 __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
197 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr);
199 __lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr);
200 filemap_nr_thps_dec(mapping);
206 * unwritten data - on ordinary filesystems.
208 * But it's harmless on in-memory filesystems like tmpfs; and can
218 mapping_can_writeback(mapping)))
219 folio_account_cleaned(folio, inode_to_wb(mapping->host));
224 * sure the page is locked and that nobody else uses it - or that usage
229 struct address_space *mapping = folio->mapping;
232 filemap_unaccount_folio(mapping, folio);
233 page_cache_delete(mapping, folio, shadow);
236 void filemap_free_folio(struct address_space *mapping, struct folio *folio)
241 free_folio = mapping->a_ops->free_folio;
251 * filemap_remove_folio - Remove folio from page cache.
260 struct address_space *mapping = folio->mapping;
263 spin_lock(&mapping->host->i_lock);
264 xa_lock_irq(&mapping->i_pages);
266 xa_unlock_irq(&mapping->i_pages);
267 if (mapping_shrinkable(mapping))
268 inode_add_lru(mapping->host);
269 spin_unlock(&mapping->host->i_lock);
271 filemap_free_folio(mapping, folio);
275 * page_cache_delete_batch - delete several folios from page cache
276 * @mapping: the mapping to which folios belong
279 * The function walks over mapping->i_pages and removes folios passed in
280 * @fbatch from the mapping. The function expects @fbatch to be sorted
282 * It tolerates holes in @fbatch (mapping entries at those indices are not
287 static void page_cache_delete_batch(struct address_space *mapping,
290 XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index);
295 mapping_set_update(&xas, mapping);
310 if (folio != fbatch->folios[i]) {
311 VM_BUG_ON_FOLIO(folio->index >
312 fbatch->folios[i]->index, folio);
318 folio->mapping = NULL;
319 /* Leave folio->index set: truncation lookup relies on it */
325 mapping->nrpages -= total_pages;
328 void delete_from_page_cache_batch(struct address_space *mapping,
336 spin_lock(&mapping->host->i_lock);
337 xa_lock_irq(&mapping->i_pages);
339 struct folio *folio = fbatch->folios[i];
342 filemap_unaccount_folio(mapping, folio);
344 page_cache_delete_batch(mapping, fbatch);
345 xa_unlock_irq(&mapping->i_pages);
346 if (mapping_shrinkable(mapping))
347 inode_add_lru(mapping->host);
348 spin_unlock(&mapping->host->i_lock);
351 filemap_free_folio(mapping, fbatch->folios[i]);
354 int filemap_check_errors(struct address_space *mapping)
358 if (test_bit(AS_ENOSPC, &mapping->flags) &&
359 test_and_clear_bit(AS_ENOSPC, &mapping->flags))
360 ret = -ENOSPC;
361 if (test_bit(AS_EIO, &mapping->flags) &&
362 test_and_clear_bit(AS_EIO, &mapping->flags))
363 ret = -EIO;
368 static int filemap_check_and_keep_errors(struct address_space *mapping)
371 if (test_bit(AS_EIO, &mapping->flags))
372 return -EIO;
373 if (test_bit(AS_ENOSPC, &mapping->flags))
374 return -ENOSPC;
379 * filemap_fdatawrite_wbc - start writeback on mapping dirty pages in range
380 * @mapping: address space structure to write
383 * Call writepages on the mapping using the provided wbc to control the
388 int filemap_fdatawrite_wbc(struct address_space *mapping,
393 if (!mapping_can_writeback(mapping) ||
394 !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
397 wbc_attach_fdatawrite_inode(wbc, mapping->host);
398 ret = do_writepages(mapping, wbc);
405 * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
406 * @mapping: address space structure to write
411 * Start writeback against all of a mapping's dirty pages that lie
414 * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
421 int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
431 return filemap_fdatawrite_wbc(mapping, &wbc);
434 static inline int __filemap_fdatawrite(struct address_space *mapping,
437 return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
440 int filemap_fdatawrite(struct address_space *mapping)
442 return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
446 int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
449 return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
454 * filemap_flush - mostly a non-blocking flush
455 * @mapping: target address_space
457 * This is a mostly non-blocking flush. Not suitable for data-integrity
458 * purposes - I/O may not be started against all dirty pages.
462 int filemap_flush(struct address_space *mapping)
464 return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
469 * filemap_range_has_page - check if a page exists in range.
470 * @mapping: address space within which to check
480 bool filemap_range_has_page(struct address_space *mapping,
484 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
511 static void __filemap_fdatawait_range(struct address_space *mapping,
524 nr_folios = filemap_get_folios_tag(mapping, &index, end,
541 * filemap_fdatawait_range - wait for writeback to complete
542 * @mapping: address space structure to wait for
546 * Walk the list of under-writeback pages of the given address space
556 int filemap_fdatawait_range(struct address_space *mapping, loff_t start_byte,
559 __filemap_fdatawait_range(mapping, start_byte, end_byte);
560 return filemap_check_errors(mapping);
565 * filemap_fdatawait_range_keep_errors - wait for writeback to complete
566 * @mapping: address space structure to wait for
570 * Walk the list of under-writeback pages of the given address space in the
575 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
578 int filemap_fdatawait_range_keep_errors(struct address_space *mapping,
581 __filemap_fdatawait_range(mapping, start_byte, end_byte);
582 return filemap_check_and_keep_errors(mapping);
587 * file_fdatawait_range - wait for writeback to complete
592 * Walk the list of under-writeback pages of the address space that file
594 * status of the address space vs. the file->f_wb_err cursor and return it.
600 * Return: error status of the address space vs. the file->f_wb_err cursor.
604 struct address_space *mapping = file->f_mapping;
606 __filemap_fdatawait_range(mapping, start_byte, end_byte);
612 * filemap_fdatawait_keep_errors - wait for writeback without clearing errors
613 * @mapping: address space structure to wait for
615 * Walk the list of under-writeback pages of the given address space
620 * call sites are system-wide / filesystem-wide data flushers: e.g. sync(2),
625 int filemap_fdatawait_keep_errors(struct address_space *mapping)
627 __filemap_fdatawait_range(mapping, 0, LLONG_MAX);
628 return filemap_check_and_keep_errors(mapping);
633 static bool mapping_needs_writeback(struct address_space *mapping)
635 return mapping->nrpages;
638 bool filemap_range_has_writeback(struct address_space *mapping,
641 XA_STATE(xas, &mapping->i_pages, start_byte >> PAGE_SHIFT);
664 * filemap_write_and_wait_range - write out & wait on a file range
665 * @mapping: the address_space for the pages
669 * Write out and wait upon file offsets lstart->lend, inclusive.
672 * that this function can be used to write to the very end-of-file (end = -1).
676 int filemap_write_and_wait_range(struct address_space *mapping,
684 if (mapping_needs_writeback(mapping)) {
685 err = __filemap_fdatawrite_range(mapping, lstart, lend,
689 * written partially (e.g. -ENOSPC), so we wait for it.
690 * But the -EIO is special case, it may indicate the worst
693 if (err != -EIO)
694 __filemap_fdatawait_range(mapping, lstart, lend);
696 err2 = filemap_check_errors(mapping);
703 void __filemap_set_wb_err(struct address_space *mapping, int err)
705 errseq_t eseq = errseq_set(&mapping->wb_err, err);
707 trace_filemap_set_wb_err(mapping, eseq);
712 * file_check_and_advance_wb_err - report wb error (if any) that was previously
720 * Grab the wb_err from the mapping. If it matches what we have in the file,
723 * If it doesn't match, then take the mapping value, set the "seen" flag in
729 * While we handle mapping->wb_err with atomic operations, the f_wb_err
738 errseq_t old = READ_ONCE(file->f_wb_err);
739 struct address_space *mapping = file->f_mapping;
742 if (errseq_check(&mapping->wb_err, old)) {
744 spin_lock(&file->f_lock);
745 old = file->f_wb_err;
746 err = errseq_check_and_advance(&mapping->wb_err,
747 &file->f_wb_err);
749 spin_unlock(&file->f_lock);
757 clear_bit(AS_EIO, &mapping->flags);
758 clear_bit(AS_ENOSPC, &mapping->flags);
764 * file_write_and_wait_range - write out & wait on a file range
769 * Write out and wait upon file offsets lstart->lend, inclusive.
772 * that this function can be used to write to the very end-of-file (end = -1).
774 * After writing out and waiting on the data, we check and advance the
782 struct address_space *mapping = file->f_mapping;
787 if (mapping_needs_writeback(mapping)) {
788 err = __filemap_fdatawrite_range(mapping, lstart, lend,
791 if (err != -EIO)
792 __filemap_fdatawait_range(mapping, lstart, lend);
802 * replace_page_cache_folio - replace a pagecache folio with a new one
816 struct address_space *mapping = old->mapping;
817 void (*free_folio)(struct folio *) = mapping->a_ops->free_folio;
818 pgoff_t offset = old->index;
819 XA_STATE(xas, &mapping->i_pages, offset);
823 VM_BUG_ON_FOLIO(new->mapping, new);
826 new->mapping = mapping;
827 new->index = offset;
834 old->mapping = NULL;
851 noinline int __filemap_add_folio(struct address_space *mapping,
854 XA_STATE(xas, &mapping->i_pages, index);
862 VM_BUG_ON_FOLIO(folio_order(folio) < mapping_min_folio_order(mapping),
864 mapping_set_update(&xas, mapping);
866 VM_BUG_ON_FOLIO(index & (folio_nr_pages(folio) - 1), folio);
873 folio->mapping = mapping;
874 folio->index = xas.xa_index;
877 int order = -1, split_order = 0;
884 xas_set_err(&xas, -EEXIST);
891 if (order == -1)
895 /* entry may have changed before we re-acquire the lock */
904 BUG_ON(shmem_mapping(mapping));
920 mapping->nrpages += nr;
954 folio->mapping = NULL;
955 /* Leave page->index set: truncation relies upon it */
961 int filemap_add_folio(struct address_space *mapping, struct folio *folio,
972 ret = __filemap_add_folio(mapping, folio, index, gfp, &shadow);
982 * data from the working set, only to cache data that will
1016 * filemap_invalidate_lock_two - lock invalidate_lock for two mappings
1018 * Lock exclusively invalidate_lock of any passed mapping that is not NULL.
1020 * @mapping1: the first mapping to lock
1021 * @mapping2: the second mapping to lock
1029 down_write(&mapping1->invalidate_lock);
1031 down_write_nested(&mapping2->invalidate_lock, 1);
1036 * filemap_invalidate_unlock_two - unlock invalidate_lock for two mappings
1038 * Unlock exclusive invalidate_lock of any passed mapping that is not NULL.
1040 * @mapping1: the first mapping to unlock
1041 * @mapping2: the second mapping to unlock
1047 up_write(&mapping1->invalidate_lock);
1049 up_write(&mapping2->invalidate_lock);
1083 * The page wait code treats the "wait->flags" somewhat unusually, because
1130 flags = wait->flags;
1132 if (test_bit(key->bit_nr, &key->folio->flags))
1133 return -1;
1135 if (test_and_set_bit(key->bit_nr, &key->folio->flags))
1136 return -1;
1142 * We are holding the wait-queue lock, but the waiter that
1147 * afterwards to avoid any races. This store-release pairs
1148 * with the load-acquire in folio_wait_bit_common().
1150 smp_store_release(&wait->flags, flags | WQ_FLAG_WOKEN);
1151 wake_up_state(wait->private, mode);
1159 * After this list_del_init(&wait->entry) the wait entry
1160 * might be de-allocated and the process might even have
1163 list_del_init_careful(&wait->entry);
1177 spin_lock_irqsave(&q->lock, flags);
1192 spin_unlock_irqrestore(&q->lock, flags);
1217 if (wait->flags & WQ_FLAG_EXCLUSIVE) {
1218 if (test_and_set_bit(bit_nr, &folio->flags))
1220 } else if (test_bit(bit_nr, &folio->flags))
1223 wait->flags |= WQ_FLAG_WOKEN | WQ_FLAG_DONE;
1249 wait->func = wake_page_function;
1254 wait->flags = 0;
1256 wait->flags = WQ_FLAG_EXCLUSIVE;
1257 if (--unfairness < 0)
1258 wait->flags |= WQ_FLAG_CUSTOM;
1275 spin_lock_irq(&q->lock);
1279 spin_unlock_irq(&q->lock);
1295 * be very careful with the 'wait->flags', because
1304 flags = smp_load_acquire(&wait->flags);
1313 /* If we were non-exclusive, we're done */
1330 wait->flags |= WQ_FLAG_DONE;
1336 * waiter from the wait-queues, but the folio waiters bit will remain
1348 * NOTE! The wait->flags weren't stable until we've done the
1357 * Also note that WQ_FLAG_WOKEN is sufficient for a non-exclusive
1361 return wait->flags & WQ_FLAG_DONE ? 0 : -EINTR;
1363 return wait->flags & WQ_FLAG_WOKEN ? 0 : -EINTR;
1368 * migration_entry_wait_on_locked - Wait for a migration entry to be removed
1402 wait->func = wake_page_function;
1405 wait->flags = 0;
1407 spin_lock_irq(&q->lock);
1411 spin_unlock_irq(&q->lock);
1426 flags = smp_load_acquire(&wait->flags);
1459 * folio_put_wait_locked - Drop a reference and wait for it to be unlocked
1469 * Return: 0 if the folio was unlocked or -EINTR if interrupted by a signal.
1477 * folio_add_wait_queue - Add an arbitrary waiter to a folio's wait queue
1488 spin_lock_irqsave(&q->lock, flags);
1491 spin_unlock_irqrestore(&q->lock, flags);
1496 * folio_unlock - Unlock a locked folio.
1516 * folio_end_read - End read on a folio.
1546 * folio_end_private_2 - Clear PG_private_2 and wake any waiters.
1566 * folio_wait_private_2 - Wait for PG_private_2 to be cleared on a folio.
1579 * folio_wait_private_2_killable - Wait for PG_private_2 to be cleared on a folio.
1586 * - 0 if successful.
1587 * - -EINTR if a fatal signal was encountered.
1604 * folio_end_writeback - End writeback against a folio.
1642 * __folio_lock - Get a lock on the folio, assuming we need to sleep to get it.
1664 wait->folio = folio;
1665 wait->bit_nr = PG_locked;
1667 spin_lock_irq(&q->lock);
1668 __add_wait_queue_entry_tail(q, &wait->wait);
1678 __remove_wait_queue(q, &wait->wait);
1680 ret = -EIOCBQUEUED;
1681 spin_unlock_irq(&q->lock);
1687 * 0 - folio is locked.
1688 * non-zero - folio is not locked.
1689 * mmap_lock or per-VMA lock has been released (mmap_read_unlock() or
1694 * with the folio locked and the mmap_lock/per-VMA lock is left unperturbed.
1698 unsigned int flags = vmf->flags;
1702 * CAUTION! In this case, mmap_lock/per-VMA lock is not
1731 * page_cache_next_miss() - Find the next gap in the page cache.
1732 * @mapping: Mapping.
1736 * Search the range [index, min(index + max_scan - 1, ULONG_MAX)] for the
1746 * range specified (in which case 'return - index >= max_scan' will be true).
1747 * In the rare case of index wrap-around, 0 will be returned.
1749 pgoff_t page_cache_next_miss(struct address_space *mapping,
1752 XA_STATE(xas, &mapping->i_pages, index);
1754 while (max_scan--) {
1767 * page_cache_prev_miss() - Find the previous gap in the page cache.
1768 * @mapping: Mapping.
1772 * Search the range [max(index - max_scan + 1, 0), index] for the
1782 * range specified (in which case 'index - return >= max_scan' will be true).
1783 * In the rare case of wrap-around, ULONG_MAX will be returned.
1785 pgoff_t page_cache_prev_miss(struct address_space *mapping,
1788 XA_STATE(xas, &mapping->i_pages, index);
1790 while (max_scan--) {
1815 * increased by a speculative page cache (or GUP-fast) lookup as it can
1823 * filemap_get_entry - Get a page cache entry.
1824 * @mapping: the address_space to search
1827 * Looks up the page cache entry at @mapping & @index. If it is a folio,
1834 void *filemap_get_entry(struct address_space *mapping, pgoff_t index)
1836 XA_STATE(xas, &mapping->i_pages, index);
1866 * __filemap_get_folio - Find and get a reference to a folio.
1867 * @mapping: The address_space to search.
1872 * Looks up the page cache entry at @mapping & @index.
1881 struct folio *__filemap_get_folio(struct address_space *mapping, pgoff_t index,
1887 folio = filemap_get_entry(mapping, index);
1897 return ERR_PTR(-EAGAIN);
1904 if (unlikely(folio->mapping != mapping)) {
1924 unsigned int min_order = mapping_min_folio_order(mapping);
1927 index = mapping_align_index(mapping, index);
1929 if ((fgp_flags & FGP_WRITE) && mapping_can_writeback(mapping))
1940 if (order > mapping_max_folio_order(mapping))
1941 order = mapping_max_folio_order(mapping);
1943 if (index & ((1UL << order) - 1))
1949 err = -ENOMEM;
1960 err = filemap_add_folio(mapping, folio, index, gfp);
1965 } while (order-- > min_order);
1967 if (err == -EEXIST)
1980 return ERR_PTR(-ENOENT);
2021 * find_get_entries - gang pagecache lookup
2022 * @mapping: The address_space to search
2029 * the mapping. The entries are placed in @fbatch. find_get_entries()
2033 * due to not-present entries or large folios.
2040 unsigned find_get_entries(struct address_space *mapping, pgoff_t *start,
2043 XA_STATE(xas, &mapping->i_pages, *start);
2048 indices[fbatch->nr] = xas.xa_index;
2055 int idx = folio_batch_count(fbatch) - 1;
2057 folio = fbatch->folios[idx];
2061 nr = 1 << xa_get_order(&mapping->i_pages, indices[idx]);
2070 * find_lock_entries - Find a batch of pagecache entries.
2071 * @mapping: The address_space to search.
2077 * find_lock_entries() will return a batch of entries from @mapping.
2084 * due to not-present entries, large folios, folios which could not be
2089 unsigned find_lock_entries(struct address_space *mapping, pgoff_t *start,
2092 XA_STATE(xas, &mapping->i_pages, *start);
2102 base = folio->index;
2107 if (base + nr - 1 > end)
2111 if (folio->mapping != mapping ||
2118 base = xas.xa_index & ~(nr - 1);
2123 if (base + nr - 1 > end)
2129 indices[fbatch->nr] = xas.xa_index;
2144 * filemap_get_folios - Get a batch of folios
2145 * @mapping: The address_space to search
2150 * Search for and return a batch of folios in the mapping starting at
2157 unsigned filemap_get_folios(struct address_space *mapping, pgoff_t *start,
2160 return filemap_get_folios_tag(mapping, start, end, XA_PRESENT, fbatch);
2165 * filemap_get_folios_contig - Get a batch of contiguous folios
2166 * @mapping: The address_space to search
2179 unsigned filemap_get_folios_contig(struct address_space *mapping,
2182 XA_STATE(xas, &mapping->i_pages, *start);
2211 *start = folio->index + nr;
2226 folio = fbatch->folios[nr - 1];
2236 * filemap_get_folios_tag - Get a batch of folios matching @tag
2237 * @mapping: The address_space to search
2254 unsigned filemap_get_folios_tag(struct address_space *mapping, pgoff_t *start,
2257 XA_STATE(xas, &mapping->i_pages, *start);
2271 *start = folio->index + nr;
2278 * breaks the iteration when there is a page at index -1 but that is
2281 if (end == (pgoff_t)-1)
2282 *start = (pgoff_t)-1;
2296 * ---R__________________________________________B__________
2309 ra->ra_pages /= 4;
2313 * filemap_get_read_batch - Get a batch of folios for read
2321 static void filemap_get_read_batch(struct address_space *mapping,
2324 XA_STATE(xas, &mapping->i_pages, index);
2347 xas_advance(&xas, folio_next_index(folio) - 1);
2379 shrink_readahead_size_eio(&file->f_ra);
2380 return -EIO;
2383 static bool filemap_range_uptodate(struct address_space *mapping,
2392 if (!mapping->a_ops->is_partially_uptodate)
2394 if (mapping->host->i_blkbits >= folio_shift(folio))
2398 count -= folio_pos(folio) - pos;
2401 pos -= folio_pos(folio);
2404 return mapping->a_ops->is_partially_uptodate(folio, pos, count);
2408 struct address_space *mapping, size_t count,
2413 if (iocb->ki_flags & IOCB_NOWAIT) {
2414 if (!filemap_invalidate_trylock_shared(mapping))
2415 return -EAGAIN;
2417 filemap_invalidate_lock_shared(mapping);
2421 error = -EAGAIN;
2422 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO))
2424 if (!(iocb->ki_flags & IOCB_WAITQ)) {
2425 filemap_invalidate_unlock_shared(mapping);
2433 error = __folio_lock_async(folio, iocb->ki_waitq);
2439 if (!folio->mapping)
2443 if (filemap_range_uptodate(mapping, iocb->ki_pos, count, folio,
2447 error = -EAGAIN;
2448 if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ))
2451 error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio,
2457 filemap_invalidate_unlock_shared(mapping);
2464 struct address_space *mapping, loff_t pos,
2469 unsigned int min_order = mapping_min_folio_order(mapping);
2472 folio = filemap_alloc_folio(mapping_gfp_mask(mapping), min_order);
2474 return -ENOMEM;
2485 * pages or ->readahead() that need to hold invalidate_lock
2486 * while mapping blocks for IO so let's hold the lock here as
2489 filemap_invalidate_lock_shared(mapping);
2491 error = filemap_add_folio(mapping, folio, index,
2492 mapping_gfp_constraint(mapping, GFP_KERNEL));
2493 if (error == -EEXIST)
2498 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio);
2502 filemap_invalidate_unlock_shared(mapping);
2506 filemap_invalidate_unlock_shared(mapping);
2512 struct address_space *mapping, struct folio *folio,
2515 DEFINE_READAHEAD(ractl, file, &file->f_ra, mapping, folio->index);
2517 if (iocb->ki_flags & IOCB_NOIO)
2518 return -EAGAIN;
2519 page_cache_async_ra(&ractl, folio, last_index - folio->index);
2526 struct file *filp = iocb->ki_filp;
2527 struct address_space *mapping = filp->f_mapping;
2528 struct file_ra_state *ra = &filp->f_ra;
2529 pgoff_t index = iocb->ki_pos >> PAGE_SHIFT;
2536 last_index = DIV_ROUND_UP(iocb->ki_pos + count, PAGE_SIZE);
2539 return -EINTR;
2541 filemap_get_read_batch(mapping, index, last_index - 1, fbatch);
2543 if (iocb->ki_flags & IOCB_NOIO)
2544 return -EAGAIN;
2545 if (iocb->ki_flags & IOCB_NOWAIT)
2547 page_cache_sync_readahead(mapping, ra, filp, index,
2548 last_index - index);
2549 if (iocb->ki_flags & IOCB_NOWAIT)
2551 filemap_get_read_batch(mapping, index, last_index - 1, fbatch);
2554 if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ))
2555 return -EAGAIN;
2556 err = filemap_create_folio(filp, mapping, iocb->ki_pos, fbatch);
2562 folio = fbatch->folios[folio_batch_count(fbatch) - 1];
2564 err = filemap_readahead(iocb, filp, mapping, folio, last_index);
2569 if ((iocb->ki_flags & IOCB_WAITQ) &&
2571 iocb->ki_flags |= IOCB_NOWAIT;
2572 err = filemap_update_page(iocb, mapping, count, folio,
2578 trace_mm_filemap_get_pages(mapping, index, last_index - 1);
2583 if (likely(--fbatch->nr))
2598 * filemap_read - Read data from the page cache.
2600 * @iter: Destination for the data.
2603 * Copies data from the page cache. If the data is not currently present,
2613 struct file *filp = iocb->ki_filp;
2614 struct file_ra_state *ra = &filp->f_ra;
2615 struct address_space *mapping = filp->f_mapping;
2616 struct inode *inode = mapping->host;
2621 loff_t last_pos = ra->prev_pos;
2623 if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes))
2628 iov_iter_truncate(iter, inode->i_sb->s_maxbytes);
2635 * If we've already successfully copied some data, then we
2636 * can no longer safely return -EIOCBQUEUED. Hence mark
2639 if ((iocb->ki_flags & IOCB_WAITQ) && already_read)
2640 iocb->ki_flags |= IOCB_NOWAIT;
2642 if (unlikely(iocb->ki_pos >= i_size_read(inode)))
2645 error = filemap_get_pages(iocb, iter->count, &fbatch, false);
2653 * the correct value for "nr", which means the zero-filled
2655 * another truncate extends the file - this is desired though).
2658 if (unlikely(iocb->ki_pos >= isize))
2660 end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count);
2663 * Once we start copying data, we don't want to be touching any
2666 writably_mapped = mapping_writably_mapped(mapping);
2672 if (!pos_same_folio(iocb->ki_pos, last_pos - 1,
2679 size_t offset = iocb->ki_pos & (fsize - 1);
2680 size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos,
2681 fsize - offset);
2699 iocb->ki_pos += copied;
2700 last_pos = iocb->ki_pos;
2703 error = -EFAULT;
2711 } while (iov_iter_count(iter) && iocb->ki_pos < isize && !error);
2714 ra->prev_pos = last_pos;
2721 struct address_space *mapping = iocb->ki_filp->f_mapping;
2722 loff_t pos = iocb->ki_pos;
2723 loff_t end = pos + count - 1;
2725 if (iocb->ki_flags & IOCB_NOWAIT) {
2726 if (filemap_range_needs_writeback(mapping, pos, end))
2727 return -EAGAIN;
2731 return filemap_write_and_wait_range(mapping, pos, end);
2735 int filemap_invalidate_pages(struct address_space *mapping,
2742 if (filemap_range_has_page(mapping, pos, end))
2743 return -EAGAIN;
2745 ret = filemap_write_and_wait_range(mapping, pos, end);
2752 * the new data. We invalidate clean cached page from the region we're
2754 * without clobbering -EIOCBQUEUED from ->direct_IO().
2756 return invalidate_inode_pages2_range(mapping, pos >> PAGE_SHIFT,
2762 struct address_space *mapping = iocb->ki_filp->f_mapping;
2764 return filemap_invalidate_pages(mapping, iocb->ki_pos,
2765 iocb->ki_pos + count - 1,
2766 iocb->ki_flags & IOCB_NOWAIT);
2771 * generic_file_read_iter - generic filesystem read routine
2773 * @iter: destination for the data read
2778 * The IOCB_NOWAIT flag in iocb->ki_flags indicates that -EAGAIN shall
2779 * be returned when no data can be read without waiting for I/O requests
2782 * The IOCB_NOIO flag in iocb->ki_flags indicates that no new I/O
2783 * requests shall be made for the read or for readahead. When no data
2784 * can be read, -EAGAIN shall be returned. When readahead would be
2800 if (iocb->ki_flags & IOCB_DIRECT) {
2801 struct file *file = iocb->ki_filp;
2802 struct address_space *mapping = file->f_mapping;
2803 struct inode *inode = mapping->host;
2810 retval = mapping->a_ops->direct_IO(iocb, iter);
2812 iocb->ki_pos += retval;
2813 count -= retval;
2815 if (retval != -EIOCBQUEUED)
2816 iov_iter_revert(iter, count - iov_iter_count(iter));
2829 if (iocb->ki_pos >= i_size_read(inode))
2847 size = min(size, folio_size(folio) - offset);
2851 !pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
2853 size_t part = min_t(size_t, PAGE_SIZE - offset, size - spliced);
2862 pipe->head++;
2872 * filemap_splice_read - Splice data from a file's pagecache into a pipe
2884 * will be updated if appropriate; 0 will be returned if there is no more data
2885 * to be read; -EAGAIN will be returned if the pipe had no space, and some
2887 * if the pipe has insufficient space, we reach the end of the data or we hit a
2901 if (unlikely(*ppos >= in->f_mapping->host->i_sb->s_maxbytes))
2907 /* Work out how much data we can actually add into the pipe */
2908 used = pipe_occupancy(pipe->head, pipe->tail);
2909 npages = max_t(ssize_t, pipe->max_usage - used, 0);
2917 if (*ppos >= i_size_read(in->f_mapping->host))
2929 * the correct value for "nr", which means the zero-filled
2931 * another truncate extends the file - this is desired though).
2933 isize = i_size_read(in->f_mapping->host);
2939 * Once we start copying data, we don't want to be touching any
2942 writably_mapped = mapping_writably_mapped(in->f_mapping);
2960 n = min_t(loff_t, len, isize - *ppos);
2964 len -= n;
2967 in->f_ra.prev_pos = *ppos;
2968 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
2984 struct address_space *mapping, struct folio *folio,
2987 const struct address_space_operations *ops = mapping->a_ops;
2988 size_t offset, bsz = i_blocksize(mapping->host);
2992 if (!ops->is_partially_uptodate)
2998 if (unlikely(folio->mapping != mapping))
3001 offset = offset_in_folio(folio, start) & ~(bsz - 1);
3004 if (ops->is_partially_uptodate(folio, offset, bsz) ==
3007 start = (start + bsz) & ~(bsz - 1);
3024 * mapping_seek_hole_data - Seek for SEEK_DATA / SEEK_HOLE in the page cache.
3025 * @mapping: Address space to search.
3031 * contain data, your filesystem can use this function to implement
3033 * entirely memory-based such as tmpfs, and filesystems which support
3036 * Return: The requested offset on success, or -ENXIO if @whence specifies
3037 * SEEK_DATA and there is no data after @start. There is an implicit hole
3038 * after @end - 1, so SEEK_HOLE returns @end if all the bytes between @start
3039 * and @end contain data.
3041 loff_t mapping_seek_hole_data(struct address_space *mapping, loff_t start,
3044 XA_STATE(xas, &mapping->i_pages, start >> PAGE_SHIFT);
3045 pgoff_t max = (end - 1) >> PAGE_SHIFT;
3050 return -ENXIO;
3065 start = folio_seek_hole_data(&xas, mapping, folio, start, pos,
3077 start = -ENXIO;
3090 * lock_folio_maybe_drop_mmap - lock the page, possibly dropping the mmap_lock
3091 * @vmf - the vm_fault for this fault.
3092 * @folio - the folio to lock.
3093 * @fpin - the pointer to the file we may pin (or is already pinned).
3112 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
3116 if (vmf->flags & FAULT_FLAG_KILLABLE) {
3144 struct file *file = vmf->vma->vm_file;
3145 struct file_ra_state *ra = &file->f_ra;
3146 struct address_space *mapping = file->f_mapping;
3147 DEFINE_READAHEAD(ractl, file, ra, mapping, vmf->pgoff);
3149 unsigned long vm_flags = vmf->vma->vm_flags;
3156 ractl._index &= ~((unsigned long)HPAGE_PMD_NR - 1);
3157 ra->size = HPAGE_PMD_NR;
3163 ra->size *= 2;
3164 ra->async_size = HPAGE_PMD_NR;
3170 /* If we don't want any read-ahead, don't bother */
3173 if (!ra->ra_pages)
3178 page_cache_sync_ra(&ractl, ra->ra_pages);
3183 mmap_miss = READ_ONCE(ra->mmap_miss);
3185 WRITE_ONCE(ra->mmap_miss, ++mmap_miss);
3189 * stop bothering with read-ahead. It will only hurt.
3195 * mmap read-around
3198 ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2);
3199 ra->size = ra->ra_pages;
3200 ra->async_size = ra->ra_pages / 4;
3201 ractl._index = ra->start;
3214 struct file *file = vmf->vma->vm_file;
3215 struct file_ra_state *ra = &file->f_ra;
3216 DEFINE_READAHEAD(ractl, file, ra, file->f_mapping, vmf->pgoff);
3220 /* If we don't want any read-ahead, don't bother */
3221 if (vmf->vma->vm_flags & VM_RAND_READ || !ra->ra_pages)
3224 mmap_miss = READ_ONCE(ra->mmap_miss);
3226 WRITE_ONCE(ra->mmap_miss, --mmap_miss);
3230 page_cache_async_ra(&ractl, folio, ra->ra_pages);
3237 struct vm_area_struct *vma = vmf->vma;
3251 * scenario while holding the PT lock, to not degrade non-mlocked
3255 if (!(vma->vm_flags & VM_LOCKED))
3258 if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
3261 ptep = pte_offset_map_nolock(vma->vm_mm, vmf->pmd, vmf->address,
3262 &vmf->ptl);
3269 spin_lock(vmf->ptl);
3272 spin_unlock(vmf->ptl);
3279 * filemap_fault - read in file data for page fault handling
3283 * mapped memory region to read in file data during a page fault.
3289 * vma->vm_mm->mmap_lock must be held on entry.
3299 * Return: bitwise-OR of %VM_FAULT_ codes.
3304 struct file *file = vmf->vma->vm_file;
3306 struct address_space *mapping = file->f_mapping;
3307 struct inode *inode = mapping->host;
3308 pgoff_t max_idx, index = vmf->pgoff;
3317 trace_mm_filemap_fault(mapping, index);
3322 folio = filemap_get_folio(mapping, index);
3328 if (!(vmf->flags & FAULT_FLAG_TRIED))
3331 filemap_invalidate_lock_shared(mapping);
3341 count_memcg_event_mm(vmf->vma->vm_mm, PGMAJFAULT);
3350 filemap_invalidate_lock_shared(mapping);
3353 folio = __filemap_get_folio(mapping, index,
3355 vmf->gfp_mask);
3359 filemap_invalidate_unlock_shared(mapping);
3368 if (unlikely(folio->mapping != mapping)) {
3377 * that it's up-to-date. If not, it is going to be due to an error,
3403 * time to return to the upper layer and have it re-find the vma and
3411 filemap_invalidate_unlock_shared(mapping);
3424 vmf->page = folio_file_page(folio, index);
3429 * Umm, take care of errors if the page isn't up-to-date.
3430 * Try to re-read it _once_. We do this synchronously,
3435 error = filemap_read_folio(file, mapping->a_ops->read_folio, folio);
3442 filemap_invalidate_unlock_shared(mapping);
3449 * re-find the vma and come back and find our hopefully still populated
3455 filemap_invalidate_unlock_shared(mapping);
3465 struct mm_struct *mm = vmf->vma->vm_mm;
3468 if (pmd_trans_huge(*vmf->pmd)) {
3474 if (pmd_none(*vmf->pmd) && folio_test_pmd_mappable(folio)) {
3484 if (pmd_none(*vmf->pmd) && vmf->prealloc_pte)
3485 pmd_install(mm, vmf->pmd, &vmf->prealloc_pte);
3491 struct address_space *mapping, pgoff_t end_pgoff)
3514 if (folio->mapping != mapping)
3518 max_idx = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE);
3519 if (xas->xa_index >= max_idx)
3543 pte_t *old_ptep = vmf->pte;
3552 * In such situation, read-ahead is only a waste of IO.
3554 * we can stop read-ahead.
3562 * fault-around logic.
3564 if (!pte_none(ptep_get(&vmf->pte[count])))
3574 if (in_range(vmf->address, addr, count * PAGE_SIZE))
3580 vmf->pte += count;
3583 } while (--nr_pages > 0);
3589 if (in_range(vmf->address, addr, count * PAGE_SIZE))
3593 vmf->pte = old_ptep;
3603 struct page *page = &folio->page;
3615 * the fault-around logic.
3617 if (!pte_none(ptep_get(vmf->pte)))
3620 if (vmf->address == addr)
3633 struct vm_area_struct *vma = vmf->vma;
3634 struct file *file = vma->vm_file;
3635 struct address_space *mapping = file->f_mapping;
3638 XA_STATE(xas, &mapping->i_pages, start_pgoff);
3645 folio = next_uptodate_folio(&xas, mapping, end_pgoff);
3654 addr = vma->vm_start + ((start_pgoff - vma->vm_pgoff) << PAGE_SHIFT);
3655 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, addr, &vmf->ptl);
3656 if (!vmf->pte) {
3662 file_end = DIV_ROUND_UP(i_size_read(mapping->host), PAGE_SIZE) - 1;
3670 addr += (xas.xa_index - last_pgoff) << PAGE_SHIFT;
3671 vmf->pte += xas.xa_index - last_pgoff;
3673 end = folio_next_index(folio) - 1;
3674 nr_pages = min(end, end_pgoff) - xas.xa_index + 1;
3681 xas.xa_index - folio->index, addr,
3686 } while ((folio = next_uptodate_folio(&xas, mapping, end_pgoff)) != NULL);
3687 add_mm_counter(vma->vm_mm, folio_type, rss);
3688 pte_unmap_unlock(vmf->pte, vmf->ptl);
3689 trace_mm_filemap_map_pages(mapping, start_pgoff, end_pgoff);
3693 mmap_miss_saved = READ_ONCE(file->f_ra.mmap_miss);
3695 WRITE_ONCE(file->f_ra.mmap_miss, 0);
3697 WRITE_ONCE(file->f_ra.mmap_miss, mmap_miss_saved - mmap_miss);
3705 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
3706 struct folio *folio = page_folio(vmf->page);
3709 sb_start_pagefault(mapping->host->i_sb);
3710 file_update_time(vmf->vma->vm_file);
3712 if (folio->mapping != mapping) {
3725 sb_end_pagefault(mapping->host->i_sb);
3739 struct address_space *mapping = file->f_mapping;
3741 if (!mapping->a_ops->read_folio)
3742 return -ENOEXEC;
3744 vma->vm_ops = &generic_file_vm_ops;
3749 * This is for filesystems which do not implement ->writepage.
3754 return -EINVAL;
3764 return -ENOSYS;
3768 return -ENOSYS;
3776 static struct folio *do_read_cache_folio(struct address_space *mapping,
3783 filler = mapping->a_ops->read_folio;
3785 folio = filemap_get_folio(mapping, index);
3788 mapping_min_folio_order(mapping));
3790 return ERR_PTR(-ENOMEM);
3791 index = mapping_align_index(mapping, index);
3792 err = filemap_add_folio(mapping, folio, index, gfp);
3795 if (err == -EEXIST)
3811 /* Folio was truncated from mapping */
3812 if (!folio->mapping) {
3839 * read_cache_folio - Read into page cache, fill it if needed.
3840 * @mapping: The address_space to read from.
3842 * @filler: Function to perform the read, or NULL to use aops->read_folio().
3851 * Context: May sleep. Expects mapping->invalidate_lock to be held.
3854 struct folio *read_cache_folio(struct address_space *mapping, pgoff_t index,
3857 return do_read_cache_folio(mapping, index, filler, file,
3858 mapping_gfp_mask(mapping));
3863 * mapping_read_folio_gfp - Read into page cache, using specified allocation flags.
3864 * @mapping: The address_space for the folio.
3868 * This is the same as "read_cache_folio(mapping, index, NULL, NULL)", but with
3872 * possible and so is EINTR. If ->read_folio returns another error,
3875 * The function expects mapping->invalidate_lock to be already held.
3879 struct folio *mapping_read_folio_gfp(struct address_space *mapping,
3882 return do_read_cache_folio(mapping, index, NULL, NULL, gfp);
3886 static struct page *do_read_cache_page(struct address_space *mapping,
3891 folio = do_read_cache_folio(mapping, index, filler, file, gfp);
3893 return &folio->page;
3897 struct page *read_cache_page(struct address_space *mapping,
3900 return do_read_cache_page(mapping, index, filler, file,
3901 mapping_gfp_mask(mapping));
3906 * read_cache_page_gfp - read into page cache, using specified page allocation flags.
3907 * @mapping: the page's address_space
3911 * This is the same as "read_mapping_page(mapping, index, NULL)", but with
3914 * If the page does not get brought uptodate, return -EIO.
3916 * The function expects mapping->invalidate_lock to be already held.
3920 struct page *read_cache_page_gfp(struct address_space *mapping,
3924 return do_read_cache_page(mapping, index, NULL, NULL, gfp);
3937 errseq_set(&filp->f_mapping->wb_err, -EIO);
3942 pr_crit("Page cache invalidation failure on direct I/O. Possible data corruption due to collision with buffered I/O!\n");
3943 pr_crit("File: %s PID: %d Comm: %.20s\n", path, current->pid,
3944 current->comm);
3950 struct address_space *mapping = iocb->ki_filp->f_mapping;
3952 if (mapping->nrpages &&
3953 invalidate_inode_pages2_range(mapping,
3954 iocb->ki_pos >> PAGE_SHIFT,
3955 (iocb->ki_pos + count - 1) >> PAGE_SHIFT))
3956 dio_warn_stale_pagecache(iocb->ki_filp);
3962 struct address_space *mapping = iocb->ki_filp->f_mapping;
3972 if (written == -EBUSY)
3977 written = mapping->a_ops->direct_IO(iocb, from);
3981 * cached by non-direct readahead, or faulted in by get_user_pages()
3994 * Skip invalidation for async writes or if mapping has no pages.
3997 struct inode *inode = mapping->host;
3998 loff_t pos = iocb->ki_pos;
4002 write_len -= written;
4003 if (pos > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
4007 iocb->ki_pos = pos;
4009 if (written != -EIOCBQUEUED)
4010 iov_iter_revert(from, write_len - iov_iter_count(from));
4017 struct file *file = iocb->ki_filp;
4018 loff_t pos = iocb->ki_pos;
4019 struct address_space *mapping = file->f_mapping;
4020 const struct address_space_operations *a_ops = mapping->a_ops;
4021 size_t chunk = mapping_max_folio_size(mapping);
4034 offset = pos & (chunk - 1);
4035 bytes = min(chunk - offset, bytes);
4036 balance_dirty_pages_ratelimited(mapping);
4042 * up-to-date.
4045 status = -EFAULT;
4050 status = -EINTR;
4054 status = a_ops->write_begin(file, mapping, pos, bytes,
4060 if (bytes > folio_size(folio) - offset)
4061 bytes = folio_size(folio) - offset;
4063 if (mapping_writably_mapped(mapping))
4069 status = a_ops->write_end(file, mapping, pos, bytes, copied,
4072 iov_iter_revert(i, copied - max(status, 0L));
4080 * A short copy made ->write_end() reject the
4099 iocb->ki_pos += written;
4105 * __generic_file_write_iter - write data to a file
4107 * @from: iov_iter with data to write
4109 * This function does all the work needed for actually writing data to a
4117 * This function does *not* take care of syncing data in case of O_SYNC write.
4123 * * negative error code if no data has been written at all
4127 struct file *file = iocb->ki_filp;
4128 struct address_space *mapping = file->f_mapping;
4129 struct inode *inode = mapping->host;
4140 if (iocb->ki_flags & IOCB_DIRECT) {
4147 * page-cache pages correctly).
4160 * generic_file_write_iter - write data to a file
4162 * @from: iov_iter with data to write
4168 * * negative error code if no data has been written at all of
4174 struct file *file = iocb->ki_filp;
4175 struct inode *inode = file->f_mapping->host;
4191 * filemap_release_folio() - Release fs-specific metadata on a folio.
4195 * The address_space is trying to release any data attached to a folio
4196 * (presumably at folio->private).
4209 struct address_space * const mapping = folio->mapping;
4217 if (mapping && mapping->a_ops->release_folio)
4218 return mapping->a_ops->release_folio(folio, gfp);
4224 * filemap_invalidate_inode - Invalidate/forcibly write back a range of an inode's pagecache
4239 struct address_space *mapping = inode->i_mapping;
4242 pgoff_t nr = end == LLONG_MAX ? ULONG_MAX : last - first + 1;
4244 if (!mapping || !mapping->nrpages || end < start)
4248 filemap_invalidate_lock(mapping);
4250 if (!mapping->nrpages)
4253 unmap_mapping_pages(mapping, first, nr, false);
4255 /* Write back the data if we're asked to. */
4264 filemap_fdatawrite_wbc(mapping, &wbc);
4268 invalidate_inode_pages2_range(mapping, start / PAGE_SIZE, end / PAGE_SIZE);
4271 filemap_invalidate_unlock(mapping);
4273 return filemap_check_errors(mapping);
4279 * filemap_cachestat() - compute the page cache statistics of a mapping
4280 * @mapping: The mapping to compute the statistics for.
4285 * This will query the page cache statistics of a mapping in the
4290 static void filemap_cachestat(struct address_space *mapping,
4293 XA_STATE(xas, &mapping->i_pages, first_index);
4313 * the rcu-protected xarray.
4322 folio_last_index = folio_first_index + nr_pages - 1;
4326 nr_pages -= first_index - folio_first_index;
4329 nr_pages -= folio_last_index - last_index;
4336 cs->nr_evicted += nr_pages;
4339 if (shmem_mapping(mapping)) {
4340 /* shmem file - in swap cache */
4363 cs->nr_recently_evicted += nr_pages;
4369 cs->nr_cache += nr_pages;
4372 cs->nr_dirty += nr_pages;
4375 cs->nr_writeback += nr_pages;
4400 * `off` and `len` must be non-negative integers. If `len` > 0,
4414 * zero - success
4415 * -EFAULT - cstat or cstat_range points to an illegal address
4416 * -EINVAL - invalid flags
4417 * -EBADF - invalid file descriptor
4418 * -EOPNOTSUPP - file descriptor is of a hugetlbfs file
4425 struct address_space *mapping;
4431 return -EBADF;
4436 return -EFAULT;
4442 return -EOPNOTSUPP;
4447 return -EINVAL;
4452 csr.len == 0 ? ULONG_MAX : (csr.off + csr.len - 1) >> PAGE_SHIFT;
4454 mapping = fd_file(f)->f_mapping;
4455 filemap_cachestat(mapping, first_index, last_index, &cs);
4459 return -EFAULT;