Home
last modified time | relevance | path

Searched refs:folio (Results 1 – 25 of 337) sorted by relevance

12345678910>>...14

/linux/include/linux/
H A Drmap.h112 static __always_inline void folio_lock_large_mapcount(struct folio *folio) in folio_lock_large_mapcount() argument
114 bit_spin_lock(FOLIO_MM_IDS_LOCK_BITNUM, &folio->_mm_ids); in folio_lock_large_mapcount()
117 static __always_inline void folio_unlock_large_mapcount(struct folio *folio) in folio_unlock_large_mapcount() argument
119 __bit_spin_unlock(FOLIO_MM_IDS_LOCK_BITNUM, &folio->_mm_ids); in folio_unlock_large_mapcount()
122 static inline unsigned int folio_mm_id(const struct folio *folio, int idx) in folio_mm_id() argument
125 return folio->_mm_id[idx] & MM_ID_MASK; in folio_mm_id()
128 static inline void folio_set_mm_id(struct folio *folio, int idx, mm_id_t id) in folio_set_mm_id() argument
131 folio->_mm_id[idx] &= ~MM_ID_MASK; in folio_set_mm_id()
132 folio->_mm_id[idx] |= id; in folio_set_mm_id()
135 static inline void __folio_large_mapcount_sanity_checks(const struct folio *folio, in __folio_large_mapcount_sanity_checks() argument
[all …]
H A Dpagemap.h549 struct address_space *folio_mapping(const struct folio *folio);
563 static inline struct address_space *folio_flush_mapping(struct folio *folio) in folio_flush_mapping() argument
565 if (unlikely(folio_test_swapcache(folio))) in folio_flush_mapping()
568 return folio_mapping(folio); in folio_flush_mapping()
580 static inline struct inode *folio_inode(struct folio *folio) in folio_inode() argument
582 return folio->mapping->host; in folio_inode()
593 static inline void folio_attach_private(struct folio *folio, void *data) in folio_attach_private() argument
595 folio_get(folio); in folio_attach_private()
596 folio->private = data; in folio_attach_private()
597 folio_set_private(folio); in folio_attach_private()
[all …]
H A Dmemremap.h83 void (*folio_free)(struct folio *folio);
108 void (*folio_split)(struct folio *head, struct folio *tail);
166 static inline bool folio_is_device_private(const struct folio *folio) in folio_is_device_private() argument
169 folio_is_zone_device(folio) && in folio_is_device_private()
170 folio->pgmap->type == MEMORY_DEVICE_PRIVATE; in folio_is_device_private()
179 static inline bool folio_is_pci_p2pdma(const struct folio *folio) in folio_is_pci_p2pdma() argument
182 folio_is_zone_device(folio) && in folio_is_pci_p2pdma()
183 folio->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA; in folio_is_pci_p2pdma()
186 static inline void *folio_zone_device_data(const struct folio *folio) in folio_zone_device_data() argument
188 VM_WARN_ON_FOLIO(!folio_is_device_private(folio), folio); in folio_zone_device_data()
[all …]
H A Dmemcontrol.h394 static inline struct obj_cgroup *folio_objcg(struct folio *folio) in folio_objcg() argument
396 unsigned long memcg_data = folio->memcg_data; in folio_objcg()
398 VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); in folio_objcg()
399 VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio); in folio_objcg()
432 static inline struct mem_cgroup *folio_memcg(struct folio *folio) in folio_memcg() argument
434 struct obj_cgroup *objcg = folio_objcg(folio); in folio_memcg()
445 static inline bool folio_memcg_charged(struct folio *folio) in folio_memcg_charged() argument
447 return folio->memcg_data != 0; in folio_memcg_charged()
465 static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) in folio_memcg_check() argument
471 unsigned long memcg_data = READ_ONCE(folio->memcg_data); in folio_memcg_check()
[all …]
H A Dswap.h283 struct folio *folio = page_folio(page); in page_swap_entry() local
284 swp_entry_t entry = folio->swap; in page_swap_entry()
286 entry.val += folio_page_idx(folio, page); in page_swap_entry()
297 void folio_add_lru(struct folio *folio);
298 void folio_mark_accessed(struct folio *folio);
306 void lru_cache_drain_for_folio(const struct folio *folio,
320 long remove_mapping(struct address_space *mapping, struct folio *folio);
354 void free_swap_cache(struct folio *folio);
355 void free_folio_and_swap_cache(struct folio *folio);
384 sector_t swap_folio_sector(struct folio *folio);
[all …]
/linux/mm/
H A Dfolio.c67 static void __page_cache_release(struct folio *folio, struct lruvec **lruvecp, in __page_cache_release() argument
70 if (folio_test_lru(folio)) { in __page_cache_release()
71 folio_lruvec_relock_irqsave(folio, lruvecp, flagsp); in __page_cache_release()
72 lruvec_del_folio(*lruvecp, folio); in __page_cache_release()
73 __folio_clear_lru_flags(folio); in __page_cache_release()
81 static void page_cache_release(struct folio *folio) in page_cache_release() argument
86 __page_cache_release(folio, &lruvec, &flags); in page_cache_release()
91 void __folio_put(struct folio *folio) in __folio_put() argument
93 if (unlikely(folio_is_zone_device(folio))) { in __folio_put()
94 free_zone_device_folio(folio); in __folio_put()
[all …]
H A Dfilemap.c130 struct folio *folio, void *shadow) in page_cache_delete() argument
132 XA_STATE(xas, &mapping->i_pages, folio->index); in page_cache_delete()
137 xas_set_order(&xas, folio->index, folio_order(folio)); in page_cache_delete()
138 nr = folio_nr_pages(folio); in page_cache_delete()
140 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); in page_cache_delete()
145 folio->mapping = NULL; in page_cache_delete()
151 struct folio *folio) in filemap_unaccount_folio() argument
155 VM_BUG_ON_FOLIO(folio_mapped(folio), folio); in filemap_unaccount_folio()
156 if (!IS_ENABLED(CONFIG_DEBUG_VM) && unlikely(folio_mapped(folio))) { in filemap_unaccount_folio()
158 current->comm, folio_pfn(folio)); in filemap_unaccount_folio()
[all …]
H A Dswap.h174 const struct folio *folio, bool irq) in __swap_cluster_get_and_lock() argument
176 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); in __swap_cluster_get_and_lock()
177 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); in __swap_cluster_get_and_lock()
178 return __swap_cluster_lock(__swap_entry_to_info(folio->swap), in __swap_cluster_get_and_lock()
179 swp_offset(folio->swap), irq); in __swap_cluster_get_and_lock()
195 const struct folio *folio) in swap_cluster_get_and_lock() argument
197 return __swap_cluster_get_and_lock(folio, false); in swap_cluster_get_and_lock()
210 const struct folio *folio) in swap_cluster_get_and_lock_irq() argument
212 return __swap_cluster_get_and_lock(folio, true); in swap_cluster_get_and_lock_irq()
246 int folio_alloc_swap(struct folio *folio);
[all …]
H A Drmap.c591 struct anon_vma *folio_get_anon_vma(const struct folio *folio) in folio_get_anon_vma() argument
596 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); in folio_get_anon_vma()
599 anon_mapping = (unsigned long)READ_ONCE(folio->mapping); in folio_get_anon_vma()
602 if (!folio_mapped(folio)) in folio_get_anon_vma()
618 if (!folio_mapped(folio)) { in folio_get_anon_vma()
637 struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, in folio_lock_anon_vma_read() argument
644 VM_WARN_ON_FOLIO(!folio_test_locked(folio), folio); in folio_lock_anon_vma_read()
647 anon_mapping = (unsigned long)READ_ONCE(folio->mapping); in folio_lock_anon_vma_read()
650 if (!folio_mapped(folio)) in folio_lock_anon_vma_read()
661 if (!folio_mapped(folio)) { in folio_lock_anon_vma_read()
[all …]
H A Dpage_io.c128 static bool is_folio_zero_filled(struct folio *folio) in is_folio_zero_filled() argument
135 for (i = 0; i < folio_nr_pages(folio); i++) { in is_folio_zero_filled()
136 data = kmap_local_folio(folio, i * PAGE_SIZE); in is_folio_zero_filled()
158 static void swap_zeromap_folio_set(struct folio *folio) in swap_zeromap_folio_set() argument
160 struct obj_cgroup *objcg = get_obj_cgroup_from_folio(folio); in swap_zeromap_folio_set()
161 int nr_pages = folio_nr_pages(folio); in swap_zeromap_folio_set()
166 VM_WARN_ON_ONCE_FOLIO(!folio_test_swapcache(folio), folio); in swap_zeromap_folio_set()
167 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); in swap_zeromap_folio_set()
169 ci = swap_cluster_get_and_lock(folio); in swap_zeromap_folio_set()
170 for (i = 0; i < folio_nr_pages(folio); i++) { in swap_zeromap_folio_set()
[all …]
H A Dmlock.c61 static struct lruvec *__mlock_folio(struct folio *folio, struct lruvec *lruvec) in __mlock_folio() argument
64 if (!folio_test_clear_lru(folio)) in __mlock_folio()
67 lruvec = folio_lruvec_relock_irq(folio, lruvec); in __mlock_folio()
69 if (unlikely(folio_evictable(folio))) { in __mlock_folio()
75 if (folio_test_unevictable(folio)) { in __mlock_folio()
76 lruvec_del_folio(lruvec, folio); in __mlock_folio()
77 folio_clear_unevictable(folio); in __mlock_folio()
78 lruvec_add_folio(lruvec, folio); in __mlock_folio()
81 folio_nr_pages(folio)); in __mlock_folio()
86 if (folio_test_unevictable(folio)) { in __mlock_folio()
[all …]
H A Dswap_state.c93 struct folio *swap_cache_get_folio(swp_entry_t entry) in swap_cache_get_folio()
96 struct folio *folio; in swap_cache_get_folio() local
103 folio = swp_tb_to_folio(swp_tb); in swap_cache_get_folio()
104 if (likely(folio_try_get(folio))) in swap_cache_get_folio()
105 return folio; in swap_cache_get_folio()
210 struct folio *folio, swp_entry_t entry) in __swap_cache_do_add_folio() argument
213 unsigned long nr_pages = folio_nr_pages(folio); in __swap_cache_do_add_folio()
214 unsigned long pfn = folio_pfn(folio); in __swap_cache_do_add_folio()
217 VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); in __swap_cache_do_add_folio()
218 VM_WARN_ON_ONCE_FOLIO(folio_test_swapcache(folio), folio); in __swap_cache_do_add_folio()
[all …]
H A Dmigrate.c120 struct folio *folio = folio_get_nontail_page(page); in isolate_movable_ops_page() local
132 if (!folio) in isolate_movable_ops_page()
157 if (unlikely(!folio_trylock(folio))) in isolate_movable_ops_page()
174 folio_unlock(folio); in isolate_movable_ops_page()
179 folio_unlock(folio); in isolate_movable_ops_page()
181 folio_put(folio); in isolate_movable_ops_page()
200 struct folio *folio = page_folio(page); in putback_movable_ops_page() local
204 folio_lock(folio); in putback_movable_ops_page()
207 folio_unlock(folio); in putback_movable_ops_page()
208 folio_put(folio); in putback_movable_ops_page()
[all …]
H A Dinternal.h38 void *workingset_eviction(struct folio *folio,
40 void workingset_refault(struct folio *folio, void *shadow);
41 void workingset_activation(struct folio *folio);
44 void folio_add_lru_vma(struct folio *folio, struct vm_area_struct *vma);
46 static inline bool folio_may_be_lru_cached(const struct folio *folio) in folio_may_be_lru_cached() argument
53 return !folio_test_large(folio); in folio_may_be_lru_cached()
65 void folio_deactivate(struct folio *folio);
66 void folio_mark_lazyfree(struct folio *folio);
197 static inline int folio_nr_pages_mapped(const struct folio *folio) in folio_nr_pages_mapped() argument
201 return atomic_read(&folio->_nr_pages_mapped) & FOLIO_PAGES_MAPPED; in folio_nr_pages_mapped()
[all …]
H A Dmigrate_device.c85 static int migrate_vma_split_folio(struct folio *folio, in migrate_vma_split_folio() argument
89 struct folio *fault_folio = fault_page ? page_folio(fault_page) : NULL; in migrate_vma_split_folio()
90 struct folio *new_fault_folio = NULL; in migrate_vma_split_folio()
92 if (folio != fault_folio) in migrate_vma_split_folio()
93 folio_lock(folio); in migrate_vma_split_folio()
95 ret = split_folio(folio); in migrate_vma_split_folio()
97 if (folio != fault_folio) { in migrate_vma_split_folio()
98 folio_unlock(folio); in migrate_vma_split_folio()
99 folio_put(folio); in migrate_vma_split_folio()
111 folio_unlock(folio); in migrate_vma_split_folio()
[all …]
H A Dhuge_memory.c82 struct folio *huge_zero_folio __read_mostly;
273 static struct folio *alloc_huge_zero_folio(void) in alloc_huge_zero_folio()
275 struct folio *zero_folio; in alloc_huge_zero_folio()
305 struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) in mm_get_huge_zero_folio()
316 struct folio *zero_folio; in get_huge_zero_folio()
364 struct folio *zero_folio; in shrink_huge_zero_folio_scan()
402 struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) in mm_get_huge_zero_folio()
1057 int folio_memcg_alloc_deferred(struct folio *folio) in folio_memcg_alloc_deferred() argument
1061 return folio_memcg_list_lru_alloc(folio, &deferred_split_lru, GFP_KERNEL); in folio_memcg_alloc_deferred()
1270 static inline bool is_transparent_hugepage(const struct folio *folio) in is_transparent_hugepage() argument
[all …]
H A Dmemory-failure.c317 void shake_folio(struct folio *folio) in shake_folio() argument
319 if (folio_test_hugetlb(folio)) in shake_folio()
325 if (folio_test_slab(folio)) in shake_folio()
545 static void collect_procs_anon(const struct folio *folio, in collect_procs_anon() argument
553 av = folio_lock_anon_vma_read(folio, NULL); in collect_procs_anon()
557 pgoff = page_pgoff(folio, page); in collect_procs_anon()
582 static void collect_procs_file(const struct folio *folio, in collect_procs_file() argument
588 struct address_space *mapping = folio->mapping; in collect_procs_file()
593 pgoff = page_pgoff(folio, page); in collect_procs_file()
610 addr = page_address_in_vma(folio, page, vma); in collect_procs_file()
[all …]
/linux/fs/btrfs/
H A Dsubpage.c39 struct folio *folio, enum btrfs_folio_type type) in btrfs_attach_folio_state() argument
45 ASSERT(!folio_test_large(folio)); in btrfs_attach_folio_state()
51 if (folio->mapping) in btrfs_attach_folio_state()
52 ASSERT(folio_test_locked(folio)); in btrfs_attach_folio_state()
55 if (folio_test_private(folio)) in btrfs_attach_folio_state()
59 if (type == BTRFS_SUBPAGE_DATA && !btrfs_is_subpage(fs_info, folio)) in btrfs_attach_folio_state()
62 bfs = btrfs_alloc_folio_state(fs_info, folio_size(folio), type, GFP_NOFS); in btrfs_attach_folio_state()
66 folio_attach_private(folio, bfs); in btrfs_attach_folio_state()
70 void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio *folio, in btrfs_detach_folio_state() argument
76 if (!folio_test_private(folio)) in btrfs_detach_folio_state()
[all …]
H A Dsubpage.h12 struct folio;
99 struct folio *folio) in btrfs_is_subpage() argument
101 if (folio->mapping && folio->mapping->host) in btrfs_is_subpage()
102 ASSERT(is_data_inode(BTRFS_I(folio->mapping->host))); in btrfs_is_subpage()
103 return fs_info->sectorsize < folio_size(folio); in btrfs_is_subpage()
107 struct folio *folio, enum btrfs_folio_type type);
108 void btrfs_detach_folio_state(const struct btrfs_fs_info *fs_info, struct folio *folio,
120 void btrfs_folio_inc_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio);
121 void btrfs_folio_dec_eb_refs(const struct btrfs_fs_info *fs_info, struct folio *folio);
124 struct folio *folio, u64 start, u32 len);
[all …]
/linux/mm/damon/
H A Dops-common.c24 struct folio *damon_get_folio(unsigned long pfn) in damon_get_folio()
27 struct folio *folio; in damon_get_folio() local
32 folio = page_folio(page); in damon_get_folio()
33 if (!folio_try_get(folio)) in damon_get_folio()
35 if (unlikely(page_folio(page) != folio) || !folio_test_lru(folio)) { in damon_get_folio()
36 folio_put(folio); in damon_get_folio()
37 folio = NULL; in damon_get_folio()
39 return folio; in damon_get_folio()
45 struct folio *folio; in damon_ptep_mkold() local
54 folio = damon_get_folio(pfn); in damon_ptep_mkold()
[all …]
/linux/fs/iomap/
H A Dbuffered-io.c35 static inline bool ifs_is_fully_uptodate(struct folio *folio, in ifs_is_fully_uptodate() argument
38 struct inode *inode = folio->mapping->host; in ifs_is_fully_uptodate()
40 return bitmap_full(ifs->state, i_blocks_per_folio(inode, folio)); in ifs_is_fully_uptodate()
47 static unsigned ifs_next_uptodate_block(struct folio *folio, in ifs_next_uptodate_block() argument
50 struct iomap_folio_state *ifs = folio->private; in ifs_next_uptodate_block()
59 static unsigned ifs_next_nonuptodate_block(struct folio *folio, in ifs_next_nonuptodate_block() argument
62 struct iomap_folio_state *ifs = folio->private; in ifs_next_nonuptodate_block()
67 static bool ifs_set_range_uptodate(struct folio *folio, in ifs_set_range_uptodate() argument
70 struct inode *inode = folio->mapping->host; in ifs_set_range_uptodate()
78 return ifs_is_fully_uptodate(folio, ifs); in ifs_set_range_uptodate()
[all …]
/linux/fs/
H A Dmpage.c51 folio_end_read(fi.folio, err == 0); in mpage_read_end_io()
63 mapping_set_error(fi.folio->mapping, err); in mpage_write_end_io()
64 folio_end_writeback(fi.folio); in mpage_write_end_io()
96 static void map_buffer_to_folio(struct folio *folio, struct buffer_head *bh, in map_buffer_to_folio() argument
99 struct inode *inode = folio->mapping->host; in map_buffer_to_folio()
103 head = folio_buffers(folio); in map_buffer_to_folio()
109 if (inode->i_blkbits == folio_shift(folio) && in map_buffer_to_folio()
111 folio_mark_uptodate(folio); in map_buffer_to_folio()
114 head = create_empty_buffers(folio, i_blocksize(inode), 0); in map_buffer_to_folio()
132 struct folio *folio; member
[all …]
H A Dbuffer.c83 void buffer_check_dirty_writeback(struct folio *folio, in buffer_check_dirty_writeback() argument
90 BUG_ON(!folio_test_locked(folio)); in buffer_check_dirty_writeback()
92 head = folio_buffers(folio); in buffer_check_dirty_writeback()
96 if (folio_test_writeback(folio)) in buffer_check_dirty_writeback()
225 struct folio *folio; in __find_get_block_slow() local
230 folio = __filemap_get_folio(bd_mapping, index, FGP_ACCESSED, 0); in __find_get_block_slow()
231 if (IS_ERR(folio)) in __find_get_block_slow()
242 folio_lock(folio); in __find_get_block_slow()
244 head = folio_buffers(folio); in __find_get_block_slow()
287 folio_unlock(folio); in __find_get_block_slow()
[all …]
/linux/fs/xfs/scrub/
H A Dxfile.c126 struct folio *folio; in xfile_load() local
130 if (shmem_get_folio(inode, pos >> PAGE_SHIFT, 0, &folio, in xfile_load()
133 if (!folio) { in xfile_load()
143 folio_unlock(folio); in xfile_load()
144 folio_put(folio); in xfile_load()
148 offset = offset_in_folio(folio, pos); in xfile_load()
149 len = min_t(ssize_t, count, folio_size(folio) - offset); in xfile_load()
150 memcpy(buf, folio_address(folio) + offset, len); in xfile_load()
152 folio_unlock(folio); in xfile_load()
153 folio_put(folio); in xfile_load()
[all …]
/linux/fs/netfs/
H A Dbuffered_write.c16 * Grab a folio for writing and lock it. Attempt to allocate as large a folio
19 static struct folio *netfs_grab_folio_for_write(struct address_space *mapping, in netfs_grab_folio_for_write()
102 struct folio *folio = NULL, *writethrough = NULL; in netfs_perform_write() local
139 size_t offset; /* Offset into pagecache folio */ in netfs_perform_write()
140 size_t part; /* Bytes to write to folio */ in netfs_perform_write()
162 folio = netfs_grab_folio_for_write(mapping, pos, part); in netfs_perform_write()
163 if (IS_ERR(folio)) { in netfs_perform_write()
164 ret = PTR_ERR(folio); in netfs_perform_write()
540 struct folio *folio = page_folio(vmf->page); netfs_page_mkwrite() local
[all...]

12345678910>>...14