Lines Matching defs:rmap_item
156 * @rmap_hlist_len: number of rmap_item entries in hlist or STABLE_NODE_CHAIN
189 * @rmap_list: next rmap_item in mm_slot's singly-linked rmap_list
192 * @mm: the memory structure this rmap_item is pointing into
193 * @address: the virtual address this rmap_item tracks (+ flags in low bits)
195 * @node: rb node of this rmap_item in the unstable tree
563 struct ksm_rmap_item *rmap_item;
565 rmap_item = kmem_cache_zalloc(rmap_item_cache, GFP_KERNEL |
567 if (rmap_item)
569 return rmap_item;
572 static inline void free_rmap_item(struct ksm_rmap_item *rmap_item)
575 rmap_item->mm->ksm_rmap_items--;
576 rmap_item->mm = NULL; /* debug safety */
577 kmem_cache_free(rmap_item_cache, rmap_item);
718 static void break_cow(struct ksm_rmap_item *rmap_item)
720 struct mm_struct *mm = rmap_item->mm;
721 unsigned long addr = rmap_item->address;
728 put_anon_vma(rmap_item->anon_vma);
737 static struct page *get_mergeable_page(struct ksm_rmap_item *rmap_item)
739 struct mm_struct *mm = rmap_item->mm;
740 unsigned long addr = rmap_item->address;
823 struct ksm_rmap_item *rmap_item;
828 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
829 if (rmap_item->hlist.next) {
831 trace_ksm_remove_rmap_item(stable_node->kpfn, rmap_item, rmap_item->mm);
836 rmap_item->mm->ksm_merging_pages--;
840 put_anon_vma(rmap_item->anon_vma);
841 rmap_item->address &= PAGE_MASK;
965 * Removing rmap_item from stable or unstable tree.
968 static void remove_rmap_item_from_tree(struct ksm_rmap_item *rmap_item)
970 if (rmap_item->address & STABLE_FLAG) {
974 stable_node = rmap_item->head;
979 hlist_del(&rmap_item->hlist);
988 rmap_item->mm->ksm_merging_pages--;
993 put_anon_vma(rmap_item->anon_vma);
994 rmap_item->head = NULL;
995 rmap_item->address &= PAGE_MASK;
997 } else if (rmap_item->address & UNSTABLE_FLAG) {
1003 * if this rmap_item was inserted by this scan, rather
1006 age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);
1009 rb_erase(&rmap_item->node,
1010 root_unstable_tree + NUMA(rmap_item->nid));
1012 rmap_item->address &= PAGE_MASK;
1021 struct ksm_rmap_item *rmap_item = *rmap_list;
1022 *rmap_list = rmap_item->rmap_list;
1023 remove_rmap_item_from_tree(rmap_item);
1024 free_rmap_item(rmap_item);
1031 * that - an rmap_item is assigned to the stable tree after inserting ksm
1517 static int try_to_merge_with_zero_page(struct ksm_rmap_item *rmap_item,
1520 struct mm_struct *mm = rmap_item->mm;
1527 if (ksm_use_zero_pages && (rmap_item->oldchecksum == zero_checksum)) {
1531 vma = find_mergeable_vma(mm, rmap_item->address);
1534 ZERO_PAGE(rmap_item->address));
1536 page_to_pfn(ZERO_PAGE(rmap_item->address)),
1537 rmap_item, mm, err);
1557 static int try_to_merge_with_ksm_page(struct ksm_rmap_item *rmap_item,
1560 struct mm_struct *mm = rmap_item->mm;
1565 vma = find_mergeable_vma(mm, rmap_item->address);
1574 remove_rmap_item_from_tree(rmap_item);
1577 rmap_item->anon_vma = vma->anon_vma;
1582 rmap_item, mm, err);
1596 static struct folio *try_to_merge_two_pages(struct ksm_rmap_item *rmap_item,
1603 err = try_to_merge_with_ksm_page(rmap_item, page, NULL);
1612 break_cow(rmap_item);
2099 * else insert rmap_item into the unstable tree.
2103 * tree, we insert rmap_item as a new object into the unstable tree.
2105 * This function returns pointer to rmap_item found to be identical
2112 struct ksm_rmap_item *unstable_tree_search_insert(struct ksm_rmap_item *rmap_item,
2168 rmap_item->address |= UNSTABLE_FLAG;
2169 rmap_item->address |= (ksm_scan.seqnr & SEQNR_MASK);
2170 DO_NUMA(rmap_item->nid = nid);
2171 rb_link_node(&rmap_item->node, parent, new);
2172 rb_insert_color(&rmap_item->node, root);
2179 * stable_tree_append - add another rmap_item to the linked list of
2183 static void stable_tree_append(struct ksm_rmap_item *rmap_item,
2189 * rmap_item in the right stable_node
2205 rmap_item->head = stable_node;
2206 rmap_item->address |= STABLE_FLAG;
2207 hlist_add_head(&rmap_item->hlist, &stable_node->hlist);
2209 if (rmap_item->hlist.next)
2214 rmap_item->mm->ksm_merging_pages++;
2224 * @rmap_item: the reverse mapping into the virtual address of this page
2226 static void cmp_and_merge_page(struct page *page, struct ksm_rmap_item *rmap_item)
2246 rmap_item->head == stable_node)
2255 remove_rmap_item_from_tree(rmap_item);
2264 if (rmap_item->oldchecksum != checksum) {
2265 rmap_item->oldchecksum = checksum;
2269 if (!try_to_merge_with_zero_page(rmap_item, page))
2275 if (&kfolio->page == page && rmap_item->head == stable_node) {
2280 remove_rmap_item_from_tree(rmap_item);
2286 err = try_to_merge_with_ksm_page(rmap_item, page, &kfolio->page);
2290 * add its rmap_item to the stable tree.
2293 stable_tree_append(rmap_item, folio_stable_node(kfolio),
2302 unstable_tree_search_insert(rmap_item, page, &tree_page);
2306 kfolio = try_to_merge_two_pages(rmap_item, page,
2331 stable_tree_append(rmap_item, stable_node,
2344 break_cow(rmap_item);
2368 struct ksm_rmap_item *rmap_item;
2371 rmap_item = *rmap_list;
2372 if ((rmap_item->address & PAGE_MASK) == addr)
2373 return rmap_item;
2374 if (rmap_item->address > addr)
2376 *rmap_list = rmap_item->rmap_list;
2377 remove_rmap_item_from_tree(rmap_item);
2378 free_rmap_item(rmap_item);
2381 rmap_item = alloc_rmap_item();
2382 if (rmap_item) {
2384 rmap_item->mm = mm_slot->slot.mm;
2385 rmap_item->mm->ksm_rmap_items++;
2386 rmap_item->address = addr;
2387 rmap_item->rmap_list = *rmap_list;
2388 *rmap_list = rmap_item;
2390 return rmap_item;
2398 * @age: rmap_item age of page
2416 * @rmap_item: associated rmap_item of page
2419 struct ksm_rmap_item *rmap_item)
2434 age = rmap_item->age;
2436 rmap_item->age++;
2449 if (!rmap_item->remaining_skips) {
2450 rmap_item->remaining_skips = skip_age(age);
2456 rmap_item->remaining_skips--;
2457 remove_rmap_item_from_tree(rmap_item);
2467 struct ksm_rmap_item *rmap_item;
2568 rmap_item = get_next_rmap_item(mm_slot,
2570 if (rmap_item) {
2572 &rmap_item->rmap_list;
2574 if (should_skip_rmap_item(folio, rmap_item)) {
2585 return rmap_item;
2657 struct ksm_rmap_item *rmap_item;
2662 rmap_item = scan_get_next_rmap_item(&page);
2663 if (!rmap_item)
2665 cmp_and_merge_page(page, rmap_item);
3019 struct ksm_rmap_item *rmap_item;
3034 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
3035 struct anon_vma *anon_vma = rmap_item->anon_vma;
3055 addr = rmap_item->address & PAGE_MASK;
3061 * rmap_item; but later, if there is still work to do,
3065 if ((rmap_item->mm == vma->vm_mm) == search_new_forks)
3094 struct ksm_rmap_item *rmap_item;
3101 hlist_for_each_entry(rmap_item, &stable_node->hlist, hlist) {
3102 struct anon_vma *av = rmap_item->anon_vma;
3118 addr = rmap_item->address & PAGE_MASK;