Lines Matching refs:ops
58 struct mem_ops *ops, bool expect);
304 static void *alloc_hpage(struct mem_ops *ops)
306 void *p = ops->setup_area(1);
308 ops->fault(p, 0, hpage_pmd_size);
322 if (!ops->check_huge(p, 1)) {
497 struct mem_ops *ops, bool expect)
517 else if (!ops->check_huge(p, expect ? nr_hpages : 0))
526 struct mem_ops *ops, bool expect)
529 if (!ops->check_huge(p, 0)) {
533 __madvise_collapse(msg, p, nr_hpages, ops, expect);
538 struct mem_ops *ops)
544 if (!ops->check_huge(p, 0)) {
556 if (ops->check_huge(p, nr_hpages))
568 struct mem_ops *ops, bool expect)
570 if (wait_for_scan(msg, p, nr_hpages, ops)) {
583 if (ops != &__anon_ops)
584 ops->fault(p, 0, nr_hpages * hpage_pmd_size);
586 if (ops->check_huge(p, expect ? nr_hpages : 0))
604 static bool is_tmpfs(struct mem_ops *ops)
606 return ops == &__file_ops && finfo.type == VMA_SHMEM;
609 static bool is_anon(struct mem_ops *ops)
611 return ops == &__anon_ops;
641 static void collapse_full(struct collapse_context *c, struct mem_ops *ops)
647 p = ops->setup_area(nr_hpages);
648 ops->fault(p, 0, size);
650 ops, true);
652 ops->cleanup_area(p, size);
655 static void collapse_empty(struct collapse_context *c, struct mem_ops *ops)
659 p = ops->setup_area(1);
660 c->collapse("Do not collapse empty PTE table", p, 1, ops, false);
661 ops->cleanup_area(p, hpage_pmd_size);
664 static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops *ops)
668 p = ops->setup_area(1);
669 ops->fault(p, 0, page_size);
671 1, ops, true);
672 ops->cleanup_area(p, hpage_pmd_size);
675 static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *ops)
680 int fault_nr_pages = is_anon(ops) ? 1 << anon_order : 1;
685 p = ops->setup_area(1);
687 if (is_tmpfs(ops)) {
694 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none - fault_nr_pages) * page_size);
696 ops, !c->enforce_pte_scan_limits);
700 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
701 c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops,
707 ops->cleanup_area(p, hpage_pmd_size);
711 static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_ops *ops)
715 p = ops->setup_area(1);
716 ops->fault(p, 0, hpage_pmd_size);
730 c->collapse("Collapse with swapping in single PTE entry", p, 1, ops,
734 ops->cleanup_area(p, hpage_pmd_size);
737 static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *ops)
742 p = ops->setup_area(1);
743 ops->fault(p, 0, hpage_pmd_size);
757 c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ops,
762 ops->fault(p, 0, hpage_pmd_size);
777 1, ops, true);
781 ops->cleanup_area(p, hpage_pmd_size);
784 static void collapse_single_pte_entry_compound(struct collapse_context *c, struct mem_ops *ops)
788 p = alloc_hpage(ops);
790 if (is_tmpfs(ops)) {
800 if (ops->check_huge(p, 0))
806 p, 1, ops, true);
809 ops->cleanup_area(p, hpage_pmd_size);
812 static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops *ops)
816 p = alloc_hpage(ops);
820 if (ops->check_huge(p, 0))
825 c->collapse("Collapse PTE table full of compound pages", p, 1, ops,
828 ops->cleanup_area(p, hpage_pmd_size);
831 static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops *ops)
836 p = ops->setup_area(1);
842 ops->fault(BASE_ADDR, 0, hpage_pmd_size);
843 if (!ops->check_huge(BASE_ADDR, 1)) {
870 ops->cleanup_area(BASE_ADDR, hpage_pmd_size);
871 ops->fault(p, 0, hpage_pmd_size);
872 if (!ops->check_huge(p, 1))
878 ops, true);
881 ops->cleanup_area(p, hpage_pmd_size);
884 static void collapse_fork(struct collapse_context *c, struct mem_ops *ops)
889 p = ops->setup_area(1);
892 ops->fault(p, 0, page_size);
893 if (ops->check_huge(p, 0))
904 if (ops->check_huge(p, 0))
909 ops->fault(p, page_size, 2 * page_size);
911 p, 1, ops, true);
914 ops->cleanup_area(p, hpage_pmd_size);
922 if (ops->check_huge(p, 0))
927 ops->cleanup_area(p, hpage_pmd_size);
930 static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *ops)
935 p = alloc_hpage(ops);
942 if (ops->check_huge(p, 1))
950 if (ops->check_huge(p, 0))
954 ops->fault(p, 0, page_size);
958 p, 1, ops, true);
963 ops->cleanup_area(p, hpage_pmd_size);
971 if (ops->check_huge(p, 1))
976 ops->cleanup_area(p, hpage_pmd_size);
979 static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops *ops)
985 p = alloc_hpage(ops);
992 if (ops->check_huge(p, 1))
999 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size);
1000 if (ops->check_huge(p, 0))
1006 1, ops, !c->enforce_pte_scan_limits);
1011 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) *
1013 if (ops->check_huge(p, 0))
1019 p, 1, ops, true);
1023 ops->cleanup_area(p, hpage_pmd_size);
1031 if (ops->check_huge(p, 1))
1036 ops->cleanup_area(p, hpage_pmd_size);
1040 struct mem_ops *ops)
1044 p = ops->setup_area(1);
1045 ops->fault(p, 0, hpage_pmd_size);
1046 c->collapse("Collapse fully populated PTE table...", p, 1, ops, true);
1050 __madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true);
1052 ops->cleanup_area(p, hpage_pmd_size);
1060 struct mem_ops *ops)
1066 p = ops->setup_area(nr_hpages);
1067 ops->fault(p, 0, size);
1071 ops)) {
1076 c->collapse("Install huge PMD from page cache", p, nr_hpages, ops,
1079 ops->cleanup_area(p, size);