1df4e817bSPasha Tatashin // SPDX-License-Identifier: GPL-2.0
2df4e817bSPasha Tatashin
3df4e817bSPasha Tatashin /*
4df4e817bSPasha Tatashin * Copyright (c) 2021, Google LLC.
5df4e817bSPasha Tatashin * Pasha Tatashin <pasha.tatashin@soleen.com>
6df4e817bSPasha Tatashin */
7f15be1b8SChristophe JAILLET #include <linux/kstrtox.h>
8df4e817bSPasha Tatashin #include <linux/mm.h>
9df4e817bSPasha Tatashin #include <linux/page_table_check.h>
10df4e817bSPasha Tatashin
11df4e817bSPasha Tatashin #undef pr_fmt
12df4e817bSPasha Tatashin #define pr_fmt(fmt) "page_table_check: " fmt
13df4e817bSPasha Tatashin
14df4e817bSPasha Tatashin struct page_table_check {
15df4e817bSPasha Tatashin atomic_t anon_map_count;
16df4e817bSPasha Tatashin atomic_t file_map_count;
17df4e817bSPasha Tatashin };
18df4e817bSPasha Tatashin
19df4e817bSPasha Tatashin static bool __page_table_check_enabled __initdata =
20df4e817bSPasha Tatashin IS_ENABLED(CONFIG_PAGE_TABLE_CHECK_ENFORCED);
21df4e817bSPasha Tatashin
22df4e817bSPasha Tatashin DEFINE_STATIC_KEY_TRUE(page_table_check_disabled);
23df4e817bSPasha Tatashin EXPORT_SYMBOL(page_table_check_disabled);
24df4e817bSPasha Tatashin
early_page_table_check_param(char * buf)25df4e817bSPasha Tatashin static int __init early_page_table_check_param(char *buf)
26df4e817bSPasha Tatashin {
27f15be1b8SChristophe JAILLET return kstrtobool(buf, &__page_table_check_enabled);
28df4e817bSPasha Tatashin }
29df4e817bSPasha Tatashin
30df4e817bSPasha Tatashin early_param("page_table_check", early_page_table_check_param);
31df4e817bSPasha Tatashin
need_page_table_check(void)32df4e817bSPasha Tatashin static bool __init need_page_table_check(void)
33df4e817bSPasha Tatashin {
34df4e817bSPasha Tatashin return __page_table_check_enabled;
35df4e817bSPasha Tatashin }
36df4e817bSPasha Tatashin
init_page_table_check(void)37df4e817bSPasha Tatashin static void __init init_page_table_check(void)
38df4e817bSPasha Tatashin {
39df4e817bSPasha Tatashin if (!__page_table_check_enabled)
40df4e817bSPasha Tatashin return;
41df4e817bSPasha Tatashin static_branch_disable(&page_table_check_disabled);
42df4e817bSPasha Tatashin }
43df4e817bSPasha Tatashin
44df4e817bSPasha Tatashin struct page_ext_operations page_table_check_ops = {
45df4e817bSPasha Tatashin .size = sizeof(struct page_table_check),
46df4e817bSPasha Tatashin .need = need_page_table_check,
47df4e817bSPasha Tatashin .init = init_page_table_check,
486189eb82SPasha Tatashin .need_shared_flags = false,
49df4e817bSPasha Tatashin };
50df4e817bSPasha Tatashin
get_page_table_check(struct page_ext * page_ext)51df4e817bSPasha Tatashin static struct page_table_check *get_page_table_check(struct page_ext *page_ext)
52df4e817bSPasha Tatashin {
53df4e817bSPasha Tatashin BUG_ON(!page_ext);
54d981e280SKemeng Shi return page_ext_data(page_ext, &page_table_check_ops);
55df4e817bSPasha Tatashin }
56df4e817bSPasha Tatashin
57df4e817bSPasha Tatashin /*
583ae6d3e3SChih-En Lin * An entry is removed from the page table, decrement the counters for that page
59df4e817bSPasha Tatashin * verify that it is of correct type and counters do not become negative.
60df4e817bSPasha Tatashin */
page_table_check_clear(unsigned long pfn,unsigned long pgcnt)6134c876ceSKemeng Shi static void page_table_check_clear(unsigned long pfn, unsigned long pgcnt)
62df4e817bSPasha Tatashin {
63df4e817bSPasha Tatashin struct page_ext *page_ext;
64df4e817bSPasha Tatashin struct page *page;
6564d8b9e1SPasha Tatashin unsigned long i;
66df4e817bSPasha Tatashin bool anon;
67df4e817bSPasha Tatashin
68df4e817bSPasha Tatashin if (!pfn_valid(pfn))
69df4e817bSPasha Tatashin return;
70df4e817bSPasha Tatashin
71df4e817bSPasha Tatashin page = pfn_to_page(pfn);
72b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page);
7344d0fb38SRuihan Li
7444d0fb38SRuihan Li BUG_ON(PageSlab(page));
75df4e817bSPasha Tatashin anon = PageAnon(page);
76df4e817bSPasha Tatashin
77df4e817bSPasha Tatashin for (i = 0; i < pgcnt; i++) {
78df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext);
79df4e817bSPasha Tatashin
80df4e817bSPasha Tatashin if (anon) {
81df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count));
82df4e817bSPasha Tatashin BUG_ON(atomic_dec_return(&ptc->anon_map_count) < 0);
83df4e817bSPasha Tatashin } else {
84df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count));
85df4e817bSPasha Tatashin BUG_ON(atomic_dec_return(&ptc->file_map_count) < 0);
86df4e817bSPasha Tatashin }
87df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext);
88df4e817bSPasha Tatashin }
89b1d5488aSCharan Teja Kalla page_ext_put(page_ext);
90df4e817bSPasha Tatashin }
91df4e817bSPasha Tatashin
92df4e817bSPasha Tatashin /*
933ae6d3e3SChih-En Lin * A new entry is added to the page table, increment the counters for that page
94df4e817bSPasha Tatashin * verify that it is of correct type and is not being mapped with a different
95df4e817bSPasha Tatashin * type to a different process.
96df4e817bSPasha Tatashin */
page_table_check_set(unsigned long pfn,unsigned long pgcnt,bool rw)972f933eafSKemeng Shi static void page_table_check_set(unsigned long pfn, unsigned long pgcnt,
98df4e817bSPasha Tatashin bool rw)
99df4e817bSPasha Tatashin {
100df4e817bSPasha Tatashin struct page_ext *page_ext;
101df4e817bSPasha Tatashin struct page *page;
10264d8b9e1SPasha Tatashin unsigned long i;
103df4e817bSPasha Tatashin bool anon;
104df4e817bSPasha Tatashin
105df4e817bSPasha Tatashin if (!pfn_valid(pfn))
106df4e817bSPasha Tatashin return;
107df4e817bSPasha Tatashin
108df4e817bSPasha Tatashin page = pfn_to_page(pfn);
109b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page);
11044d0fb38SRuihan Li
11144d0fb38SRuihan Li BUG_ON(PageSlab(page));
112df4e817bSPasha Tatashin anon = PageAnon(page);
113df4e817bSPasha Tatashin
114df4e817bSPasha Tatashin for (i = 0; i < pgcnt; i++) {
115df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext);
116df4e817bSPasha Tatashin
117df4e817bSPasha Tatashin if (anon) {
118df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count));
119df4e817bSPasha Tatashin BUG_ON(atomic_inc_return(&ptc->anon_map_count) > 1 && rw);
120df4e817bSPasha Tatashin } else {
121df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count));
122df4e817bSPasha Tatashin BUG_ON(atomic_inc_return(&ptc->file_map_count) < 0);
123df4e817bSPasha Tatashin }
124df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext);
125df4e817bSPasha Tatashin }
126b1d5488aSCharan Teja Kalla page_ext_put(page_ext);
127df4e817bSPasha Tatashin }
128df4e817bSPasha Tatashin
129df4e817bSPasha Tatashin /*
130df4e817bSPasha Tatashin * page is on free list, or is being allocated, verify that counters are zeroes
131df4e817bSPasha Tatashin * crash if they are not.
132df4e817bSPasha Tatashin */
__page_table_check_zero(struct page * page,unsigned int order)133df4e817bSPasha Tatashin void __page_table_check_zero(struct page *page, unsigned int order)
134df4e817bSPasha Tatashin {
135b1d5488aSCharan Teja Kalla struct page_ext *page_ext;
13664d8b9e1SPasha Tatashin unsigned long i;
137df4e817bSPasha Tatashin
13844d0fb38SRuihan Li BUG_ON(PageSlab(page));
13944d0fb38SRuihan Li
140b1d5488aSCharan Teja Kalla page_ext = page_ext_get(page);
141df4e817bSPasha Tatashin BUG_ON(!page_ext);
14264d8b9e1SPasha Tatashin for (i = 0; i < (1ul << order); i++) {
143df4e817bSPasha Tatashin struct page_table_check *ptc = get_page_table_check(page_ext);
144df4e817bSPasha Tatashin
145df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->anon_map_count));
146df4e817bSPasha Tatashin BUG_ON(atomic_read(&ptc->file_map_count));
147df4e817bSPasha Tatashin page_ext = page_ext_next(page_ext);
148df4e817bSPasha Tatashin }
149b1d5488aSCharan Teja Kalla page_ext_put(page_ext);
150df4e817bSPasha Tatashin }
151df4e817bSPasha Tatashin
__page_table_check_pte_clear(struct mm_struct * mm,pte_t pte)152aa232204SKemeng Shi void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
153df4e817bSPasha Tatashin {
154df4e817bSPasha Tatashin if (&init_mm == mm)
155df4e817bSPasha Tatashin return;
156df4e817bSPasha Tatashin
157df4e817bSPasha Tatashin if (pte_user_accessible_page(pte)) {
15834c876ceSKemeng Shi page_table_check_clear(pte_pfn(pte), PAGE_SIZE >> PAGE_SHIFT);
159df4e817bSPasha Tatashin }
160df4e817bSPasha Tatashin }
161df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pte_clear);
162df4e817bSPasha Tatashin
__page_table_check_pmd_clear(struct mm_struct * mm,pmd_t pmd)1631831414cSKemeng Shi void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
164df4e817bSPasha Tatashin {
165df4e817bSPasha Tatashin if (&init_mm == mm)
166df4e817bSPasha Tatashin return;
167df4e817bSPasha Tatashin
168df4e817bSPasha Tatashin if (pmd_user_accessible_page(pmd)) {
16934c876ceSKemeng Shi page_table_check_clear(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT);
170df4e817bSPasha Tatashin }
171df4e817bSPasha Tatashin }
172df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pmd_clear);
173df4e817bSPasha Tatashin
__page_table_check_pud_clear(struct mm_struct * mm,pud_t pud)174931c38e1SKemeng Shi void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
175df4e817bSPasha Tatashin {
176df4e817bSPasha Tatashin if (&init_mm == mm)
177df4e817bSPasha Tatashin return;
178df4e817bSPasha Tatashin
179df4e817bSPasha Tatashin if (pud_user_accessible_page(pud)) {
18034c876ceSKemeng Shi page_table_check_clear(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT);
181df4e817bSPasha Tatashin }
182df4e817bSPasha Tatashin }
183df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pud_clear);
184df4e817bSPasha Tatashin
__page_table_check_ptes_set(struct mm_struct * mm,pte_t * ptep,pte_t pte,unsigned int nr)185*a3793220SMatthew Wilcox (Oracle) void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
186*a3793220SMatthew Wilcox (Oracle) unsigned int nr)
187df4e817bSPasha Tatashin {
188*a3793220SMatthew Wilcox (Oracle) unsigned int i;
189*a3793220SMatthew Wilcox (Oracle)
190df4e817bSPasha Tatashin if (&init_mm == mm)
191df4e817bSPasha Tatashin return;
192df4e817bSPasha Tatashin
193*a3793220SMatthew Wilcox (Oracle) for (i = 0; i < nr; i++)
194*a3793220SMatthew Wilcox (Oracle) __page_table_check_pte_clear(mm, ptep_get(ptep + i));
195*a3793220SMatthew Wilcox (Oracle) if (pte_user_accessible_page(pte))
196*a3793220SMatthew Wilcox (Oracle) page_table_check_set(pte_pfn(pte), nr, pte_write(pte));
197df4e817bSPasha Tatashin }
198*a3793220SMatthew Wilcox (Oracle) EXPORT_SYMBOL(__page_table_check_ptes_set);
199df4e817bSPasha Tatashin
__page_table_check_pmd_set(struct mm_struct * mm,pmd_t * pmdp,pmd_t pmd)200a3b83713SKemeng Shi void __page_table_check_pmd_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd)
201df4e817bSPasha Tatashin {
202df4e817bSPasha Tatashin if (&init_mm == mm)
203df4e817bSPasha Tatashin return;
204df4e817bSPasha Tatashin
2051831414cSKemeng Shi __page_table_check_pmd_clear(mm, *pmdp);
206df4e817bSPasha Tatashin if (pmd_user_accessible_page(pmd)) {
2072f933eafSKemeng Shi page_table_check_set(pmd_pfn(pmd), PMD_SIZE >> PAGE_SHIFT,
208df4e817bSPasha Tatashin pmd_write(pmd));
209df4e817bSPasha Tatashin }
210df4e817bSPasha Tatashin }
211df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pmd_set);
212df4e817bSPasha Tatashin
__page_table_check_pud_set(struct mm_struct * mm,pud_t * pudp,pud_t pud)2136d144436SKemeng Shi void __page_table_check_pud_set(struct mm_struct *mm, pud_t *pudp, pud_t pud)
214df4e817bSPasha Tatashin {
215df4e817bSPasha Tatashin if (&init_mm == mm)
216df4e817bSPasha Tatashin return;
217df4e817bSPasha Tatashin
218931c38e1SKemeng Shi __page_table_check_pud_clear(mm, *pudp);
219df4e817bSPasha Tatashin if (pud_user_accessible_page(pud)) {
2202f933eafSKemeng Shi page_table_check_set(pud_pfn(pud), PUD_SIZE >> PAGE_SHIFT,
221df4e817bSPasha Tatashin pud_write(pud));
222df4e817bSPasha Tatashin }
223df4e817bSPasha Tatashin }
224df4e817bSPasha Tatashin EXPORT_SYMBOL(__page_table_check_pud_set);
22580110bbfSPasha Tatashin
__page_table_check_pte_clear_range(struct mm_struct * mm,unsigned long addr,pmd_t pmd)22680110bbfSPasha Tatashin void __page_table_check_pte_clear_range(struct mm_struct *mm,
22780110bbfSPasha Tatashin unsigned long addr,
22880110bbfSPasha Tatashin pmd_t pmd)
22980110bbfSPasha Tatashin {
23080110bbfSPasha Tatashin if (&init_mm == mm)
23180110bbfSPasha Tatashin return;
23280110bbfSPasha Tatashin
23380110bbfSPasha Tatashin if (!pmd_bad(pmd) && !pmd_leaf(pmd)) {
23480110bbfSPasha Tatashin pte_t *ptep = pte_offset_map(&pmd, addr);
23580110bbfSPasha Tatashin unsigned long i;
23680110bbfSPasha Tatashin
2379f2bad09SHugh Dickins if (WARN_ON(!ptep))
2389f2bad09SHugh Dickins return;
23980110bbfSPasha Tatashin for (i = 0; i < PTRS_PER_PTE; i++) {
240aa232204SKemeng Shi __page_table_check_pte_clear(mm, ptep_get(ptep));
24180110bbfSPasha Tatashin addr += PAGE_SIZE;
24280110bbfSPasha Tatashin ptep++;
24380110bbfSPasha Tatashin }
24424c8e27eSMiaohe Lin pte_unmap(ptep - PTRS_PER_PTE);
24580110bbfSPasha Tatashin }
24680110bbfSPasha Tatashin }
247