xref: /linux/arch/riscv/mm/pageattr.c (revision f1f9984fdc5e37303d7180ff7a85dfecb8e57e85)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2019 SiFive
4  */
5 
6 #include <linux/pagewalk.h>
7 #include <linux/pgtable.h>
8 #include <linux/vmalloc.h>
9 #include <asm/tlbflush.h>
10 #include <asm/bitops.h>
11 #include <asm/set_memory.h>
12 
13 struct pageattr_masks {
14 	pgprot_t set_mask;
15 	pgprot_t clear_mask;
16 };
17 
set_pageattr_masks(unsigned long val,struct mm_walk * walk)18 static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk)
19 {
20 	struct pageattr_masks *masks = walk->private;
21 	unsigned long new_val = val;
22 
23 	new_val &= ~(pgprot_val(masks->clear_mask));
24 	new_val |= (pgprot_val(masks->set_mask));
25 
26 	return new_val;
27 }
28 
pageattr_p4d_entry(p4d_t * p4d,unsigned long addr,unsigned long next,struct mm_walk * walk)29 static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr,
30 			      unsigned long next, struct mm_walk *walk)
31 {
32 	p4d_t val = p4dp_get(p4d);
33 
34 	if (p4d_leaf(val)) {
35 		val = __p4d(set_pageattr_masks(p4d_val(val), walk));
36 		set_p4d(p4d, val);
37 	}
38 
39 	return 0;
40 }
41 
pageattr_pud_entry(pud_t * pud,unsigned long addr,unsigned long next,struct mm_walk * walk)42 static int pageattr_pud_entry(pud_t *pud, unsigned long addr,
43 			      unsigned long next, struct mm_walk *walk)
44 {
45 	pud_t val = pudp_get(pud);
46 
47 	if (pud_leaf(val)) {
48 		val = __pud(set_pageattr_masks(pud_val(val), walk));
49 		set_pud(pud, val);
50 	}
51 
52 	return 0;
53 }
54 
pageattr_pmd_entry(pmd_t * pmd,unsigned long addr,unsigned long next,struct mm_walk * walk)55 static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr,
56 			      unsigned long next, struct mm_walk *walk)
57 {
58 	pmd_t val = pmdp_get(pmd);
59 
60 	if (pmd_leaf(val)) {
61 		val = __pmd(set_pageattr_masks(pmd_val(val), walk));
62 		set_pmd(pmd, val);
63 	}
64 
65 	return 0;
66 }
67 
pageattr_pte_entry(pte_t * pte,unsigned long addr,unsigned long next,struct mm_walk * walk)68 static int pageattr_pte_entry(pte_t *pte, unsigned long addr,
69 			      unsigned long next, struct mm_walk *walk)
70 {
71 	pte_t val = ptep_get(pte);
72 
73 	val = __pte(set_pageattr_masks(pte_val(val), walk));
74 	set_pte(pte, val);
75 
76 	return 0;
77 }
78 
pageattr_pte_hole(unsigned long addr,unsigned long next,int depth,struct mm_walk * walk)79 static int pageattr_pte_hole(unsigned long addr, unsigned long next,
80 			     int depth, struct mm_walk *walk)
81 {
82 	/* Nothing to do here */
83 	return 0;
84 }
85 
86 static const struct mm_walk_ops pageattr_ops = {
87 	.p4d_entry = pageattr_p4d_entry,
88 	.pud_entry = pageattr_pud_entry,
89 	.pmd_entry = pageattr_pmd_entry,
90 	.pte_entry = pageattr_pte_entry,
91 	.pte_hole = pageattr_pte_hole,
92 	.walk_lock = PGWALK_RDLOCK,
93 };
94 
95 #ifdef CONFIG_64BIT
__split_linear_mapping_pmd(pud_t * pudp,unsigned long vaddr,unsigned long end)96 static int __split_linear_mapping_pmd(pud_t *pudp,
97 				      unsigned long vaddr, unsigned long end)
98 {
99 	pmd_t *pmdp;
100 	unsigned long next;
101 
102 	pmdp = pmd_offset(pudp, vaddr);
103 
104 	do {
105 		next = pmd_addr_end(vaddr, end);
106 
107 		if (next - vaddr >= PMD_SIZE &&
108 		    vaddr <= (vaddr & PMD_MASK) && end >= next)
109 			continue;
110 
111 		if (pmd_leaf(pmdp_get(pmdp))) {
112 			struct page *pte_page;
113 			unsigned long pfn = _pmd_pfn(pmdp_get(pmdp));
114 			pgprot_t prot = __pgprot(pmd_val(pmdp_get(pmdp)) & ~_PAGE_PFN_MASK);
115 			pte_t *ptep_new;
116 			int i;
117 
118 			pte_page = alloc_page(GFP_KERNEL);
119 			if (!pte_page)
120 				return -ENOMEM;
121 
122 			ptep_new = (pte_t *)page_address(pte_page);
123 			for (i = 0; i < PTRS_PER_PTE; ++i, ++ptep_new)
124 				set_pte(ptep_new, pfn_pte(pfn + i, prot));
125 
126 			smp_wmb();
127 
128 			set_pmd(pmdp, pfn_pmd(page_to_pfn(pte_page), PAGE_TABLE));
129 		}
130 	} while (pmdp++, vaddr = next, vaddr != end);
131 
132 	return 0;
133 }
134 
__split_linear_mapping_pud(p4d_t * p4dp,unsigned long vaddr,unsigned long end)135 static int __split_linear_mapping_pud(p4d_t *p4dp,
136 				      unsigned long vaddr, unsigned long end)
137 {
138 	pud_t *pudp;
139 	unsigned long next;
140 	int ret;
141 
142 	pudp = pud_offset(p4dp, vaddr);
143 
144 	do {
145 		next = pud_addr_end(vaddr, end);
146 
147 		if (next - vaddr >= PUD_SIZE &&
148 		    vaddr <= (vaddr & PUD_MASK) && end >= next)
149 			continue;
150 
151 		if (pud_leaf(pudp_get(pudp))) {
152 			struct page *pmd_page;
153 			unsigned long pfn = _pud_pfn(pudp_get(pudp));
154 			pgprot_t prot = __pgprot(pud_val(pudp_get(pudp)) & ~_PAGE_PFN_MASK);
155 			pmd_t *pmdp_new;
156 			int i;
157 
158 			pmd_page = alloc_page(GFP_KERNEL);
159 			if (!pmd_page)
160 				return -ENOMEM;
161 
162 			pmdp_new = (pmd_t *)page_address(pmd_page);
163 			for (i = 0; i < PTRS_PER_PMD; ++i, ++pmdp_new)
164 				set_pmd(pmdp_new,
165 					pfn_pmd(pfn + ((i * PMD_SIZE) >> PAGE_SHIFT), prot));
166 
167 			smp_wmb();
168 
169 			set_pud(pudp, pfn_pud(page_to_pfn(pmd_page), PAGE_TABLE));
170 		}
171 
172 		ret = __split_linear_mapping_pmd(pudp, vaddr, next);
173 		if (ret)
174 			return ret;
175 	} while (pudp++, vaddr = next, vaddr != end);
176 
177 	return 0;
178 }
179 
__split_linear_mapping_p4d(pgd_t * pgdp,unsigned long vaddr,unsigned long end)180 static int __split_linear_mapping_p4d(pgd_t *pgdp,
181 				      unsigned long vaddr, unsigned long end)
182 {
183 	p4d_t *p4dp;
184 	unsigned long next;
185 	int ret;
186 
187 	p4dp = p4d_offset(pgdp, vaddr);
188 
189 	do {
190 		next = p4d_addr_end(vaddr, end);
191 
192 		/*
193 		 * If [vaddr; end] contains [vaddr & P4D_MASK; next], we don't
194 		 * need to split, we'll change the protections on the whole P4D.
195 		 */
196 		if (next - vaddr >= P4D_SIZE &&
197 		    vaddr <= (vaddr & P4D_MASK) && end >= next)
198 			continue;
199 
200 		if (p4d_leaf(p4dp_get(p4dp))) {
201 			struct page *pud_page;
202 			unsigned long pfn = _p4d_pfn(p4dp_get(p4dp));
203 			pgprot_t prot = __pgprot(p4d_val(p4dp_get(p4dp)) & ~_PAGE_PFN_MASK);
204 			pud_t *pudp_new;
205 			int i;
206 
207 			pud_page = alloc_page(GFP_KERNEL);
208 			if (!pud_page)
209 				return -ENOMEM;
210 
211 			/*
212 			 * Fill the pud level with leaf puds that have the same
213 			 * protections as the leaf p4d.
214 			 */
215 			pudp_new = (pud_t *)page_address(pud_page);
216 			for (i = 0; i < PTRS_PER_PUD; ++i, ++pudp_new)
217 				set_pud(pudp_new,
218 					pfn_pud(pfn + ((i * PUD_SIZE) >> PAGE_SHIFT), prot));
219 
220 			/*
221 			 * Make sure the pud filling is not reordered with the
222 			 * p4d store which could result in seeing a partially
223 			 * filled pud level.
224 			 */
225 			smp_wmb();
226 
227 			set_p4d(p4dp, pfn_p4d(page_to_pfn(pud_page), PAGE_TABLE));
228 		}
229 
230 		ret = __split_linear_mapping_pud(p4dp, vaddr, next);
231 		if (ret)
232 			return ret;
233 	} while (p4dp++, vaddr = next, vaddr != end);
234 
235 	return 0;
236 }
237 
__split_linear_mapping_pgd(pgd_t * pgdp,unsigned long vaddr,unsigned long end)238 static int __split_linear_mapping_pgd(pgd_t *pgdp,
239 				      unsigned long vaddr,
240 				      unsigned long end)
241 {
242 	unsigned long next;
243 	int ret;
244 
245 	do {
246 		next = pgd_addr_end(vaddr, end);
247 		/* We never use PGD mappings for the linear mapping */
248 		ret = __split_linear_mapping_p4d(pgdp, vaddr, next);
249 		if (ret)
250 			return ret;
251 	} while (pgdp++, vaddr = next, vaddr != end);
252 
253 	return 0;
254 }
255 
split_linear_mapping(unsigned long start,unsigned long end)256 static int split_linear_mapping(unsigned long start, unsigned long end)
257 {
258 	return __split_linear_mapping_pgd(pgd_offset_k(start), start, end);
259 }
260 #endif	/* CONFIG_64BIT */
261 
__set_memory(unsigned long addr,int numpages,pgprot_t set_mask,pgprot_t clear_mask)262 static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,
263 			pgprot_t clear_mask)
264 {
265 	int ret;
266 	unsigned long start = addr;
267 	unsigned long end = start + PAGE_SIZE * numpages;
268 	unsigned long __maybe_unused lm_start;
269 	unsigned long __maybe_unused lm_end;
270 	struct pageattr_masks masks = {
271 		.set_mask = set_mask,
272 		.clear_mask = clear_mask
273 	};
274 
275 	if (!numpages)
276 		return 0;
277 
278 	mmap_write_lock(&init_mm);
279 
280 #ifdef CONFIG_64BIT
281 	/*
282 	 * We are about to change the permissions of a kernel mapping, we must
283 	 * apply the same changes to its linear mapping alias, which may imply
284 	 * splitting a huge mapping.
285 	 */
286 
287 	if (is_vmalloc_or_module_addr((void *)start)) {
288 		struct vm_struct *area = NULL;
289 		int i, page_start;
290 
291 		area = find_vm_area((void *)start);
292 		page_start = (start - (unsigned long)area->addr) >> PAGE_SHIFT;
293 
294 		for (i = page_start; i < page_start + numpages; ++i) {
295 			lm_start = (unsigned long)page_address(area->pages[i]);
296 			lm_end = lm_start + PAGE_SIZE;
297 
298 			ret = split_linear_mapping(lm_start, lm_end);
299 			if (ret)
300 				goto unlock;
301 
302 			ret = walk_page_range_novma(&init_mm, lm_start, lm_end,
303 						    &pageattr_ops, NULL, &masks);
304 			if (ret)
305 				goto unlock;
306 		}
307 	} else if (is_kernel_mapping(start) || is_linear_mapping(start)) {
308 		if (is_kernel_mapping(start)) {
309 			lm_start = (unsigned long)lm_alias(start);
310 			lm_end = (unsigned long)lm_alias(end);
311 		} else {
312 			lm_start = start;
313 			lm_end = end;
314 		}
315 
316 		ret = split_linear_mapping(lm_start, lm_end);
317 		if (ret)
318 			goto unlock;
319 
320 		ret = walk_page_range_novma(&init_mm, lm_start, lm_end,
321 					    &pageattr_ops, NULL, &masks);
322 		if (ret)
323 			goto unlock;
324 	}
325 
326 	ret =  walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL,
327 				     &masks);
328 
329 unlock:
330 	mmap_write_unlock(&init_mm);
331 
332 	/*
333 	 * We can't use flush_tlb_kernel_range() here as we may have split a
334 	 * hugepage that is larger than that, so let's flush everything.
335 	 */
336 	flush_tlb_all();
337 #else
338 	ret =  walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL,
339 				     &masks);
340 
341 	mmap_write_unlock(&init_mm);
342 
343 	flush_tlb_kernel_range(start, end);
344 #endif
345 
346 	return ret;
347 }
348 
set_memory_rw_nx(unsigned long addr,int numpages)349 int set_memory_rw_nx(unsigned long addr, int numpages)
350 {
351 	return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
352 			    __pgprot(_PAGE_EXEC));
353 }
354 
set_memory_ro(unsigned long addr,int numpages)355 int set_memory_ro(unsigned long addr, int numpages)
356 {
357 	return __set_memory(addr, numpages, __pgprot(_PAGE_READ),
358 			    __pgprot(_PAGE_WRITE));
359 }
360 
set_memory_rw(unsigned long addr,int numpages)361 int set_memory_rw(unsigned long addr, int numpages)
362 {
363 	return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
364 			    __pgprot(0));
365 }
366 
set_memory_x(unsigned long addr,int numpages)367 int set_memory_x(unsigned long addr, int numpages)
368 {
369 	return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0));
370 }
371 
set_memory_nx(unsigned long addr,int numpages)372 int set_memory_nx(unsigned long addr, int numpages)
373 {
374 	return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC));
375 }
376 
set_direct_map_invalid_noflush(struct page * page)377 int set_direct_map_invalid_noflush(struct page *page)
378 {
379 	return __set_memory((unsigned long)page_address(page), 1,
380 			    __pgprot(0), __pgprot(_PAGE_PRESENT));
381 }
382 
set_direct_map_default_noflush(struct page * page)383 int set_direct_map_default_noflush(struct page *page)
384 {
385 	return __set_memory((unsigned long)page_address(page), 1,
386 			    PAGE_KERNEL, __pgprot(_PAGE_EXEC));
387 }
388 
389 #ifdef CONFIG_DEBUG_PAGEALLOC
debug_pagealloc_set_page(pte_t * pte,unsigned long addr,void * data)390 static int debug_pagealloc_set_page(pte_t *pte, unsigned long addr, void *data)
391 {
392 	int enable = *(int *)data;
393 
394 	unsigned long val = pte_val(ptep_get(pte));
395 
396 	if (enable)
397 		val |= _PAGE_PRESENT;
398 	else
399 		val &= ~_PAGE_PRESENT;
400 
401 	set_pte(pte, __pte(val));
402 
403 	return 0;
404 }
405 
__kernel_map_pages(struct page * page,int numpages,int enable)406 void __kernel_map_pages(struct page *page, int numpages, int enable)
407 {
408 	if (!debug_pagealloc_enabled())
409 		return;
410 
411 	unsigned long start = (unsigned long)page_address(page);
412 	unsigned long size = PAGE_SIZE * numpages;
413 
414 	apply_to_existing_page_range(&init_mm, start, size, debug_pagealloc_set_page, &enable);
415 
416 	flush_tlb_kernel_range(start, start + size);
417 }
418 #endif
419 
kernel_page_present(struct page * page)420 bool kernel_page_present(struct page *page)
421 {
422 	unsigned long addr = (unsigned long)page_address(page);
423 	pgd_t *pgd;
424 	pud_t *pud;
425 	p4d_t *p4d;
426 	pmd_t *pmd;
427 	pte_t *pte;
428 
429 	pgd = pgd_offset_k(addr);
430 	if (!pgd_present(pgdp_get(pgd)))
431 		return false;
432 	if (pgd_leaf(pgdp_get(pgd)))
433 		return true;
434 
435 	p4d = p4d_offset(pgd, addr);
436 	if (!p4d_present(p4dp_get(p4d)))
437 		return false;
438 	if (p4d_leaf(p4dp_get(p4d)))
439 		return true;
440 
441 	pud = pud_offset(p4d, addr);
442 	if (!pud_present(pudp_get(pud)))
443 		return false;
444 	if (pud_leaf(pudp_get(pud)))
445 		return true;
446 
447 	pmd = pmd_offset(pud, addr);
448 	if (!pmd_present(pmdp_get(pmd)))
449 		return false;
450 	if (pmd_leaf(pmdp_get(pmd)))
451 		return true;
452 
453 	pte = pte_offset_kernel(pmd, addr);
454 	return pte_present(ptep_get(pte));
455 }
456