1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Page table allocation functions
4 *
5 * Copyright IBM Corp. 2016
6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
7 */
8
9 #include <linux/sysctl.h>
10 #include <linux/slab.h>
11 #include <linux/mm.h>
12 #include <asm/mmu_context.h>
13 #include <asm/page-states.h>
14 #include <asm/pgalloc.h>
15 #include <asm/tlbflush.h>
16
crst_table_alloc_noprof(struct mm_struct * mm)17 unsigned long *crst_table_alloc_noprof(struct mm_struct *mm)
18 {
19 gfp_t gfp = GFP_KERNEL_ACCOUNT;
20 struct ptdesc *ptdesc;
21 unsigned long *table;
22
23 if (mm == &init_mm)
24 gfp &= ~__GFP_ACCOUNT;
25 ptdesc = pagetable_alloc_noprof(gfp, CRST_ALLOC_ORDER);
26 if (!ptdesc)
27 return NULL;
28 table = ptdesc_address(ptdesc);
29 __arch_set_page_dat(table, 1UL << CRST_ALLOC_ORDER);
30 return table;
31 }
32
crst_table_free(struct mm_struct * mm,unsigned long * table)33 void crst_table_free(struct mm_struct *mm, unsigned long *table)
34 {
35 if (!table)
36 return;
37 pagetable_free(virt_to_ptdesc(table));
38 }
39
__crst_table_upgrade(void * arg)40 static void __crst_table_upgrade(void *arg)
41 {
42 struct mm_struct *mm = arg;
43 struct ctlreg asce;
44
45 /* change all active ASCEs to avoid the creation of new TLBs */
46 if (current->active_mm == mm) {
47 asce.val = mm->context.asce;
48 get_lowcore()->user_asce = asce;
49 local_ctl_load(7, &asce);
50 if (!test_thread_flag(TIF_ASCE_PRIMARY))
51 local_ctl_load(1, &asce);
52 }
53 __tlb_flush_local();
54 }
55
crst_table_upgrade(struct mm_struct * mm,unsigned long end)56 int crst_table_upgrade(struct mm_struct *mm, unsigned long end)
57 {
58 unsigned long *table, *pgd;
59 int rc, notify;
60
61 mmap_assert_write_locked(mm);
62 /* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */
63 VM_BUG_ON(mm->context.asce_limit < _REGION2_SIZE);
64 rc = 0;
65 notify = 0;
66 while (mm->context.asce_limit < end) {
67 table = crst_table_alloc(mm);
68 if (!table) {
69 rc = -ENOMEM;
70 break;
71 }
72 spin_lock_bh(&mm->page_table_lock);
73 pgd = (unsigned long *)mm->pgd;
74 if (mm->context.asce_limit == _REGION2_SIZE) {
75 crst_table_init(table, _REGION2_ENTRY_EMPTY);
76 p4d_populate(mm, (p4d_t *)table, (pud_t *)pgd);
77 pagetable_p4d_ctor(virt_to_ptdesc(table));
78 mm->pgd = (pgd_t *)table;
79 mm->context.asce_limit = _REGION1_SIZE;
80 mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
81 _ASCE_USER_BITS | _ASCE_TYPE_REGION2;
82 mm_inc_nr_puds(mm);
83 } else {
84 crst_table_init(table, _REGION1_ENTRY_EMPTY);
85 pgd_populate(mm, (pgd_t *)table, (p4d_t *)pgd);
86 pagetable_pgd_ctor(virt_to_ptdesc(table));
87 mm->pgd = (pgd_t *)table;
88 mm->context.asce_limit = TASK_SIZE_MAX;
89 mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |
90 _ASCE_USER_BITS | _ASCE_TYPE_REGION1;
91 }
92 notify = 1;
93 spin_unlock_bh(&mm->page_table_lock);
94 }
95 if (notify)
96 on_each_cpu(__crst_table_upgrade, mm, 0);
97 return rc;
98 }
99
page_table_alloc_noprof(struct mm_struct * mm)100 unsigned long *page_table_alloc_noprof(struct mm_struct *mm)
101 {
102 gfp_t gfp = GFP_KERNEL_ACCOUNT;
103 struct ptdesc *ptdesc;
104 unsigned long *table;
105
106 if (mm == &init_mm)
107 gfp &= ~__GFP_ACCOUNT;
108 ptdesc = pagetable_alloc_noprof(gfp, 0);
109 if (!ptdesc)
110 return NULL;
111 if (!pagetable_pte_ctor(mm, ptdesc)) {
112 pagetable_free(ptdesc);
113 return NULL;
114 }
115 table = ptdesc_address(ptdesc);
116 __arch_set_page_dat(table, 1);
117 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);
118 memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
119 return table;
120 }
121
page_table_free(struct mm_struct * mm,unsigned long * table)122 void page_table_free(struct mm_struct *mm, unsigned long *table)
123 {
124 struct ptdesc *ptdesc = virt_to_ptdesc(table);
125
126 if (pagetable_is_reserved(ptdesc))
127 return free_reserved_ptdesc(ptdesc);
128 pagetable_dtor_free(ptdesc);
129 }
130
131 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
pte_free_now(struct rcu_head * head)132 static void pte_free_now(struct rcu_head *head)
133 {
134 struct ptdesc *ptdesc = container_of(head, struct ptdesc, pt_rcu_head);
135
136 pagetable_dtor_free(ptdesc);
137 }
138
pte_free_defer(struct mm_struct * mm,pgtable_t pgtable)139 void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable)
140 {
141 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable);
142
143 call_rcu(&ptdesc->pt_rcu_head, pte_free_now);
144 }
145 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
146
147 /*
148 * Base infrastructure required to generate basic asces, region, segment,
149 * and page tables that do not make use of enhanced features like EDAT1.
150 */
151
152 static struct kmem_cache *base_pgt_cache;
153
base_pgt_alloc(void)154 static unsigned long *base_pgt_alloc(void)
155 {
156 unsigned long *table;
157
158 table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL);
159 if (table)
160 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);
161 return table;
162 }
163
base_pgt_free(unsigned long * table)164 static void base_pgt_free(unsigned long *table)
165 {
166 kmem_cache_free(base_pgt_cache, table);
167 }
168
base_crst_alloc(unsigned long val)169 static unsigned long *base_crst_alloc(unsigned long val)
170 {
171 unsigned long *table;
172 struct ptdesc *ptdesc;
173
174 ptdesc = pagetable_alloc(GFP_KERNEL, CRST_ALLOC_ORDER);
175 if (!ptdesc)
176 return NULL;
177 table = ptdesc_address(ptdesc);
178 crst_table_init(table, val);
179 return table;
180 }
181
base_crst_free(unsigned long * table)182 static void base_crst_free(unsigned long *table)
183 {
184 if (!table)
185 return;
186 pagetable_free(virt_to_ptdesc(table));
187 }
188
189 #define BASE_ADDR_END_FUNC(NAME, SIZE) \
190 static inline unsigned long base_##NAME##_addr_end(unsigned long addr, \
191 unsigned long end) \
192 { \
193 unsigned long next = (addr + (SIZE)) & ~((SIZE) - 1); \
194 \
195 return (next - 1) < (end - 1) ? next : end; \
196 }
197
BASE_ADDR_END_FUNC(page,PAGE_SIZE)198 BASE_ADDR_END_FUNC(page, PAGE_SIZE)
199 BASE_ADDR_END_FUNC(segment, _SEGMENT_SIZE)
200 BASE_ADDR_END_FUNC(region3, _REGION3_SIZE)
201 BASE_ADDR_END_FUNC(region2, _REGION2_SIZE)
202 BASE_ADDR_END_FUNC(region1, _REGION1_SIZE)
203
204 static inline unsigned long base_lra(unsigned long address)
205 {
206 unsigned long real;
207
208 asm volatile(
209 " lra %0,0(%1)"
210 : "=d" (real) : "a" (address) : "cc");
211 return real;
212 }
213
base_page_walk(unsigned long * origin,unsigned long addr,unsigned long end,int alloc)214 static int base_page_walk(unsigned long *origin, unsigned long addr,
215 unsigned long end, int alloc)
216 {
217 unsigned long *pte, next;
218
219 if (!alloc)
220 return 0;
221 pte = origin;
222 pte += (addr & _PAGE_INDEX) >> PAGE_SHIFT;
223 do {
224 next = base_page_addr_end(addr, end);
225 *pte = base_lra(addr);
226 } while (pte++, addr = next, addr < end);
227 return 0;
228 }
229
base_segment_walk(unsigned long * origin,unsigned long addr,unsigned long end,int alloc)230 static int base_segment_walk(unsigned long *origin, unsigned long addr,
231 unsigned long end, int alloc)
232 {
233 unsigned long *ste, next, *table;
234 int rc;
235
236 ste = origin;
237 ste += (addr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
238 do {
239 next = base_segment_addr_end(addr, end);
240 if (*ste & _SEGMENT_ENTRY_INVALID) {
241 if (!alloc)
242 continue;
243 table = base_pgt_alloc();
244 if (!table)
245 return -ENOMEM;
246 *ste = __pa(table) | _SEGMENT_ENTRY;
247 }
248 table = __va(*ste & _SEGMENT_ENTRY_ORIGIN);
249 rc = base_page_walk(table, addr, next, alloc);
250 if (rc)
251 return rc;
252 if (!alloc)
253 base_pgt_free(table);
254 } while (ste++, addr = next, addr < end);
255 return 0;
256 }
257
base_region3_walk(unsigned long * origin,unsigned long addr,unsigned long end,int alloc)258 static int base_region3_walk(unsigned long *origin, unsigned long addr,
259 unsigned long end, int alloc)
260 {
261 unsigned long *rtte, next, *table;
262 int rc;
263
264 rtte = origin;
265 rtte += (addr & _REGION3_INDEX) >> _REGION3_SHIFT;
266 do {
267 next = base_region3_addr_end(addr, end);
268 if (*rtte & _REGION_ENTRY_INVALID) {
269 if (!alloc)
270 continue;
271 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);
272 if (!table)
273 return -ENOMEM;
274 *rtte = __pa(table) | _REGION3_ENTRY;
275 }
276 table = __va(*rtte & _REGION_ENTRY_ORIGIN);
277 rc = base_segment_walk(table, addr, next, alloc);
278 if (rc)
279 return rc;
280 if (!alloc)
281 base_crst_free(table);
282 } while (rtte++, addr = next, addr < end);
283 return 0;
284 }
285
base_region2_walk(unsigned long * origin,unsigned long addr,unsigned long end,int alloc)286 static int base_region2_walk(unsigned long *origin, unsigned long addr,
287 unsigned long end, int alloc)
288 {
289 unsigned long *rste, next, *table;
290 int rc;
291
292 rste = origin;
293 rste += (addr & _REGION2_INDEX) >> _REGION2_SHIFT;
294 do {
295 next = base_region2_addr_end(addr, end);
296 if (*rste & _REGION_ENTRY_INVALID) {
297 if (!alloc)
298 continue;
299 table = base_crst_alloc(_REGION3_ENTRY_EMPTY);
300 if (!table)
301 return -ENOMEM;
302 *rste = __pa(table) | _REGION2_ENTRY;
303 }
304 table = __va(*rste & _REGION_ENTRY_ORIGIN);
305 rc = base_region3_walk(table, addr, next, alloc);
306 if (rc)
307 return rc;
308 if (!alloc)
309 base_crst_free(table);
310 } while (rste++, addr = next, addr < end);
311 return 0;
312 }
313
base_region1_walk(unsigned long * origin,unsigned long addr,unsigned long end,int alloc)314 static int base_region1_walk(unsigned long *origin, unsigned long addr,
315 unsigned long end, int alloc)
316 {
317 unsigned long *rfte, next, *table;
318 int rc;
319
320 rfte = origin;
321 rfte += (addr & _REGION1_INDEX) >> _REGION1_SHIFT;
322 do {
323 next = base_region1_addr_end(addr, end);
324 if (*rfte & _REGION_ENTRY_INVALID) {
325 if (!alloc)
326 continue;
327 table = base_crst_alloc(_REGION2_ENTRY_EMPTY);
328 if (!table)
329 return -ENOMEM;
330 *rfte = __pa(table) | _REGION1_ENTRY;
331 }
332 table = __va(*rfte & _REGION_ENTRY_ORIGIN);
333 rc = base_region2_walk(table, addr, next, alloc);
334 if (rc)
335 return rc;
336 if (!alloc)
337 base_crst_free(table);
338 } while (rfte++, addr = next, addr < end);
339 return 0;
340 }
341
342 /**
343 * base_asce_free - free asce and tables returned from base_asce_alloc()
344 * @asce: asce to be freed
345 *
346 * Frees all region, segment, and page tables that were allocated with a
347 * corresponding base_asce_alloc() call.
348 */
base_asce_free(unsigned long asce)349 void base_asce_free(unsigned long asce)
350 {
351 unsigned long *table = __va(asce & _ASCE_ORIGIN);
352
353 if (!asce)
354 return;
355 switch (asce & _ASCE_TYPE_MASK) {
356 case _ASCE_TYPE_SEGMENT:
357 base_segment_walk(table, 0, _REGION3_SIZE, 0);
358 break;
359 case _ASCE_TYPE_REGION3:
360 base_region3_walk(table, 0, _REGION2_SIZE, 0);
361 break;
362 case _ASCE_TYPE_REGION2:
363 base_region2_walk(table, 0, _REGION1_SIZE, 0);
364 break;
365 case _ASCE_TYPE_REGION1:
366 base_region1_walk(table, 0, TASK_SIZE_MAX, 0);
367 break;
368 }
369 base_crst_free(table);
370 }
371
base_pgt_cache_init(void)372 static int base_pgt_cache_init(void)
373 {
374 static DEFINE_MUTEX(base_pgt_cache_mutex);
375 unsigned long sz = _PAGE_TABLE_SIZE;
376
377 if (base_pgt_cache)
378 return 0;
379 mutex_lock(&base_pgt_cache_mutex);
380 if (!base_pgt_cache)
381 base_pgt_cache = kmem_cache_create("base_pgt", sz, sz, 0, NULL);
382 mutex_unlock(&base_pgt_cache_mutex);
383 return base_pgt_cache ? 0 : -ENOMEM;
384 }
385
386 /**
387 * base_asce_alloc - create kernel mapping without enhanced DAT features
388 * @addr: virtual start address of kernel mapping
389 * @num_pages: number of consecutive pages
390 *
391 * Generate an asce, including all required region, segment and page tables,
392 * that can be used to access the virtual kernel mapping. The difference is
393 * that the returned asce does not make use of any enhanced DAT features like
394 * e.g. large pages. This is required for some I/O functions that pass an
395 * asce, like e.g. some service call requests.
396 *
397 * Note: the returned asce may NEVER be attached to any cpu. It may only be
398 * used for I/O requests. tlb entries that might result because the
399 * asce was attached to a cpu won't be cleared.
400 */
base_asce_alloc(unsigned long addr,unsigned long num_pages)401 unsigned long base_asce_alloc(unsigned long addr, unsigned long num_pages)
402 {
403 unsigned long asce, *table, end;
404 int rc;
405
406 if (base_pgt_cache_init())
407 return 0;
408 end = addr + num_pages * PAGE_SIZE;
409 if (end <= _REGION3_SIZE) {
410 table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);
411 if (!table)
412 return 0;
413 rc = base_segment_walk(table, addr, end, 1);
414 asce = __pa(table) | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH;
415 } else if (end <= _REGION2_SIZE) {
416 table = base_crst_alloc(_REGION3_ENTRY_EMPTY);
417 if (!table)
418 return 0;
419 rc = base_region3_walk(table, addr, end, 1);
420 asce = __pa(table) | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;
421 } else if (end <= _REGION1_SIZE) {
422 table = base_crst_alloc(_REGION2_ENTRY_EMPTY);
423 if (!table)
424 return 0;
425 rc = base_region2_walk(table, addr, end, 1);
426 asce = __pa(table) | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;
427 } else {
428 table = base_crst_alloc(_REGION1_ENTRY_EMPTY);
429 if (!table)
430 return 0;
431 rc = base_region1_walk(table, addr, end, 1);
432 asce = __pa(table) | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH;
433 }
434 if (rc) {
435 base_asce_free(asce);
436 asce = 0;
437 }
438 return asce;
439 }
440