21 	unsigned long *table;  in crst_table_alloc_noprof()  local
28 	table = ptdesc_address(ptdesc);  in crst_table_alloc_noprof()
29 	__arch_set_page_dat(table, 1UL << CRST_ALLOC_ORDER);  in crst_table_alloc_noprof()
30 	return table;  in crst_table_alloc_noprof()
33 void crst_table_free(struct mm_struct *mm, unsigned long *table)  in crst_table_free()  argument
35 	if (!table)  in crst_table_free()
37 	pagetable_free(virt_to_ptdesc(table));  in crst_table_free()
58 	unsigned long *table, *pgd;  in crst_table_upgrade()  local
67 		table = crst_table_alloc(mm);  in crst_table_upgrade()
68 		if (!table) {  in crst_table_upgrade()
75 			crst_table_init(table, _REGION2_ENTRY_EMPTY);  in crst_table_upgrade()
76 			p4d_populate(mm, (p4d_t *)table, (pud_t *)pgd);  in crst_table_upgrade()
77 			pagetable_p4d_ctor(virt_to_ptdesc(table));  in crst_table_upgrade()
78 			mm->pgd = (pgd_t *)table;  in crst_table_upgrade()
84 			crst_table_init(table, _REGION1_ENTRY_EMPTY);  in crst_table_upgrade()
85 			pgd_populate(mm, (pgd_t *)table, (p4d_t *)pgd);  in crst_table_upgrade()
86 			pagetable_pgd_ctor(virt_to_ptdesc(table));  in crst_table_upgrade()
87 			mm->pgd = (pgd_t *)table;  in crst_table_upgrade()
104 	unsigned long *table;  in page_table_alloc_noprof()  local
115 	table = ptdesc_address(ptdesc);  in page_table_alloc_noprof()
116 	__arch_set_page_dat(table, 1);  in page_table_alloc_noprof()
117 	memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in page_table_alloc_noprof()
118 	memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);  in page_table_alloc_noprof()
119 	return table;  in page_table_alloc_noprof()
122 void page_table_free(struct mm_struct *mm, unsigned long *table)  in page_table_free()  argument
124 	struct ptdesc *ptdesc = virt_to_ptdesc(table);  in page_table_free()
156 	unsigned long *table;  in base_pgt_alloc()  local
158 	table = kmem_cache_alloc(base_pgt_cache, GFP_KERNEL);  in base_pgt_alloc()
159 	if (table)  in base_pgt_alloc()
160 		memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);  in base_pgt_alloc()
161 	return table;  in base_pgt_alloc()
164 static void base_pgt_free(unsigned long *table)  in base_pgt_free()  argument
166 	kmem_cache_free(base_pgt_cache, table);  in base_pgt_free()
171 	unsigned long *table;  in base_crst_alloc()  local
177 	table = ptdesc_address(ptdesc);  in base_crst_alloc()
178 	crst_table_init(table, val);  in base_crst_alloc()
179 	return table;  in base_crst_alloc()
182 static void base_crst_free(unsigned long *table)  in base_crst_free()  argument
184 	if (!table)  in base_crst_free()
186 	pagetable_free(virt_to_ptdesc(table));  in base_crst_free()
233 	unsigned long *ste, next, *table;  in base_segment_walk()  local
243 			table = base_pgt_alloc();  in base_segment_walk()
244 			if (!table)  in base_segment_walk()
246 			*ste = __pa(table) | _SEGMENT_ENTRY;  in base_segment_walk()
248 		table = __va(*ste & _SEGMENT_ENTRY_ORIGIN);  in base_segment_walk()
249 		rc = base_page_walk(table, addr, next, alloc);  in base_segment_walk()
253 			base_pgt_free(table);  in base_segment_walk()
261 	unsigned long *rtte, next, *table;  in base_region3_walk()  local
271 			table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_region3_walk()
272 			if (!table)  in base_region3_walk()
274 			*rtte = __pa(table) | _REGION3_ENTRY;  in base_region3_walk()
276 		table = __va(*rtte & _REGION_ENTRY_ORIGIN);  in base_region3_walk()
277 		rc = base_segment_walk(table, addr, next, alloc);  in base_region3_walk()
281 			base_crst_free(table);  in base_region3_walk()
289 	unsigned long *rste, next, *table;  in base_region2_walk()  local
299 			table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_region2_walk()
300 			if (!table)  in base_region2_walk()
302 			*rste = __pa(table) | _REGION2_ENTRY;  in base_region2_walk()
304 		table = __va(*rste & _REGION_ENTRY_ORIGIN);  in base_region2_walk()
305 		rc = base_region3_walk(table, addr, next, alloc);  in base_region2_walk()
309 			base_crst_free(table);  in base_region2_walk()
317 	unsigned long *rfte, next, *table;  in base_region1_walk()  local
327 			table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_region1_walk()
328 			if (!table)  in base_region1_walk()
330 			*rfte = __pa(table) | _REGION1_ENTRY;  in base_region1_walk()
332 		table = __va(*rfte & _REGION_ENTRY_ORIGIN);  in base_region1_walk()
333 		rc = base_region2_walk(table, addr, next, alloc);  in base_region1_walk()
337 			base_crst_free(table);  in base_region1_walk()
351 	unsigned long *table = __va(asce & _ASCE_ORIGIN);  in base_asce_free()  local
357 		base_segment_walk(table, 0, _REGION3_SIZE, 0);  in base_asce_free()
360 		base_region3_walk(table, 0, _REGION2_SIZE, 0);  in base_asce_free()
363 		base_region2_walk(table, 0, _REGION1_SIZE, 0);  in base_asce_free()
366 		base_region1_walk(table, 0, TASK_SIZE_MAX, 0);  in base_asce_free()
369 	base_crst_free(table);  in base_asce_free()
403 	unsigned long asce, *table, end;  in base_asce_alloc()  local
410 		table = base_crst_alloc(_SEGMENT_ENTRY_EMPTY);  in base_asce_alloc()
411 		if (!table)  in base_asce_alloc()
413 		rc = base_segment_walk(table, addr, end, 1);  in base_asce_alloc()
414 		asce = __pa(table) | _ASCE_TYPE_SEGMENT | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
416 		table = base_crst_alloc(_REGION3_ENTRY_EMPTY);  in base_asce_alloc()
417 		if (!table)  in base_asce_alloc()
419 		rc = base_region3_walk(table, addr, end, 1);  in base_asce_alloc()
420 		asce = __pa(table) | _ASCE_TYPE_REGION3 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
422 		table = base_crst_alloc(_REGION2_ENTRY_EMPTY);  in base_asce_alloc()
423 		if (!table)  in base_asce_alloc()
425 		rc = base_region2_walk(table, addr, end, 1);  in base_asce_alloc()
426 		asce = __pa(table) | _ASCE_TYPE_REGION2 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()
428 		table = base_crst_alloc(_REGION1_ENTRY_EMPTY);  in base_asce_alloc()
429 		if (!table)  in base_asce_alloc()
431 		rc = base_region1_walk(table, addr, end, 1);  in base_asce_alloc()
432 		asce = __pa(table) | _ASCE_TYPE_REGION1 | _ASCE_TABLE_LENGTH;  in base_asce_alloc()