155 vm_offset_t kernstart;
159 static vm_offset_t data_start;
170 static vm_offset_t zero_page_va;
174 static vm_offset_t copy_page_src_va;
175 static vm_offset_t copy_page_dst_va;
185 static int mmu_booke_enter_locked(pmap_t, vm_offset_t, vm_page_t,
236 static int tlb1_iomapped(int, vm_paddr_t, vm_size_t, vm_offset_t *);
237 static vm_size_t tlb1_mapin_region(vm_offset_t, vm_paddr_t, vm_size_t, int);
246 static inline void tlb0_flush_entry(vm_offset_t);
247 static inline unsigned int tlb0_tableidx(vm_offset_t, unsigned int);
265 static vm_paddr_t pte_vatopa(pmap_t, vm_offset_t);
266 static int pte_enter(pmap_t, vm_page_t, vm_offset_t, uint32_t, bool);
267 static int pte_remove(pmap_t, vm_offset_t, uint8_t);
268 static pte_t *pte_find(pmap_t, vm_offset_t);
269 static void kernel_pte_alloc(vm_offset_t, vm_offset_t);
273 static void pv_insert(pmap_t, vm_offset_t, vm_page_t);
274 static void pv_remove(pmap_t, vm_offset_t, vm_page_t);
290 static void		mmu_booke_copy(pmap_t, pmap_t, vm_offset_t,
291     vm_size_t, vm_offset_t);
294     vm_offset_t, vm_page_t *, vm_offset_t, int);
295 static int		mmu_booke_enter(pmap_t, vm_offset_t, vm_page_t,
297 static void		mmu_booke_enter_object(pmap_t, vm_offset_t, vm_offset_t,
299 static void		mmu_booke_enter_quick(pmap_t, vm_offset_t, vm_page_t,
301 static vm_paddr_t	mmu_booke_extract(pmap_t, vm_offset_t);
302 static vm_page_t	mmu_booke_extract_and_hold(pmap_t, vm_offset_t,
306 static bool		mmu_booke_is_prefaultable(pmap_t, vm_offset_t);
309 static vm_offset_t	mmu_booke_map(vm_offset_t *, vm_paddr_t, vm_paddr_t,
311 static int		mmu_booke_mincore(pmap_t, vm_offset_t,
313 static void		mmu_booke_object_init_pt(pmap_t, vm_offset_t,
320 static void		mmu_booke_protect(pmap_t, vm_offset_t, vm_offset_t,
322 static void		mmu_booke_qenter(vm_offset_t, vm_page_t *, int);
323 static void		mmu_booke_qremove(vm_offset_t, int);
325 static void		mmu_booke_remove(pmap_t, vm_offset_t, vm_offset_t);
328 static void		mmu_booke_unwire(pmap_t, vm_offset_t, vm_offset_t);
333 static void		mmu_booke_bootstrap(vm_offset_t, vm_offset_t);
337 static vm_paddr_t	mmu_booke_kextract(vm_offset_t);
338 static void		mmu_booke_kenter(vm_offset_t, vm_paddr_t);
339 static void		mmu_booke_kenter_attr(vm_offset_t, vm_paddr_t, vm_memattr_t);
340 static void		mmu_booke_kremove(vm_offset_t);
342 static void		mmu_booke_sync_icache(pmap_t, vm_offset_t,
349 static vm_offset_t	mmu_booke_quick_enter_page(vm_page_t m);
350 static void		mmu_booke_quick_remove_page(vm_offset_t addr);
351 static int		mmu_booke_change_attr(vm_offset_t addr,
353 static int		mmu_booke_decode_kernel_ptr(vm_offset_t addr,
354     int *is_user, vm_offset_t *decoded_addr);
429 static vm_offset_t tlb1_map_base = VM_MAPDEV_BASE;
567 pv_insert(pmap_t pmap, vm_offset_t va, vm_page_t m)  in pv_insert()
593 pv_remove(pmap_t pmap, vm_offset_t va, vm_page_t m)  in pv_remove()
628 mmu_booke_bootstrap(vm_offset_t start, vm_offset_t kernelend)  in mmu_booke_bootstrap()
638 	vm_offset_t kstack0;  in mmu_booke_bootstrap()
1012 mmu_booke_extract(pmap_t pmap, vm_offset_t va)  in mmu_booke_extract()
1028 mmu_booke_kextract(vm_offset_t va)  in mmu_booke_kextract()
1100 mmu_booke_qenter(vm_offset_t sva, vm_page_t *m, int count)  in mmu_booke_qenter()
1102 	vm_offset_t va;  in mmu_booke_qenter()
1117 mmu_booke_qremove(vm_offset_t sva, int count)  in mmu_booke_qremove()
1119 	vm_offset_t va;  in mmu_booke_qremove()
1132 mmu_booke_kenter(vm_offset_t va, vm_paddr_t pa)  in mmu_booke_kenter()
1139 mmu_booke_kenter_attr(vm_offset_t va, vm_paddr_t pa, vm_memattr_t ma)  in mmu_booke_kenter_attr()
1182 mmu_booke_kremove(vm_offset_t va)  in mmu_booke_kremove()
1217 mmu_booke_decode_kernel_ptr(vm_offset_t addr, int *is_user,  in mmu_booke_decode_kernel_ptr()
1218     vm_offset_t *decoded_addr)  in mmu_booke_decode_kernel_ptr()
1261 mmu_booke_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,  in mmu_booke_enter()
1275 mmu_booke_enter_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,  in mmu_booke_enter_locked()
1458 mmu_booke_enter_object(pmap_t pmap, vm_offset_t start,  in mmu_booke_enter_object()
1459     vm_offset_t end, vm_page_t m_start, vm_prot_t prot)  in mmu_booke_enter_object()
1462 	vm_offset_t va;  in mmu_booke_enter_object()
1484 mmu_booke_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,  in mmu_booke_enter_quick()
1503 mmu_booke_remove(pmap_t pmap, vm_offset_t va, vm_offset_t endva)  in mmu_booke_remove()
1569 static vm_offset_t
1570 mmu_booke_map(vm_offset_t *virt, vm_paddr_t pa_start,  in mmu_booke_map()
1573 	vm_offset_t sva = *virt;  in mmu_booke_map()
1574 	vm_offset_t va = sva;  in mmu_booke_map()
1658     vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr)  in mmu_booke_copy()
1667 mmu_booke_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,  in mmu_booke_protect()
1670 	vm_offset_t va;  in mmu_booke_protect()
1754 mmu_booke_extract_and_hold(pmap_t pmap, vm_offset_t va,  in mmu_booke_extract_and_hold()
1833 mmu_booke_is_prefaultable(pmap_t pmap, vm_offset_t addr)  in mmu_booke_is_prefaultable()
1970 mmu_booke_unwire(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)  in mmu_booke_unwire()
1972 	vm_offset_t va;  in mmu_booke_unwire()
2049 	vm_offset_t va;  in mmu_booke_dev_direct_mapped()
2067 	vm_offset_t ofs;  in mmu_booke_dumpsys_map()
2072 		*va = (void *)(vm_offset_t)pa;  in mmu_booke_dumpsys_map()
2082 	tlb1_set_entry((vm_offset_t)va, ppa, gran, _TLB_ENTRY_IO);  in mmu_booke_dumpsys_map()
2085 		tlb1_set_entry((vm_offset_t)(va + gran), ppa + gran, gran,  in mmu_booke_dumpsys_map()
2093 	vm_offset_t ofs;  in mmu_booke_dumpsys_unmap()
2133 	vm_offset_t va;  in mmu_booke_scan_init()
2328 	vm_offset_t base, offset, va;  in mmu_booke_unmapdev()
2333 	va = (vm_offset_t)p;  in mmu_booke_unmapdev()
2350 mmu_booke_object_init_pt(pmap_t pmap, vm_offset_t addr,  in mmu_booke_object_init_pt()
2363 mmu_booke_mincore(pmap_t pmap, vm_offset_t addr, vm_paddr_t *pap)  in mmu_booke_mincore()
2371 mmu_booke_change_attr(vm_offset_t addr, vm_size_t sz, vm_memattr_t mode)  in mmu_booke_change_attr()
2373 	vm_offset_t va;  in mmu_booke_change_attr()
2504 tlb0_tableidx(vm_offset_t va, unsigned int way)  in tlb0_tableidx()
2517 tlb0_flush_entry(vm_offset_t va)  in tlb0_flush_entry()
2605 tlb1_purge_va_range(vm_offset_t va, vm_size_t size)  in tlb1_purge_va_range()
2726 tlb1_set_entry(vm_offset_t va, vm_paddr_t pa, vm_size_t size,  in tlb1_set_entry()
2771 tlb1_mapin_region(vm_offset_t va, vm_paddr_t pa, vm_size_t size, int wimge)  in tlb1_mapin_region()
2773 	vm_offset_t base;  in tlb1_mapin_region()
2836 	vm_offset_t mas2;  in tlb1_init()
2873 pmap_early_io_unmap(vm_offset_t va, vm_size_t size)  in pmap_early_io_unmap()
2895 vm_offset_t 
2899 	vm_offset_t va, sz;  in pmap_early_io_map()
2932 pmap_track_page(pmap_t pmap, vm_offset_t va)  in pmap_track_page()
2981 tlb1_iomapped(int i, vm_paddr_t pa, vm_size_t size, vm_offset_t *va)  in tlb1_iomapped()
2990 	*va = (vm_offset_t)NULL;  in tlb1_iomapped()