Lines Matching full:pte
82 #include <machine/pte.h>
330 pt_entry_t *pte; in pmap_l3e_to_pte() local
334 pte = (pt_entry_t *)PHYS_TO_DMAP(ptepa); in pmap_l3e_to_pte()
335 return (&pte[pmap_pte_index(va)]); in pmap_l3e_to_pte()
841 #define pte_load_store(ptep, pte) atomic_swap_long(ptep, pte) argument
843 #define pte_store(ptep, pte) do { \ argument
844 MPASS((pte) & (RPTE_EAA_R | RPTE_EAA_W | RPTE_EAA_X)); \
845 *(u_long *)(ptep) = htobe64((u_long)((pte) | PG_V | RPTE_LEAF)); \
862 * (PTE) page mappings have identical settings for the following fields:
919 pt_entry_t *pte; in mmu_radix_kenter() local
921 pte = kvtopte(va); in mmu_radix_kenter()
922 MPASS(pte != NULL); in mmu_radix_kenter()
923 *pte = htobe64(pa | RPTE_VALID | RPTE_LEAF | RPTE_EAA_R | \ in mmu_radix_kenter()
937 pt_entry_t *pte; in pmap_nofault_pte() local
950 pte = pmap_l3e_to_pte(l3e, va); in pmap_nofault_pte()
951 if (pte == NULL || (be64toh(*pte) & PG_V) == 0) in pmap_nofault_pte()
953 return (pte); in pmap_nofault_pte()
959 pt_entry_t *pte; in pmap_nofault() local
966 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL) in pmap_nofault()
968 origpte = newpte = be64toh(*pte); in pmap_nofault()
988 if ((pte = pmap_nofault_pte(pmap, va, &is_l3e)) == NULL || in pmap_nofault()
989 be64toh(*pte) != origpte) { in pmap_nofault()
1018 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) in pmap_nofault()
1052 * Determine the appropriate bits to set in a PTE or PDE for a specified
1381 pt_entry_t *pte, tpte; in reclaim_pv_chunk() local
1460 pte = pmap_l3e_to_pte(l3e, va); in reclaim_pv_chunk()
1461 if ((be64toh(*pte) & PG_W) != 0) in reclaim_pv_chunk()
1463 tpte = be64toh(pte_load_clear(pte)); in reclaim_pv_chunk()
1784 pt_entry_t *pte, pteval; in mmu_radix_dmap_range() local
1791 pte = pmap_pml1e(kernel_pmap, PHYS_TO_DMAP(start)); in mmu_radix_dmap_range()
1792 if ((be64toh(*pte) & RPTE_VALID) == 0) { in mmu_radix_dmap_range()
1794 pde_store(pte, page); in mmu_radix_dmap_range()
1796 pte = pmap_l1e_to_l2e(pte, PHYS_TO_DMAP(start)); in mmu_radix_dmap_range()
1801 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { in mmu_radix_dmap_range()
1803 pde_store(pte, page); in mmu_radix_dmap_range()
1806 pte = pmap_l2e_to_l3e(pte, PHYS_TO_DMAP(start)); in mmu_radix_dmap_range()
1811 } else if ((be64toh(*pte) & RPTE_VALID) == 0) { in mmu_radix_dmap_range()
1813 pde_store(pte, page); in mmu_radix_dmap_range()
1815 pte = pmap_l3e_to_pte(pte, PHYS_TO_DMAP(start)); in mmu_radix_dmap_range()
1818 pte_store(pte, pteval); in mmu_radix_dmap_range()
1842 pt_entry_t *pte; in mmu_radix_setup_pagetables() local
1865 pte = pmap_pml1e(kernel_pmap, VM_MIN_KERNEL_ADDRESS); in mmu_radix_setup_pagetables()
1866 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); in mmu_radix_setup_pagetables()
1868 pte = pmap_l1e_to_l2e(pte, VM_MIN_KERNEL_ADDRESS); in mmu_radix_setup_pagetables()
1869 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); in mmu_radix_setup_pagetables()
1871 pte = pmap_l2e_to_l3e(pte, VM_MIN_KERNEL_ADDRESS); in mmu_radix_setup_pagetables()
1884 for (int i = 0; i < nkpt; i++, pte++, pages += PAGE_SIZE) in mmu_radix_setup_pagetables()
1885 *pte = htobe64(pages | RPTE_VALID | RPTE_SHIFT); in mmu_radix_setup_pagetables()
2225 pt_entry_t *pte; in mmu_radix_advise() local
2286 pte = pmap_l3e_to_pte(l3e, va); in mmu_radix_advise()
2287 KASSERT((be64toh(*pte) & PG_V) != 0, in mmu_radix_advise()
2288 ("pmap_advise: invalid PTE")); in mmu_radix_advise()
2289 pmap_remove_pte(pmap, pte, va, be64toh(*l3e), NULL, in mmu_radix_advise()
2299 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; in mmu_radix_advise()
2300 pte++, sva += PAGE_SIZE) { in mmu_radix_advise()
2301 MPASS(pte == pmap_pte(pmap, sva)); in mmu_radix_advise()
2303 if ((be64toh(*pte) & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) in mmu_radix_advise()
2305 else if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { in mmu_radix_advise()
2312 m = PHYS_TO_VM_PAGE(be64toh(*pte) & PG_FRAME); in mmu_radix_advise()
2315 atomic_clear_long(pte, htobe64(PG_M | PG_A)); in mmu_radix_advise()
2316 } else if ((be64toh(*pte) & PG_A) != 0) in mmu_radix_advise()
2317 atomic_clear_long(pte, htobe64(PG_A)); in mmu_radix_advise()
2431 pt_entry_t oldpte, *pte; in mmu_radix_clear_modify() local
2478 pte = pmap_l3e_to_pte(l3e, va); in mmu_radix_clear_modify()
2479 oldpte = be64toh(*pte); in mmu_radix_clear_modify()
2480 while (!atomic_cmpset_long(pte, in mmu_radix_clear_modify()
2483 oldpte = be64toh(*pte); in mmu_radix_clear_modify()
2505 pte = pmap_l3e_to_pte(l3e, pv->pv_va); in mmu_radix_clear_modify()
2506 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) { in mmu_radix_clear_modify()
2507 atomic_clear_long(pte, htobe64(PG_M)); in mmu_radix_clear_modify()
2727 pt_entry_t *firstpte, oldpte, pa, *pte; in pmap_promote_l3e() local
2733 * Examine the first PTE in the specified PTP. Abort if this PTE is in pmap_promote_l3e()
2757 * PTE maps an unexpected 4KB physical page or does not have identical in pmap_promote_l3e()
2758 * characteristics to the first PTE. in pmap_promote_l3e()
2761 for (pte = firstpte + NPTEPG - 1; pte > firstpte; pte--) { in pmap_promote_l3e()
2763 oldpte = be64toh(*pte); in pmap_promote_l3e()
2774 if (!atomic_cmpset_long(pte, htobe64(oldpte), htobe64((oldpte | RPTE_EAA_R) & ~RPTE_EAA_W))) in pmap_promote_l3e()
2831 pt_entry_t *pte; in mmu_radix_enter() local
2902 pte = pmap_l3e_to_pte(l3e, va); in mmu_radix_enter()
2909 * Here if the pte page isn't mapped, or if it has been in mmu_radix_enter()
2926 origpte = be64toh(*pte); in mmu_radix_enter()
2978 if (!atomic_cmpset_long(pte, htobe64(origpte), htobe64(newpte))) in mmu_radix_enter()
3003 origpte = be64toh(pte_load_clear(pte)); in mmu_radix_enter()
3081 * Update the PTE. in mmu_radix_enter()
3085 origpte = be64toh(pte_load_store(pte, htobe64(newpte))); in mmu_radix_enter()
3095 * Although the PTE may still have PG_RW set, TLB in mmu_radix_enter()
3097 * the PTE no longer has PG_M set. in mmu_radix_enter()
3109 pte_store(pte, newpte); in mmu_radix_enter()
3379 pt_entry_t *pte; in mmu_radix_enter_quick_locked() local
3428 pte = (pt_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(mpte)); in mmu_radix_enter_quick_locked()
3429 pte = &pte[pmap_pte_index(va)]; in mmu_radix_enter_quick_locked()
3432 pte = pmap_pte(pmap, va); in mmu_radix_enter_quick_locked()
3434 if (be64toh(*pte)) { in mmu_radix_enter_quick_locked()
3477 pte_store(pte, pa); in mmu_radix_enter_quick_locked()
3505 pt_entry_t *pte; in mmu_radix_extract() local
3518 * be used to access the PTE because it would use the in mmu_radix_extract()
3523 pte = pmap_l3e_to_pte(l3e, va); in mmu_radix_extract()
3524 if (__predict_false(pte == NULL)) in mmu_radix_extract()
3526 pa = be64toh(*pte); in mmu_radix_extract()
3537 pt_entry_t pte; in mmu_radix_extract_and_hold() local
3550 /* Native endian PTE, do not pass to pmap functions */ in mmu_radix_extract_and_hold()
3551 pte = be64toh(*pmap_l3e_to_pte(l3ep, va)); in mmu_radix_extract_and_hold()
3552 if ((pte & PG_V) && in mmu_radix_extract_and_hold()
3553 ((pte & PG_RW) || (prot & VM_PROT_WRITE) == 0)) in mmu_radix_extract_and_hold()
3554 m = PHYS_TO_VM_PAGE(pte & PG_FRAME); in mmu_radix_extract_and_hold()
3761 pt_entry_t *pte, mask; in pmap_page_test_mappings() local
3782 pte = pmap_pte(pmap, pv->pv_va); in pmap_page_test_mappings()
3788 rv = (be64toh(*pte) & mask) == mask; in pmap_page_test_mappings()
3809 pte = pmap_pml3e(pmap, pv->pv_va); in pmap_page_test_mappings()
3815 rv = (be64toh(*pte) & mask) == mask; in pmap_page_test_mappings()
3852 pt_entry_t *pte; in mmu_radix_is_prefaultable() local
3860 pte = pmap_l3e_to_pte(l3e, addr); in mmu_radix_is_prefaultable()
3861 rv = (be64toh(*pte) & PG_V) == 0; in mmu_radix_is_prefaultable()
3904 pt_entry_t *pte; in mmu_radix_ts_referenced() local
4011 pte = pmap_l3e_to_pte(l3e, pv->pv_va); in mmu_radix_ts_referenced()
4012 if ((be64toh(*pte) & (PG_M | PG_RW)) == (PG_M | PG_RW)) in mmu_radix_ts_referenced()
4014 if ((be64toh(*pte) & PG_A) != 0) { in mmu_radix_ts_referenced()
4015 atomic_clear_long(pte, htobe64(PG_A)); in mmu_radix_ts_referenced()
4186 pt_entry_t *pte; in mmu_radix_page_wired_mappings() local
4209 pte = pmap_pte(pmap, pv->pv_va); in mmu_radix_page_wired_mappings()
4210 if ((be64toh(*pte) & PG_W) != 0) in mmu_radix_page_wired_mappings()
4230 pte = pmap_pml3e(pmap, pv->pv_va); in mmu_radix_page_wired_mappings()
4231 if ((be64toh(*pte) & PG_W) != 0) in mmu_radix_page_wired_mappings()
4364 /* Wire up a new PTE page */ in _pmap_allocpte()
4473 * Here if the pte page isn't mapped, or if it has been in pmap_allocpte()
4549 pt_entry_t *pte; in mmu_radix_protect() local
4626 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, in mmu_radix_protect()
4632 MPASS(pte == pmap_pte(pmap, sva)); in mmu_radix_protect()
4633 obits = pbits = be64toh(*pte); in mmu_radix_protect()
4650 if (!atomic_cmpset_long(pte, htobe64(obits), htobe64(pbits))) in mmu_radix_protect()
4673 pt_entry_t oldpte, pa, *pte; in mmu_radix_qenter() local
4681 pte = kvtopte(va); in mmu_radix_qenter()
4684 pte = kvtopte(va); in mmu_radix_qenter()
4685 MPASS(pte == pmap_pte(kernel_pmap, va)); in mmu_radix_qenter()
4696 if (be64toh(*pte) != pa) { in mmu_radix_qenter()
4697 oldpte |= be64toh(*pte); in mmu_radix_qenter()
4698 pte_store(pte, pa); in mmu_radix_qenter()
4701 pte++; in mmu_radix_qenter()
4714 pt_entry_t *pte; in mmu_radix_qremove() local
4720 pte = kvtopte(va); in mmu_radix_qremove()
4723 pte = kvtopte(va); in mmu_radix_qremove()
4724 pte_clear(pte); in mmu_radix_qremove()
4725 pte++; in mmu_radix_qremove()
4815 /* PTE page */ in _pmap_unwire_ptp()
4910 pt_entry_t *pte; in pmap_fill_ptp() local
4912 for (pte = firstpte; pte < firstpte + NPTEPG; pte++) { in pmap_fill_ptp()
4913 *pte = htobe64(newpte); in pmap_fill_ptp()
5110 ("pmap_remove_l3e: pte page wire count error")); in pmap_remove_l3e()
5160 pt_entry_t *pte; in pmap_remove_page() local
5167 pte = pmap_l3e_to_pte(l3e, va); in pmap_remove_page()
5168 if ((be64toh(*pte) & RPTE_VALID) == 0) { in pmap_remove_page()
5173 invalidate_all = pmap_remove_pte(pmap, pte, va, be64toh(*l3e), free, &lock); in pmap_remove_page()
5188 pt_entry_t *pte; in pmap_remove_ptes() local
5195 for (pte = pmap_l3e_to_pte(l3e, sva); sva != eva; pte++, in pmap_remove_ptes()
5197 MPASS(pte == pmap_pte(pmap, sva)); in pmap_remove_ptes()
5198 if (*pte == 0) { in pmap_remove_ptes()
5207 if (pmap_remove_pte(pmap, pte, sva, be64toh(*l3e), free, lockp)) { in pmap_remove_ptes()
5344 pt_entry_t *pte, tpte; in mmu_radix_remove_all() local
5395 pte = pmap_l3e_to_pte(l3e, pv->pv_va); in mmu_radix_remove_all()
5396 tpte = be64toh(pte_load_clear(pte)); in mmu_radix_remove_all()
5451 pt_entry_t *pte, tpte; in mmu_radix_remove_pages() local
5494 pte = pmap_pml2e(pmap, pv->pv_va); in mmu_radix_remove_pages()
5495 ptel3e = be64toh(*pte); in mmu_radix_remove_pages()
5496 pte = pmap_l2e_to_l3e(pte, pv->pv_va); in mmu_radix_remove_pages()
5497 tpte = be64toh(*pte); in mmu_radix_remove_pages()
5501 pte = (pt_entry_t *)PHYS_TO_DMAP(tpte & in mmu_radix_remove_pages()
5503 pte = &pte[pmap_pte_index(pv->pv_va)]; in mmu_radix_remove_pages()
5504 tpte = be64toh(*pte); in mmu_radix_remove_pages()
5520 panic("bad pte va %lx pte %lx", in mmu_radix_remove_pages()
5548 pte_clear(pte); in mmu_radix_remove_pages()
5580 ("pmap_remove_pages: pte page wire count error")); in mmu_radix_remove_pages()
5629 pt_entry_t oldpte, *pte; in mmu_radix_remove_write() local
5684 pte = pmap_l3e_to_pte(l3e, pv->pv_va); in mmu_radix_remove_write()
5686 oldpte = be64toh(*pte); in mmu_radix_remove_write()
5688 if (!atomic_cmpset_long(pte, htobe64(oldpte), in mmu_radix_remove_write()
5720 pt_entry_t *pte; in mmu_radix_unwire() local
5764 for (pte = pmap_l3e_to_pte(l3e, sva); sva != va_next; pte++, in mmu_radix_unwire()
5766 MPASS(pte == pmap_pte(pmap, sva)); in mmu_radix_unwire()
5767 if ((be64toh(*pte) & PG_V) == 0) in mmu_radix_unwire()
5769 if ((be64toh(*pte) & PG_W) == 0) in mmu_radix_unwire()
5770 panic("pmap_unwire: pte %#jx is missing PG_W", in mmu_radix_unwire()
5771 (uintmax_t)(be64toh(*pte))); in mmu_radix_unwire()
5778 atomic_clear_long(pte, htobe64(PG_W)); in mmu_radix_unwire()
5810 pt_entry_t pte; in mmu_radix_mincore() local
5820 pte = be64toh(*l3ep); in mmu_radix_mincore()
5826 /* Native endian PTE, do not pass to functions */ in mmu_radix_mincore()
5827 pte = be64toh(*pmap_l3e_to_pte(l3ep, addr)); in mmu_radix_mincore()
5828 pa = pte & PG_FRAME; in mmu_radix_mincore()
5832 pte = 0; in mmu_radix_mincore()
5836 if ((pte & PG_V) != 0) { in mmu_radix_mincore()
5838 if ((pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) in mmu_radix_mincore()
5840 if ((pte & PG_A) != 0) in mmu_radix_mincore()
5845 (pte & (PG_MANAGED | PG_V)) == (PG_MANAGED | PG_V)) { in mmu_radix_mincore()
6002 pmap_pte_attr(pt_entry_t *pte, uint64_t cache_bits, uint64_t mask) in pmap_pte_attr() argument
6008 * PTE, so we can just spin on updating the low 32-bits. in pmap_pte_attr()
6011 opte = be64toh(*pte); in pmap_pte_attr()
6014 } while (npte != opte && !atomic_cmpset_long(pte, htobe64(opte), htobe64(npte))); in pmap_pte_attr()
6081 /* Big-endian PTE on stack */ in mmu_radix_kextract()
6090 * be used to access the PTE because it would use the in mmu_radix_kextract()
6126 pt_entry_t *pte, pteval; in mmu_radix_kenter_attr() local
6129 pte = kvtopte(va); in mmu_radix_kenter_attr()
6130 MPASS(pte != NULL); in mmu_radix_kenter_attr()
6133 pte_store(pte, pteval | cache_bits); in mmu_radix_kenter_attr()
6139 pt_entry_t *pte; in mmu_radix_kremove() local
6143 pte = kvtopte(va); in mmu_radix_kremove()
6144 pte_clear(pte); in mmu_radix_kremove()
6225 pt_entry_t *pte; in pmap_change_attr_locked() local
6305 pte = pmap_l3e_to_pte(l3e, tmpva); in pmap_change_attr_locked()
6306 if (*pte == 0) in pmap_change_attr_locked()
6377 pte = pmap_l3e_to_pte(l3e, tmpva); in pmap_change_attr_locked()
6378 if ((be64toh(*pte) & RPTE_ATTR_MASK) != cache_bits) { in pmap_change_attr_locked()
6379 pmap_pte_attr(pte, cache_bits, in pmap_change_attr_locked()
6384 (be64toh(*pte) & PG_FRAME) < dmaplimit) { in pmap_change_attr_locked()
6387 pa_start = be64toh(*pte) & PG_FRAME; in pmap_change_attr_locked()
6389 } else if (pa_end == (be64toh(*pte) & PG_FRAME)) in pmap_change_attr_locked()
6399 pa_start = be64toh(*pte) & PG_FRAME; in pmap_change_attr_locked()
6487 pt_entry_t *pte; in pmap_pte_walk() local
6507 pte = pmap_l3e_to_pte(l3e, va); in pmap_pte_walk()
6508 db_printf(" pte %#016lx\n", be64toh(*pte)); in pmap_pte_walk()
6531 DB_SHOW_COMMAND(pte, pmap_print_pte) in DB_SHOW_COMMAND() argument
6537 db_printf("show pte addr\n"); in DB_SHOW_COMMAND()