Lines Matching +full:page +full:- +full:level
1 // SPDX-License-Identifier: GPL-2.0-only
35 #include <asm/hyperv-tlfs.h>
41 * The current flushing context - we pass it instead of 5 arguments:
55 struct page **pages;
69 * entries change the page attribute in parallel to some other cpu
70 * splitting a large page entry along with changing the attribute.
87 void update_page_count(int level, unsigned long pages) in update_page_count() argument
91 direct_pages_count[level] += pages; in update_page_count()
95 static void split_page_count(int level) in split_page_count() argument
97 if (direct_pages_count[level] == 0) in split_page_count()
100 direct_pages_count[level]--; in split_page_count()
102 if (level == PG_LEVEL_2M) in split_page_count()
104 else if (level == PG_LEVEL_1G) in split_page_count()
107 direct_pages_count[level - 1] += PTRS_PER_PTE; in split_page_count()
126 static inline void split_page_count(int level) { } in split_page_count() argument
154 static inline void cpa_inc_lp_sameprot(int level) in cpa_inc_lp_sameprot() argument
156 if (level == PG_LEVEL_1G) in cpa_inc_lp_sameprot()
162 static inline void cpa_inc_lp_preserved(int level) in cpa_inc_lp_preserved() argument
164 if (level == PG_LEVEL_1G) in cpa_inc_lp_preserved()
178 seq_printf(m, "4K pages set-checked: %16lu\n", cpa_4k_install); in cpastats_show()
205 static inline void cpa_inc_lp_sameprot(int level) { } in cpa_inc_lp_sameprot() argument
206 static inline void cpa_inc_lp_preserved(int level) { } in cpa_inc_lp_preserved() argument
232 * points to #2, but almost all physical-to-virtual translations point to #1.
249 return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT; in highmap_end_pfn()
265 /* There is no highmap on 32-bit */ in __cpa_pfn_in_highmap()
278 * works fine for the code that updates the page tables. But at the end of the
279 * process we need to flush the TLB and cache and the non-canonical address
283 * will fix the top bit if needed and is a no-op otherwise.
296 if (cpa->flags & CPA_PAGES_ARRAY) { in __cpa_addr()
297 struct page *page = cpa->pages[idx]; in __cpa_addr() local
299 if (unlikely(PageHighMem(page))) in __cpa_addr()
302 return (unsigned long)page_address(page); in __cpa_addr()
305 if (cpa->flags & CPA_ARRAY) in __cpa_addr()
306 return cpa->vaddr[idx]; in __cpa_addr()
308 return *cpa->vaddr + idx * PAGE_SIZE; in __cpa_addr()
318 void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1)); in clflush_cache_range_opt()
329 * clflush_cache_range - flush a cache range with clflush
362 return -ENXIO; in cpu_cache_invalidate_memregion()
375 * large page flushing. in __cpa_flush_all()
395 for (i = 0; i < cpa->numpages; i++) in __cpa_flush_tlb()
411 if (cpa->force_flush_all || cpa->numpages > tlb_single_page_flush_ceiling) in cpa_flush()
420 for (i = 0; i < cpa->numpages; i++) { in cpa_flush()
422 unsigned int level; in cpa_flush() local
424 pte_t *pte = lookup_address(addr, &level); in cpa_flush()
448 #define BIOS_PFN_END PFN_DOWN(BIOS_END - 1)
464 * The .rodata section needs to be read-only. Using the pfn catches all
473 * Note: __end_rodata is at page aligned and not inclusive, so in protect_rodata()
476 epfn_ro = PFN_DOWN(__pa_symbol(__end_rodata)) - 1; in protect_rodata()
485 * _PAGE_NX. This protects only the high kernel mapping (_text -> _etext)
493 unsigned long t_end = (unsigned long)_etext - 1; in protect_kernel_text()
504 * kernel text mappings for the large page aligned text, rodata sections
505 * will be always read-only. For the kernel identity mappings covering the
508 * This will preserve the large page mappings for kernel text/data at no
514 unsigned long t_end = (unsigned long)__end_rodata_hpage_align - 1; in protect_kernel_text_ro()
516 unsigned int level; in protect_kernel_text_ro() local
522 * the current mapping is already using small page mapping. No in protect_kernel_text_ro()
523 * need to work hard to preserve large page mappings in this case. in protect_kernel_text_ro()
526 * by unexpected read-only mappings for kernel identity in protect_kernel_text_ro()
528 * and the kernel identity mapping share the same page-table pages, in protect_kernel_text_ro()
532 if (lookup_address(start, &level) && (level != PG_LEVEL_4K)) in protect_kernel_text_ro()
562 pr_warn("CPA %8s %10s: 0x%016lx - 0x%016lx PFN %lx req %016llx prevent %016llx\n", in check_conflict()
582 * mapping is setting the page !PRESENT. in static_protections()
588 end = start + npg * PAGE_SIZE - 1; in static_protections()
595 * Special case to preserve a large page. If the change spawns the in static_protections()
596 * full large page mapping then there is no point to split it in static_protections()
600 if (lpsize != (npg * PAGE_SIZE) || (start & (lpsize - 1))) { in static_protections()
607 res = protect_pci_bios(pfn, pfn + npg - 1); in static_protections()
611 res = protect_rodata(pfn, pfn + npg - 1); in static_protections()
628 * 32-bit has some unfixable W+X issues, like EFI code in verify_rwx()
629 * and writeable data being in the same page. Disable in verify_rwx()
645 /* Non-leaf translation entries can disable writing or execution. */ in verify_rwx()
649 end = start + npg * PAGE_SIZE - 1; in verify_rwx()
650 WARN_ONCE(1, "CPA detected W^X violation: %016llx -> %016llx range: 0x%016lx - 0x%016lx PFN %lx\n", in verify_rwx()
664 * Lookup the page table entry for a virtual address in a specific pgd.
666 * the level of the entry, and the effective NX and RW bits of all
667 * page table levels.
670 unsigned int *level, bool *nx, bool *rw) in lookup_address_in_pgd_attr() argument
676 *level = PG_LEVEL_256T; in lookup_address_in_pgd_attr()
683 *level = PG_LEVEL_512G; in lookup_address_in_pgd_attr()
694 *level = PG_LEVEL_1G; in lookup_address_in_pgd_attr()
705 *level = PG_LEVEL_2M; in lookup_address_in_pgd_attr()
716 *level = PG_LEVEL_4K; in lookup_address_in_pgd_attr()
724 * Lookup the page table entry for a virtual address in a specific pgd.
725 * Return a pointer to the entry and the level of the mapping.
728 unsigned int *level) in lookup_address_in_pgd() argument
732 return lookup_address_in_pgd_attr(pgd, address, level, &nx, &rw); in lookup_address_in_pgd()
736 * Lookup the page table entry for a virtual address. Return a pointer
737 * to the entry and the level of the mapping.
742 pte_t *lookup_address(unsigned long address, unsigned int *level) in lookup_address() argument
744 return lookup_address_in_pgd(pgd_offset_k(address), address, level); in lookup_address()
749 unsigned int *level, bool *nx, bool *rw) in _lookup_address_cpa() argument
753 if (!cpa->pgd) in _lookup_address_cpa()
756 pgd = cpa->pgd + pgd_index(address); in _lookup_address_cpa()
758 return lookup_address_in_pgd_attr(pgd, address, level, nx, rw); in _lookup_address_cpa()
789 * areas on 32-bit NUMA systems. The percpu areas can
792 * Note that as long as the PTEs are well-formed with correct PFNs, this
806 enum pg_level level; in slow_virt_to_phys() local
809 pte = lookup_address(virt_addr, &level); in slow_virt_to_phys()
814 * before being left-shifted PAGE_SHIFT bits -- this trick is to in slow_virt_to_phys()
815 * make 32-PAE kernel work correctly. in slow_virt_to_phys()
817 switch (level) { in slow_virt_to_phys()
844 struct page *page; in __set_pmd_pte() local
846 list_for_each_entry(page, &pgd_list, lru) { in __set_pmd_pte()
852 pgd = (pgd_t *)page_address(page) + pgd_index(address); in __set_pmd_pte()
865 * _PAGE_GLOBAL means "global page" for present PTEs. in pgprot_clear_protnone_bits()
867 * for non-present PTEs. in pgprot_clear_protnone_bits()
870 * present to non-present is not confused as in pgprot_clear_protnone_bits()
885 enum pg_level level; in __should_split_large_page() local
889 * Check for races, another CPU might have split this page in __should_split_large_page()
892 tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw); in __should_split_large_page()
896 switch (level) { in __should_split_large_page()
908 return -EINVAL; in __should_split_large_page()
911 psize = page_level_size(level); in __should_split_large_page()
912 pmask = page_level_mask(level); in __should_split_large_page()
916 * page starting at address: in __should_split_large_page()
919 numpages = (lpaddr - address) >> PAGE_SHIFT; in __should_split_large_page()
920 if (numpages < cpa->numpages) in __should_split_large_page()
921 cpa->numpages = numpages; in __should_split_large_page()
925 * Convert protection attributes to 4k-format, as cpa->mask* are set in __should_split_large_page()
932 pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr); in __should_split_large_page()
933 pgprot_val(req_prot) |= pgprot_val(cpa->mask_set); in __should_split_large_page()
937 * page format: the caching mode includes the PAT bit located at in __should_split_large_page()
946 * old_pfn points to the large page base pfn. So we need to add the in __should_split_large_page()
949 pfn = old_pfn + ((address & (psize - 1)) >> PAGE_SHIFT); in __should_split_large_page()
950 cpa->pfn = pfn; in __should_split_large_page()
953 * Calculate the large page base address and the number of 4K pages in __should_split_large_page()
954 * in the large page in __should_split_large_page()
969 * Split the large page and tell the split code to in __should_split_large_page()
972 cpa->force_static_prot = 1; in __should_split_large_page()
978 * pgprot, then the large page can be preserved and no updates are in __should_split_large_page()
986 cpa_inc_lp_sameprot(level); in __should_split_large_page()
991 * If the requested range does not cover the full page, split it up in __should_split_large_page()
993 if (address != lpaddr || cpa->numpages != numpages) in __should_split_large_page()
998 * protection requirement in the large page. in __should_split_large_page()
1007 * If there is a conflict, split the large page. in __should_split_large_page()
1012 * preserve one large page occasionally, but it's really not worth the in __should_split_large_page()
1018 /* All checks passed. Update the large page mapping. */ in __should_split_large_page()
1021 cpa->flags |= CPA_FLUSHTLB; in __should_split_large_page()
1022 cpa_inc_lp_preserved(level); in __should_split_large_page()
1031 if (cpa->force_split) in should_split_large_page()
1052 if (!cpa->force_static_prot) in split_set_pte()
1079 struct page *base) in __split_large_page()
1083 unsigned int i, level; in __split_large_page() local
1090 * Check for races, another CPU might have split this page in __split_large_page()
1093 tmp = _lookup_address_cpa(cpa, address, &level, &nx, &rw); in __split_large_page()
1101 switch (level) { in __split_large_page()
1147 split_page_count(level); in __split_large_page()
1160 * Do a global flush tlb after splitting the large page in __split_large_page()
1161 * and before we do the actual change page attribute in the PTE. in __split_large_page()
1164 * "The TLBs may contain both ordinary and large-page in __split_large_page()
1165 * translations for a 4-KByte range of linear addresses. This in __split_large_page()
1167 * the page size used for the address range changes. If the two in __split_large_page()
1168 * translations differ with respect to page frame or attributes in __split_large_page()
1170 * be implementation-specific." in __split_large_page()
1174 * page attribute in parallel, that also falls into the in __split_large_page()
1175 * just split large page entry. in __split_large_page()
1186 struct page *base; in split_large_page()
1194 return -ENOMEM; in split_large_page()
1257 * Not on a 2MB page boundary? in unmap_pmd_range()
1259 if (start & (PMD_SIZE - 1)) { in unmap_pmd_range()
1272 while (end - start >= PMD_SIZE) { in unmap_pmd_range()
1289 * Try again to free the PMD page if haven't succeeded above. in unmap_pmd_range()
1301 * Not on a GB page boundary? in unmap_pud_range()
1303 if (start & (PUD_SIZE - 1)) { in unmap_pud_range()
1316 while (end - start >= PUD_SIZE) { in unmap_pud_range()
1334 * No need to try to free the PUD page because we'll free it in in unmap_pud_range()
1343 return -1; in alloc_pte_page()
1353 return -1; in alloc_pmd_page()
1369 while (num_pages-- && start < end) { in populate_pte()
1370 set_pte(pte, pfn_pte(cpa->pfn, pgprot)); in populate_pte()
1373 cpa->pfn++; in populate_pte()
1389 if (start & (PMD_SIZE - 1)) { in populate_pmd()
1394 cur_pages = (pre_end - start) >> PAGE_SHIFT; in populate_pmd()
1398 * Need a PTE page? in populate_pmd()
1403 return -1; in populate_pmd()
1418 while (end - start >= PMD_SIZE) { in populate_pmd()
1421 * We cannot use a 1G page so allocate a PMD page if needed. in populate_pmd()
1425 return -1; in populate_pmd()
1429 set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn, in populate_pmd()
1433 cpa->pfn += PMD_SIZE >> PAGE_SHIFT; in populate_pmd()
1444 return -1; in populate_pmd()
1446 populate_pte(cpa, start, end, num_pages - cur_pages, in populate_pmd()
1460 end = start + (cpa->numpages << PAGE_SHIFT); in populate_pud()
1463 * Not on a Gb page boundary? => map everything up to it with in populate_pud()
1466 if (start & (PUD_SIZE - 1)) { in populate_pud()
1471 cur_pages = (pre_end - start) >> PAGE_SHIFT; in populate_pud()
1472 cur_pages = min_t(int, (int)cpa->numpages, cur_pages); in populate_pud()
1477 * Need a PMD page? in populate_pud()
1481 return -1; in populate_pud()
1492 if (cpa->numpages == cur_pages) in populate_pud()
1501 while (boot_cpu_has(X86_FEATURE_GBPAGES) && end - start >= PUD_SIZE) { in populate_pud()
1502 set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn, in populate_pud()
1506 cpa->pfn += PUD_SIZE >> PAGE_SHIFT; in populate_pud()
1518 return -1; in populate_pud()
1520 tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages, in populate_pud()
1531 * Restrictions for kernel page table do not necessarily apply when mapping in
1542 pgd_entry = cpa->pgd + pgd_index(addr); in populate_pgd()
1547 return -1; in populate_pgd()
1553 * Allocate a PUD page and hand it down for mapping. in populate_pgd()
1559 return -1; in populate_pgd()
1564 pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr); in populate_pgd()
1565 pgprot_val(pgprot) |= pgprot_val(cpa->mask_set); in populate_pgd()
1570 * Leave the PUD page in place in case some other CPU or thread in populate_pgd()
1575 addr + (cpa->numpages << PAGE_SHIFT)); in populate_pgd()
1579 cpa->numpages = ret; in populate_pgd()
1586 if (cpa->pgd) { in __cpa_process_fault()
1590 * provide a ->pgd value. This may change in the future. in __cpa_process_fault()
1599 cpa->numpages = 1; in __cpa_process_fault()
1607 * one virtual address page and its pfn. TBD: numpages can be set based in __cpa_process_fault()
1608 * on the initial value and the level returned by lookup_address(). in __cpa_process_fault()
1612 cpa->numpages = 1; in __cpa_process_fault()
1613 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT; in __cpa_process_fault()
1616 } else if (__cpa_pfn_in_highmap(cpa->pfn)) { in __cpa_process_fault()
1618 return -EFAULT; in __cpa_process_fault()
1621 "vaddr = %lx cpa->vaddr = %lx\n", vaddr, in __cpa_process_fault()
1622 *cpa->vaddr); in __cpa_process_fault()
1624 return -EFAULT; in __cpa_process_fault()
1632 unsigned int level; in __change_page_attr() local
1636 address = __cpa_addr(cpa, cpa->curpage); in __change_page_attr()
1638 kpte = _lookup_address_cpa(cpa, address, &level, &nx, &rw); in __change_page_attr()
1646 if (level == PG_LEVEL_4K) { in __change_page_attr()
1652 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); in __change_page_attr()
1653 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set); in __change_page_attr()
1671 cpa->pfn = pfn; in __change_page_attr()
1677 cpa->flags |= CPA_FLUSHTLB; in __change_page_attr()
1679 cpa->numpages = 1; in __change_page_attr()
1684 * Check, whether we can keep the large page intact in __change_page_attr()
1689 * When the range fits into the existing large page, in __change_page_attr()
1690 * return. cp->numpages and cpa->tlbflush have been updated in in __change_page_attr()
1697 * We have to split the large page: in __change_page_attr()
1714 unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT); in cpa_process_alias()
1718 if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1)) in cpa_process_alias()
1725 vaddr = __cpa_addr(cpa, cpa->curpage); in cpa_process_alias()
1740 cpa->force_flush_all = 1; in cpa_process_alias()
1754 __cpa_pfn_in_highmap(cpa->pfn)) { in cpa_process_alias()
1755 unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + in cpa_process_alias()
1756 __START_KERNEL_map - phys_base; in cpa_process_alias()
1771 cpa->force_flush_all = 1; in cpa_process_alias()
1785 unsigned long numpages = cpa->numpages; in __change_page_attr_set_clr()
1792 if (!(pgprot_val(cpa->mask_set) | pgprot_val(cpa->mask_clr)) && in __change_page_attr_set_clr()
1793 !cpa->force_split) in __change_page_attr_set_clr()
1798 * Store the remaining nr of pages for the large page in __change_page_attr_set_clr()
1801 cpa->numpages = rempages; in __change_page_attr_set_clr()
1802 /* for array changes, we can't use large page */ in __change_page_attr_set_clr()
1803 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) in __change_page_attr_set_clr()
1804 cpa->numpages = 1; in __change_page_attr_set_clr()
1814 if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS)) { in __change_page_attr_set_clr()
1822 * CPA operation. Either a large page has been in __change_page_attr_set_clr()
1823 * preserved or a single page update happened. in __change_page_attr_set_clr()
1825 BUG_ON(cpa->numpages > rempages || !cpa->numpages); in __change_page_attr_set_clr()
1826 rempages -= cpa->numpages; in __change_page_attr_set_clr()
1827 cpa->curpage += cpa->numpages; in __change_page_attr_set_clr()
1832 cpa->numpages = numpages; in __change_page_attr_set_clr()
1839 struct page **pages) in change_page_attr_set_clr()
1848 * feature. Clearing non-supported features is OK. in change_page_attr_set_clr()
1933 static inline int cpa_set_pages_array(struct page **pages, int numpages, in cpa_set_pages_array()
1940 static inline int cpa_clear_pages_array(struct page **pages, int numpages, in cpa_clear_pages_array()
1950 * If you want to set the pgprot to a specific page protocol, use the
2057 /* Prevent speculative access to a page by marking it not-present */
2071 * speculative access to the poison page because we'd have in set_mce_nospec()
2074 * Instead we get tricky. We create a non-canonical address in set_mce_nospec()
2170 * informed about "encryption" status via page tables.
2239 * Stop new private<->shared conversions.
2241 * Taking the exclusive mem_enc_lock waits for in-flight conversions to complete.
2264 return -EBUSY; in __set_memory_enc_dec()
2286 int set_pages_uc(struct page *page, int numpages) in set_pages_uc() argument
2288 unsigned long addr = (unsigned long)page_address(page); in set_pages_uc()
2294 static int _set_pages_array(struct page **pages, int numpages, in _set_pages_array()
2313 /* If WC, set to UC- first and then WC */ in _set_pages_array()
2337 return -EINVAL; in _set_pages_array()
2340 int set_pages_array_uc(struct page **pages, int numpages) in set_pages_array_uc()
2346 int set_pages_array_wc(struct page **pages, int numpages) in set_pages_array_wc()
2352 int set_pages_wb(struct page *page, int numpages) in set_pages_wb() argument
2354 unsigned long addr = (unsigned long)page_address(page); in set_pages_wb()
2360 int set_pages_array_wb(struct page **pages, int numpages) in set_pages_array_wb()
2385 int set_pages_ro(struct page *page, int numpages) in set_pages_ro() argument
2387 unsigned long addr = (unsigned long)page_address(page); in set_pages_ro()
2392 int set_pages_rw(struct page *page, int numpages) in set_pages_rw() argument
2394 unsigned long addr = (unsigned long)page_address(page); in set_pages_rw()
2399 static int __set_pages_p(struct page *page, int numpages) in __set_pages_p() argument
2401 unsigned long tempaddr = (unsigned long) page_address(page); in __set_pages_p()
2411 * we may need to break large pages for 64-bit kernel text in __set_pages_p()
2418 static int __set_pages_np(struct page *page, int numpages) in __set_pages_np() argument
2420 unsigned long tempaddr = (unsigned long) page_address(page); in __set_pages_np()
2430 * we may need to break large pages for 64-bit kernel text in __set_pages_np()
2437 int set_direct_map_invalid_noflush(struct page *page) in set_direct_map_invalid_noflush() argument
2439 return __set_pages_np(page, 1); in set_direct_map_invalid_noflush()
2442 int set_direct_map_default_noflush(struct page *page) in set_direct_map_default_noflush() argument
2444 return __set_pages_p(page, 1); in set_direct_map_default_noflush()
2448 void __kernel_map_pages(struct page *page, int numpages, int enable) in __kernel_map_pages() argument
2450 if (PageHighMem(page)) in __kernel_map_pages()
2453 debug_check_no_locks_freed(page_address(page), in __kernel_map_pages()
2460 * and hence no memory allocations during large page split. in __kernel_map_pages()
2463 __set_pages_p(page, numpages); in __kernel_map_pages()
2465 __set_pages_np(page, numpages); in __kernel_map_pages()
2469 * but that can deadlock->flush only current cpu. in __kernel_map_pages()
2481 bool kernel_page_present(struct page *page) in kernel_page_present() argument
2483 unsigned int level; in kernel_page_present() local
2486 if (PageHighMem(page)) in kernel_page_present()
2489 pte = lookup_address((unsigned long)page_address(page), &level); in kernel_page_present()
2496 int retval = -EINVAL; in kernel_map_pages_in_pgd()
2564 #include "cpa-test.c"