Lines Matching +full:d +full:- +full:tlb +full:- +full:size
6 * Synthesize TLB refill handlers at runtime.
10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
34 #include <asm/cpu-type.h>
54 * TLB load/store/modify handlers.
133 * CVMSEG starts at address -32768 and extends for in scratchpad_offset()
137 return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; in scratchpad_offset()
232 * TLB exception handlers.
243 pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); in output_pgtable_bits_defines()
244 pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); in output_pgtable_bits_defines()
245 pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); in output_pgtable_bits_defines()
246 pr_define("_PAGE_ACCESSED_SHIFT %d\n", _PAGE_ACCESSED_SHIFT); in output_pgtable_bits_defines()
247 pr_define("_PAGE_MODIFIED_SHIFT %d\n", _PAGE_MODIFIED_SHIFT); in output_pgtable_bits_defines()
249 pr_define("_PAGE_HUGE_SHIFT %d\n", _PAGE_HUGE_SHIFT); in output_pgtable_bits_defines()
253 pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); in output_pgtable_bits_defines()
255 pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); in output_pgtable_bits_defines()
256 pr_define("_PAGE_VALID_SHIFT %d\n", _PAGE_VALID_SHIFT); in output_pgtable_bits_defines()
257 pr_define("_PAGE_DIRTY_SHIFT %d\n", _PAGE_DIRTY_SHIFT); in output_pgtable_bits_defines()
258 pr_define("PFN_PTE_SHIFT %d\n", PFN_PTE_SHIFT); in output_pgtable_bits_defines()
264 unsigned int count = (end - start) / sizeof(u32); in dump_handler()
288 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
292 * We deliberately chose a buffer size of 128, so we won't scribble
297 /* simply assume worst case size for labels and relocs */
319 return -1; in allocate_kscratch()
321 r--; /* make it zero based */ in allocate_kscratch()
387 * The R3000 TLB handler is simple.
416 panic("TLB refill handler space exceeded"); in build_r3000_tlb_refill_handler()
418 pr_debug("Wrote TLB refill handler (%u instructions).\n", in build_r3000_tlb_refill_handler()
419 (unsigned int)(p - tlb_handler)); in build_r3000_tlb_refill_handler()
428 * The R4000 TLB handler is much more complicated. We have two
451 * The software work-around is to not allow the instruction preceding the TLBP
452 * to stall - make it an NOP or some other instruction guaranteed not to stall.
576 panic("No TLB refill handler yet (CPU type: %d)", in build_tlb_write_entry()
597 ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); in build_convert_pte_to_entrylo()
622 /* Reset default page size */ in build_restore_pagemask()
641 /* Reset default page size */ in build_restore_pagemask()
664 /* Set huge page tlb entry size */ in build_huge_tlb_write_entry()
696 * A huge PTE describes an area the size of the in build_huge_update_entries()
697 * configured huge page size. This is twice the in build_huge_update_entries()
698 * of the large TLB entry size we intend to use. in build_huge_update_entries()
699 * A TLB entry half the size of the configured in build_huge_update_entries()
700 * huge page size is configured into entrylo0 in build_huge_update_entries()
783 uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); in build_get_pmde64()
790 if (pgd_reg != -1) { in build_get_pmde64()
825 uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3); in build_get_pmde64()
827 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3); in build_get_pmde64()
832 uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */ in build_get_pmde64()
833 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3); in build_get_pmde64()
839 uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */ in build_get_pmde64()
840 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3); in build_get_pmde64()
928 if (pgd_reg != -1) { in build_get_pgde32()
957 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; in build_adjust_context()
958 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); in build_adjust_context()
1065 if (pgd_reg != -1) in build_fast_tlb_refill_handler()
1076 PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); in build_fast_tlb_refill_handler()
1079 if (pgd_reg == -1) { in build_fast_tlb_refill_handler()
1085 if (pgd_reg != -1) in build_fast_tlb_refill_handler()
1097 if (pgd_reg == -1) in build_fast_tlb_refill_handler()
1104 if (pgd_reg == -1) { in build_fast_tlb_refill_handler()
1120 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1126 * fall-through case = badvaddr *pgd_current in build_fast_tlb_refill_handler()
1132 uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); in build_fast_tlb_refill_handler()
1137 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); in build_fast_tlb_refill_handler()
1148 uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3); in build_fast_tlb_refill_handler()
1149 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3); in build_fast_tlb_refill_handler()
1163 uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); in build_fast_tlb_refill_handler()
1164 uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); in build_fast_tlb_refill_handler()
1237 * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
1240 * unused TLB refill exception.
1279 uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_refill_handler()
1316 * free instruction slot for the wrap-around branch. In worst in build_r4000_tlb_refill_handler()
1327 if ((p - tlb_handler) > 64) in build_r4000_tlb_refill_handler()
1328 panic("TLB refill handler space exceeded"); in build_r4000_tlb_refill_handler()
1330 * Now fold the handler in the TLB refill handler space. in build_r4000_tlb_refill_handler()
1335 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1338 if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1) in build_r4000_tlb_refill_handler()
1339 || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3) in build_r4000_tlb_refill_handler()
1341 tlb_handler + MIPS64_REFILL_INSNS - 3))) in build_r4000_tlb_refill_handler()
1342 panic("TLB refill handler space exceeded"); in build_r4000_tlb_refill_handler()
1344 * Now fold the handler in the TLB refill handler space. in build_r4000_tlb_refill_handler()
1347 if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) { in build_r4000_tlb_refill_handler()
1350 final_len = p - tlb_handler; in build_r4000_tlb_refill_handler()
1370 split < p - MIPS64_REFILL_INSNS) in build_r4000_tlb_refill_handler()
1379 split = tlb_handler + MIPS64_REFILL_INSNS - 2; in build_r4000_tlb_refill_handler()
1386 if (uasm_insn_has_bdelay(relocs, split - 1)) in build_r4000_tlb_refill_handler()
1387 split--; in build_r4000_tlb_refill_handler()
1391 f += split - tlb_handler; in build_r4000_tlb_refill_handler()
1402 uasm_move_labels(labels, f, f + 1, -1); in build_r4000_tlb_refill_handler()
1410 final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) + in build_r4000_tlb_refill_handler()
1411 (p - split); in build_r4000_tlb_refill_handler()
1418 pr_debug("Wrote TLB refill handler (%u instructions).\n", in build_r4000_tlb_refill_handler()
1442 pgd_w = PGDIR_SHIFT - PMD_SHIFT + PGD_TABLE_ORDER; in setup_pw()
1445 pmd_w = PMD_SHIFT - PAGE_SHIFT; in setup_pw()
1447 pgd_w = PGDIR_SHIFT - PAGE_SHIFT + PGD_TABLE_ORDER; in setup_pw()
1451 pt_w = PAGE_SHIFT - 3; in setup_pw()
1486 PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3); in build_loongson3_tlb_refill_handler()
1543 memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p); in build_setup_pgd()
1548 if (pgd_reg == -1) { in build_setup_pgd()
1563 uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); in build_setup_pgd()
1592 if (pgd_reg != -1) { in build_setup_pgd()
1606 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd)); in build_setup_pgd()
1699 * the page table where this PTE is located, PTE will be re-loaded
1721 /* You lose the SMP race :-(*/ in build_pte_present()
1734 /* You lose the SMP race :-(*/ in build_pte_present()
1771 /* You lose the SMP race :-(*/ in build_pte_writable()
1808 /* You lose the SMP race :-(*/ in build_pte_modifiable()
1817 * R3000 style TLB load/store/modify handlers.
1884 memset(p, 0, handle_tlbl_end - (char *)p); in build_r3000_tlb_load_handler()
1889 build_pte_present(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbl); in build_r3000_tlb_load_handler()
1891 build_make_valid(&p, &r, GPR_K0, GPR_K1, -1); in build_r3000_tlb_load_handler()
1899 panic("TLB load handler fastpath space exceeded"); in build_r3000_tlb_load_handler()
1902 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", in build_r3000_tlb_load_handler()
1903 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r3000_tlb_load_handler()
1914 memset(p, 0, handle_tlbs_end - (char *)p); in build_r3000_tlb_store_handler()
1919 build_pte_writable(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbs); in build_r3000_tlb_store_handler()
1921 build_make_write(&p, &r, GPR_K0, GPR_K1, -1); in build_r3000_tlb_store_handler()
1929 panic("TLB store handler fastpath space exceeded"); in build_r3000_tlb_store_handler()
1932 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", in build_r3000_tlb_store_handler()
1933 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r3000_tlb_store_handler()
1944 memset(p, 0, handle_tlbm_end - (char *)p); in build_r3000_tlb_modify_handler()
1949 build_pte_modifiable(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbm); in build_r3000_tlb_modify_handler()
1951 build_make_write(&p, &r, GPR_K0, GPR_K1, -1); in build_r3000_tlb_modify_handler()
1959 panic("TLB modify handler fastpath space exceeded"); in build_r3000_tlb_modify_handler()
1962 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", in build_r3000_tlb_modify_handler()
1963 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r3000_tlb_modify_handler()
1972 * When a Hardware Table Walker is running it can replace TLB entries in cpu_has_tlbex_tlbp_race()
1990 * R4000 style TLB load/store/modify handlers.
2006 * For huge tlb entries, pmd doesn't contain an address but in build_r4000_tlbchange_handler_head()
2007 * instead contains the tlb pte. Check the PAGE_HUGE bit and in build_r4000_tlbchange_handler_head()
2008 * see if we need to jump to huge tlb processing. in build_r4000_tlbchange_handler_head()
2015 UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT - PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2016 uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2); in build_r4000_tlbchange_handler_head()
2061 memset(p, 0, handle_tlbl_end - (char *)p); in build_r4000_tlb_load_handler()
2073 uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits); in build_r4000_tlb_load_handler()
2099 * Warn if something may race with us & replace the TLB entry in build_r4000_tlb_load_handler()
2165 * Warn if something may race with us & replace the TLB entry in build_r4000_tlb_load_handler()
2227 panic("TLB load handler fastpath space exceeded"); in build_r4000_tlb_load_handler()
2230 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n", in build_r4000_tlb_load_handler()
2231 (unsigned int)(p - (u32 *)handle_tlbl)); in build_r4000_tlb_load_handler()
2243 memset(p, 0, handle_tlbs_end - (char *)p); in build_r4000_tlb_store_handler()
2283 panic("TLB store handler fastpath space exceeded"); in build_r4000_tlb_store_handler()
2286 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n", in build_r4000_tlb_store_handler()
2287 (unsigned int)(p - (u32 *)handle_tlbs)); in build_r4000_tlb_store_handler()
2299 memset(p, 0, handle_tlbm_end - (char *)p); in build_r4000_tlb_modify_handler()
2340 panic("TLB modify handler fastpath space exceeded"); in build_r4000_tlb_modify_handler()
2343 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n", in build_r4000_tlb_modify_handler()
2344 (unsigned int)(p - (u32 *)handle_tlbm)); in build_r4000_tlb_modify_handler()
2404 * We are using 2-level page tables, so we only need to in config_htw_params()
2413 /* re-initialize the GDI field */ in config_htw_params()
2416 /* re-initialize the PTI field including the even/odd bit */ in config_htw_params()
2447 /* Set pointer size to size of directory pointers */ in config_htw_params()
2451 pwsize |= ((PTE_T_LOG2 - PGD_T_LOG2) << MIPS_PWSIZE_PTEW_SHIFT) in config_htw_params()
2460 * Enable HTW (and only for XUSeg on 64-bit), and disable the rest of in config_htw_params()
2512 /* clear all non-PFN bits */ in check_pabits()
2513 entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1); in check_pabits()
2518 fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0); in check_pabits()
2521 fillbits -= min_t(unsigned, fillbits, 2); in check_pabits()
2532 * The refill handler is generated per-CPU, multi-node systems in build_tlb_refill_handler()
2545 …check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3… in build_tlb_refill_handler()
2560 panic("No R3000 TLB refill handler"); in build_tlb_refill_handler()