10  * Copyright (C) 2006  Ralf Baechle (ralf@linux-mips.org)
34 #include <asm/cpu-type.h>
133 	 * CVMSEG starts at address -32768 and extends for  in scratchpad_offset()
137 	return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768;  in scratchpad_offset()
264 	unsigned int count = (end - start) / sizeof(u32);  in dump_handler()
288  * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
319 		return -1;  in allocate_kscratch()
321 	r--; /* make it zero based */  in allocate_kscratch()
419 		 (unsigned int)(p - tlb_handler));  in build_r3000_tlb_refill_handler()
451  * The software work-around is to not allow the instruction preceding the TLBP
452  * to stall - make it an NOP or some other instruction guaranteed not to stall.
597 				    ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));  in build_convert_pte_to_entrylo()
758  * TMP will be clobbered, PTR will hold the pmd entry.
783 		uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3);  in build_get_pmde64()
790 	if (pgd_reg != -1) {  in build_get_pmde64()
825 	uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);  in build_get_pmde64()
827 	uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);  in build_get_pmde64()
832 	uasm_i_dsrl_safe(p, tmp, tmp, PUD_SHIFT - 3); /* get pud offset in bytes */  in build_get_pmde64()
833 	uasm_i_andi(p, tmp, tmp, (PTRS_PER_PUD - 1) << 3);  in build_get_pmde64()
839 	uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */  in build_get_pmde64()
840 	uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);  in build_get_pmde64()
848  * PTR will hold the pgd for vmalloc.
924  * TMP will be clobbered, PTR will hold the pgd entry.
928 	if (pgd_reg != -1) {  in build_get_pgde32()
957 	unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;  in build_adjust_context()
958 	unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);  in build_adjust_context()
1065 		if (pgd_reg != -1)  in build_fast_tlb_refill_handler()
1076 				 PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3);  in build_fast_tlb_refill_handler()
1079 		if (pgd_reg == -1) {  in build_fast_tlb_refill_handler()
1085 		if (pgd_reg != -1)  in build_fast_tlb_refill_handler()
1097 		if (pgd_reg == -1)  in build_fast_tlb_refill_handler()
1104 	if (pgd_reg == -1) {  in build_fast_tlb_refill_handler()
1120 		uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);  in build_fast_tlb_refill_handler()
1126 	 * fall-through case =	 badvaddr  *pgd_current  in build_fast_tlb_refill_handler()
1132 		uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3);  in build_fast_tlb_refill_handler()
1137 	uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3);  in build_fast_tlb_refill_handler()
1148 	uasm_i_dsrl_safe(p, scratch, tmp, PUD_SHIFT - 3);  in build_fast_tlb_refill_handler()
1149 	uasm_i_andi(p, scratch, scratch, (PTRS_PER_PUD - 1) << 3);  in build_fast_tlb_refill_handler()
1163 	uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3);  in build_fast_tlb_refill_handler()
1164 	uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3);  in build_fast_tlb_refill_handler()
1237  * For a 64-bit kernel, we are using the 64-bit XTLB refill exception
1279 			uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits);  in build_r4000_tlb_refill_handler()
1316 	 * free instruction slot for the wrap-around branch. In worst  in build_r4000_tlb_refill_handler()
1327 			if ((p - tlb_handler) > 64)  in build_r4000_tlb_refill_handler()
1335 			final_len = p - tlb_handler;  in build_r4000_tlb_refill_handler()
1338 			if (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 1)  in build_r4000_tlb_refill_handler()
1339 			    || (((p - tlb_handler) > (MIPS64_REFILL_INSNS * 2) - 3)  in build_r4000_tlb_refill_handler()
1341 							tlb_handler + MIPS64_REFILL_INSNS - 3)))  in build_r4000_tlb_refill_handler()
1347 			if ((p - tlb_handler) <= MIPS64_REFILL_INSNS) {  in build_r4000_tlb_refill_handler()
1350 				final_len = p - tlb_handler;  in build_r4000_tlb_refill_handler()
1370 				    split < p - MIPS64_REFILL_INSNS)  in build_r4000_tlb_refill_handler()
1379 					split = tlb_handler + MIPS64_REFILL_INSNS - 2;  in build_r4000_tlb_refill_handler()
1386 					if (uasm_insn_has_bdelay(relocs, split - 1))  in build_r4000_tlb_refill_handler()
1387 						split--;  in build_r4000_tlb_refill_handler()
1391 				f += split - tlb_handler;  in build_r4000_tlb_refill_handler()
1402 						uasm_move_labels(labels, f, f + 1, -1);  in build_r4000_tlb_refill_handler()
1410 				final_len = (f - (final_handler + MIPS64_REFILL_INSNS)) +  in build_r4000_tlb_refill_handler()
1411 					    (p - split);  in build_r4000_tlb_refill_handler()
1442 	pgd_w = PGDIR_SHIFT - PMD_SHIFT + PGD_TABLE_ORDER;  in setup_pw()
1445 	pmd_w = PMD_SHIFT - PAGE_SHIFT;  in setup_pw()
1447 	pgd_w = PGDIR_SHIFT - PAGE_SHIFT + PGD_TABLE_ORDER;  in setup_pw()
1451 	pt_w  = PAGE_SHIFT - 3;  in setup_pw()
1486 				PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3);  in build_loongson3_tlb_refill_handler()
1543 	memset(p, 0, tlbmiss_handler_setup_pgd_end - (char *)p);  in build_setup_pgd()
1548 	if (pgd_reg == -1) {  in build_setup_pgd()
1563 		uasm_i_dinsm(&p, a0, 0, 29, 64 - 29);  in build_setup_pgd()
1592 	if (pgd_reg != -1) {  in build_setup_pgd()
1606 		 (unsigned int)(p - (u32 *)tlbmiss_handler_setup_pgd));  in build_setup_pgd()
1699  * the page table where this PTE is located, PTE will be re-loaded
1721 				/* You lose the SMP race :-(*/  in build_pte_present()
1734 			/* You lose the SMP race :-(*/  in build_pte_present()
1771 		/* You lose the SMP race :-(*/  in build_pte_writable()
1808 			/* You lose the SMP race :-(*/  in build_pte_modifiable()
1884 	memset(p, 0, handle_tlbl_end - (char *)p);  in build_r3000_tlb_load_handler()
1889 	build_pte_present(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbl);  in build_r3000_tlb_load_handler()
1891 	build_make_valid(&p, &r, GPR_K0, GPR_K1, -1);  in build_r3000_tlb_load_handler()
1903 		 (unsigned int)(p - (u32 *)handle_tlbl));  in build_r3000_tlb_load_handler()
1914 	memset(p, 0, handle_tlbs_end - (char *)p);  in build_r3000_tlb_store_handler()
1919 	build_pte_writable(&p, &r, GPR_K0, GPR_K1, -1, label_nopage_tlbs);  in build_r3000_tlb_store_handler()
1921 	build_make_write(&p, &r, GPR_K0, GPR_K1, -1);  in build_r3000_tlb_store_handler()
1933 		 (unsigned int)(p - (u32 *)handle_tlbs));  in build_r3000_tlb_store_handler()
1944 	memset(p, 0, handle_tlbm_end - (char *)p);  in build_r3000_tlb_modify_handler()
1949 	build_pte_modifiable(&p, &r, GPR_K0, GPR_K1,  -1, label_nopage_tlbm);  in build_r3000_tlb_modify_handler()
1951 	build_make_write(&p, &r, GPR_K0, GPR_K1, -1);  in build_r3000_tlb_modify_handler()
1963 		 (unsigned int)(p - (u32 *)handle_tlbm));  in build_r3000_tlb_modify_handler()
1996 	struct work_registers wr = build_get_work_registers(p);  in build_r4000_tlbchange_handler_head()  local
1999 	build_get_pmde64(p, l, r, wr.r1, wr.r2); /* get pmd in ptr */  in build_r4000_tlbchange_handler_head()
2001 	build_get_pgde32(p, wr.r1, wr.r2); /* get pgd in ptr */  in build_r4000_tlbchange_handler_head()
2010 	build_is_huge_pte(p, r, wr.r1, wr.r2, label_tlb_huge_update);  in build_r4000_tlbchange_handler_head()
2013 	UASM_i_MFC0(p, wr.r1, C0_BADVADDR);  in build_r4000_tlbchange_handler_head()
2014 	UASM_i_LW(p, wr.r2, 0, wr.r2);  in build_r4000_tlbchange_handler_head()
2015 	UASM_i_SRL(p, wr.r1, wr.r1, PAGE_SHIFT - PTE_T_LOG2);  in build_r4000_tlbchange_handler_head()
2016 	uasm_i_andi(p, wr.r1, wr.r1, (PTRS_PER_PTE - 1) << PTE_T_LOG2);  in build_r4000_tlbchange_handler_head()
2017 	UASM_i_ADDU(p, wr.r2, wr.r2, wr.r1);  in build_r4000_tlbchange_handler_head()
2022 	iPTE_LW(p, wr.r1, wr.r2); /* get even pte */  in build_r4000_tlbchange_handler_head()
2028 			uasm_i_mfc0(p, wr.r3, C0_INDEX);  in build_r4000_tlbchange_handler_head()
2029 			uasm_il_bltz(p, r, wr.r3, label_leave);  in build_r4000_tlbchange_handler_head()
2033 	return wr;  in build_r4000_tlbchange_handler_head()
2059 	struct work_registers wr;  in build_r4000_tlb_load_handler()  local
2061 	memset(p, 0, handle_tlbl_end - (char *)p);  in build_r4000_tlb_load_handler()
2073 		uasm_i_dsll_safe(&p, GPR_K0, GPR_K0, 64 + 12 + 1 - segbits);  in build_r4000_tlb_load_handler()
2079 	wr = build_r4000_tlbchange_handler_head(&p, &l, &r);  in build_r4000_tlb_load_handler()
2080 	build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);  in build_r4000_tlb_load_handler()
2090 			uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),  in build_r4000_tlb_load_handler()
2093 			uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);  in build_r4000_tlb_load_handler()
2094 			uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround1);  in build_r4000_tlb_load_handler()
2113 			uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);  in build_r4000_tlb_load_handler()
2115 			uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));  in build_r4000_tlb_load_handler()
2116 			uasm_i_beqz(&p, wr.r3, 8);  in build_r4000_tlb_load_handler()
2119 		UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);  in build_r4000_tlb_load_handler()
2121 		UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);  in build_r4000_tlb_load_handler()
2123 		 * If the entryLo (now in wr.r3) is valid (bit 1), RI or  in build_r4000_tlb_load_handler()
2127 			uasm_il_bbit1(&p, &r, wr.r3, 1, label_nopage_tlbl);  in build_r4000_tlb_load_handler()
2131 			uasm_i_andi(&p, wr.r3, wr.r3, 2);  in build_r4000_tlb_load_handler()
2132 			uasm_il_bnez(&p, &r, wr.r3, label_nopage_tlbl);  in build_r4000_tlb_load_handler()
2137 	build_make_valid(&p, &r, wr.r1, wr.r2, wr.r3);  in build_r4000_tlb_load_handler()
2138 	build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);  in build_r4000_tlb_load_handler()
2146 	iPTE_LW(&p, wr.r1, wr.r2);  in build_r4000_tlb_load_handler()
2147 	build_pte_present(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbl);  in build_r4000_tlb_load_handler()
2156 			uasm_il_bbit0(&p, &r, wr.r1, ilog2(_PAGE_VALID),  in build_r4000_tlb_load_handler()
2159 			uasm_i_andi(&p, wr.r3, wr.r1, _PAGE_VALID);  in build_r4000_tlb_load_handler()
2160 			uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);  in build_r4000_tlb_load_handler()
2179 			uasm_i_bbit0(&p, wr.r2, ilog2(sizeof(pte_t)), 8);  in build_r4000_tlb_load_handler()
2181 			uasm_i_andi(&p, wr.r3, wr.r2, sizeof(pte_t));  in build_r4000_tlb_load_handler()
2182 			uasm_i_beqz(&p, wr.r3, 8);  in build_r4000_tlb_load_handler()
2185 		UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO0);  in build_r4000_tlb_load_handler()
2187 		UASM_i_MFC0(&p, wr.r3, C0_ENTRYLO1);  in build_r4000_tlb_load_handler()
2189 		 * If the entryLo (now in wr.r3) is valid (bit 1), RI or  in build_r4000_tlb_load_handler()
2193 			uasm_il_bbit0(&p, &r, wr.r3, 1, label_tlbl_goaround2);  in build_r4000_tlb_load_handler()
2195 			uasm_i_andi(&p, wr.r3, wr.r3, 2);  in build_r4000_tlb_load_handler()
2196 			uasm_il_beqz(&p, &r, wr.r3, label_tlbl_goaround2);  in build_r4000_tlb_load_handler()
2204 		build_restore_pagemask(&p, &r, wr.r3, label_nopage_tlbl, 0);  in build_r4000_tlb_load_handler()
2208 	uasm_i_ori(&p, wr.r1, wr.r1, (_PAGE_ACCESSED | _PAGE_VALID));  in build_r4000_tlb_load_handler()
2209 	build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1);  in build_r4000_tlb_load_handler()
2231 		 (unsigned int)(p - (u32 *)handle_tlbl));  in build_r4000_tlb_load_handler()
2241 	struct work_registers wr;  in build_r4000_tlb_store_handler()  local
2243 	memset(p, 0, handle_tlbs_end - (char *)p);  in build_r4000_tlb_store_handler()
2247 	wr = build_r4000_tlbchange_handler_head(&p, &l, &r);  in build_r4000_tlb_store_handler()
2248 	build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);  in build_r4000_tlb_store_handler()
2251 	build_make_write(&p, &r, wr.r1, wr.r2, wr.r3);  in build_r4000_tlb_store_handler()
2252 	build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);  in build_r4000_tlb_store_handler()
2260 	iPTE_LW(&p, wr.r1, wr.r2);  in build_r4000_tlb_store_handler()
2261 	build_pte_writable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbs);  in build_r4000_tlb_store_handler()
2263 	uasm_i_ori(&p, wr.r1, wr.r1,  in build_r4000_tlb_store_handler()
2265 	build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 1);  in build_r4000_tlb_store_handler()
2287 		 (unsigned int)(p - (u32 *)handle_tlbs));  in build_r4000_tlb_store_handler()
2297 	struct work_registers wr;  in build_r4000_tlb_modify_handler()  local
2299 	memset(p, 0, handle_tlbm_end - (char *)p);  in build_r4000_tlb_modify_handler()
2303 	wr = build_r4000_tlbchange_handler_head(&p, &l, &r);  in build_r4000_tlb_modify_handler()
2304 	build_pte_modifiable(&p, &r, wr.r1, wr.r2, wr.r3, label_nopage_tlbm);  in build_r4000_tlb_modify_handler()
2308 	build_make_write(&p, &r, wr.r1, wr.r2, wr.r3);  in build_r4000_tlb_modify_handler()
2309 	build_r4000_tlbchange_handler_tail(&p, &l, &r, wr.r1, wr.r2);  in build_r4000_tlb_modify_handler()
2317 	iPTE_LW(&p, wr.r1, wr.r2);  in build_r4000_tlb_modify_handler()
2318 	build_pte_modifiable(&p, &r, wr.r1, wr.r2,  wr.r3, label_nopage_tlbm);  in build_r4000_tlb_modify_handler()
2320 	uasm_i_ori(&p, wr.r1, wr.r1,  in build_r4000_tlb_modify_handler()
2322 	build_huge_handler_tail(&p, &r, &l, wr.r1, wr.r2, 0);  in build_r4000_tlb_modify_handler()
2344 		 (unsigned int)(p - (u32 *)handle_tlbm));  in build_r4000_tlb_modify_handler()
2404 	 * We are using 2-level page tables, so we only need to  in config_htw_params()
2413 	/* re-initialize the GDI field */  in config_htw_params()
2416 	/* re-initialize the PTI field including the even/odd bit */  in config_htw_params()
2451 	pwsize |= ((PTE_T_LOG2 - PGD_T_LOG2) << MIPS_PWSIZE_PTEW_SHIFT)  in config_htw_params()
2460 	 * Enable HTW (and only for XUSeg on 64-bit), and disable the rest of  in config_htw_params()
2512 	/* clear all non-PFN bits */  in check_pabits()
2513 	entry &= ~((1 << MIPS_ENTRYLO_PFN_SHIFT) - 1);  in check_pabits()
2518 	fillbits = max_t(int, (int)BITS_PER_LONG - pabits, 0);  in check_pabits()
2521 	fillbits -= min_t(unsigned, fillbits, 2);  in check_pabits()
2532 	 * The refill handler is generated per-CPU, multi-node systems  in build_tlb_refill_handler()
2545 …check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_TABLE_ORDER + PAGE_SHIFT - 3…  in build_tlb_refill_handler()