tlbex.c (bd45ac0c5daae35e7c71138172e63df5cf644cf6) tlbex.c (e30ec4525d4731d828a754823a83a3d6b50bb230)
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Synthesize TLB refill handlers at runtime.
7 *
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Synthesize TLB refill handlers at runtime.
7 *
8 * Copyright (C) 2004,2005,2006 by Thiemo Seufer
8 * Copyright (C) 2004, 2005, 2006, 2008 Thiemo Seufer
9 * Copyright (C) 2005, 2007 Maciej W. Rozycki
10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
11 *
12 * ... and the days got worse and worse and now you see
13 * I've gone completly out of my mind.
14 *
15 * They're coming to take me a away haha
16 * they're coming to take me a away hoho hihi haha
17 * to the funny farm where code is beautiful all the time ...
18 *
19 * (Condolences to Napoleon XIV)
20 */
21
22#include <linux/kernel.h>
23#include <linux/types.h>
24#include <linux/string.h>
25#include <linux/init.h>
26
9 * Copyright (C) 2005, 2007 Maciej W. Rozycki
10 * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
11 *
12 * ... and the days got worse and worse and now you see
13 * I've gone completly out of my mind.
14 *
15 * They're coming to take me a away haha
16 * they're coming to take me a away hoho hihi haha
17 * to the funny farm where code is beautiful all the time ...
18 *
19 * (Condolences to Napoleon XIV)
20 */
21
22#include <linux/kernel.h>
23#include <linux/types.h>
24#include <linux/string.h>
25#include <linux/init.h>
26
27#include <asm/bugs.h>
28#include <asm/mmu_context.h>
27#include <asm/mmu_context.h>
29#include <asm/inst.h>
30#include <asm/elf.h>
31#include <asm/war.h>
32
28#include <asm/war.h>
29
30#include "uasm.h"
31
33static inline int r45k_bvahwbug(void)
34{
35 /* XXX: We should probe for the presence of this bug, but we don't. */
36 return 0;
37}
38
39static inline int r4k_250MHZhwbug(void)
40{

--- 21 unchanged lines hidden (view full) ---

62 *
63 */
64static int __init m4kc_tlbp_war(void)
65{
66 return (current_cpu_data.processor_id & 0xffff00) ==
67 (PRID_COMP_MIPS | PRID_IMP_4KC);
68}
69
32static inline int r45k_bvahwbug(void)
33{
34 /* XXX: We should probe for the presence of this bug, but we don't. */
35 return 0;
36}
37
38static inline int r4k_250MHZhwbug(void)
39{

--- 21 unchanged lines hidden (view full) ---

61 *
62 */
63static int __init m4kc_tlbp_war(void)
64{
65 return (current_cpu_data.processor_id & 0xffff00) ==
66 (PRID_COMP_MIPS | PRID_IMP_4KC);
67}
68
70/*
71 * A little micro-assembler, intended for TLB refill handler
72 * synthesizing. It is intentionally kept simple, does only support
73 * a subset of instructions, and does not try to hide pipeline effects
74 * like branch delay slots.
75 */
76
77enum fields
78{
79 RS = 0x001,
80 RT = 0x002,
81 RD = 0x004,
82 RE = 0x008,
83 SIMM = 0x010,
84 UIMM = 0x020,
85 BIMM = 0x040,
86 JIMM = 0x080,
87 FUNC = 0x100,
88 SET = 0x200
89};
90
91#define OP_MASK 0x3f
92#define OP_SH 26
93#define RS_MASK 0x1f
94#define RS_SH 21
95#define RT_MASK 0x1f
96#define RT_SH 16
97#define RD_MASK 0x1f
98#define RD_SH 11
99#define RE_MASK 0x1f
100#define RE_SH 6
101#define IMM_MASK 0xffff
102#define IMM_SH 0
103#define JIMM_MASK 0x3ffffff
104#define JIMM_SH 0
105#define FUNC_MASK 0x3f
106#define FUNC_SH 0
107#define SET_MASK 0x7
108#define SET_SH 0
109
110enum opcode {
111 insn_invalid,
112 insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
113 insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
114 insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
115 insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32,
116 insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
117 insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
118 insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
119 insn_sra, insn_srl, insn_subu, insn_sw, insn_tlbp, insn_tlbwi,
120 insn_tlbwr, insn_xor, insn_xori
121};
122
123struct insn {
124 enum opcode opcode;
125 u32 match;
126 enum fields fields;
127};
128
129/* This macro sets the non-variable bits of an instruction. */
130#define M(a, b, c, d, e, f) \
131 ((a) << OP_SH \
132 | (b) << RS_SH \
133 | (c) << RT_SH \
134 | (d) << RD_SH \
135 | (e) << RE_SH \
136 | (f) << FUNC_SH)
137
138static struct insn insn_table[] __initdata = {
139 { insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
140 { insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD },
141 { insn_and, M(spec_op, 0, 0, 0, 0, and_op), RS | RT | RD },
142 { insn_andi, M(andi_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
143 { insn_beq, M(beq_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
144 { insn_beql, M(beql_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
145 { insn_bgez, M(bcond_op, 0, bgez_op, 0, 0, 0), RS | BIMM },
146 { insn_bgezl, M(bcond_op, 0, bgezl_op, 0, 0, 0), RS | BIMM },
147 { insn_bltz, M(bcond_op, 0, bltz_op, 0, 0, 0), RS | BIMM },
148 { insn_bltzl, M(bcond_op, 0, bltzl_op, 0, 0, 0), RS | BIMM },
149 { insn_bne, M(bne_op, 0, 0, 0, 0, 0), RS | RT | BIMM },
150 { insn_daddiu, M(daddiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
151 { insn_daddu, M(spec_op, 0, 0, 0, 0, daddu_op), RS | RT | RD },
152 { insn_dmfc0, M(cop0_op, dmfc_op, 0, 0, 0, 0), RT | RD | SET},
153 { insn_dmtc0, M(cop0_op, dmtc_op, 0, 0, 0, 0), RT | RD | SET},
154 { insn_dsll, M(spec_op, 0, 0, 0, 0, dsll_op), RT | RD | RE },
155 { insn_dsll32, M(spec_op, 0, 0, 0, 0, dsll32_op), RT | RD | RE },
156 { insn_dsra, M(spec_op, 0, 0, 0, 0, dsra_op), RT | RD | RE },
157 { insn_dsrl, M(spec_op, 0, 0, 0, 0, dsrl_op), RT | RD | RE },
158 { insn_dsrl32, M(spec_op, 0, 0, 0, 0, dsrl32_op), RT | RD | RE },
159 { insn_dsubu, M(spec_op, 0, 0, 0, 0, dsubu_op), RS | RT | RD },
160 { insn_eret, M(cop0_op, cop_op, 0, 0, 0, eret_op), 0 },
161 { insn_j, M(j_op, 0, 0, 0, 0, 0), JIMM },
162 { insn_jal, M(jal_op, 0, 0, 0, 0, 0), JIMM },
163 { insn_jr, M(spec_op, 0, 0, 0, 0, jr_op), RS },
164 { insn_ld, M(ld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
165 { insn_ll, M(ll_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
166 { insn_lld, M(lld_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
167 { insn_lui, M(lui_op, 0, 0, 0, 0, 0), RT | SIMM },
168 { insn_lw, M(lw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
169 { insn_mfc0, M(cop0_op, mfc_op, 0, 0, 0, 0), RT | RD | SET},
170 { insn_mtc0, M(cop0_op, mtc_op, 0, 0, 0, 0), RT | RD | SET},
171 { insn_ori, M(ori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
172 { insn_rfe, M(cop0_op, cop_op, 0, 0, 0, rfe_op), 0 },
173 { insn_sc, M(sc_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
174 { insn_scd, M(scd_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
175 { insn_sd, M(sd_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
176 { insn_sll, M(spec_op, 0, 0, 0, 0, sll_op), RT | RD | RE },
177 { insn_sra, M(spec_op, 0, 0, 0, 0, sra_op), RT | RD | RE },
178 { insn_srl, M(spec_op, 0, 0, 0, 0, srl_op), RT | RD | RE },
179 { insn_subu, M(spec_op, 0, 0, 0, 0, subu_op), RS | RT | RD },
180 { insn_sw, M(sw_op, 0, 0, 0, 0, 0), RS | RT | SIMM },
181 { insn_tlbp, M(cop0_op, cop_op, 0, 0, 0, tlbp_op), 0 },
182 { insn_tlbwi, M(cop0_op, cop_op, 0, 0, 0, tlbwi_op), 0 },
183 { insn_tlbwr, M(cop0_op, cop_op, 0, 0, 0, tlbwr_op), 0 },
184 { insn_xor, M(spec_op, 0, 0, 0, 0, xor_op), RS | RT | RD },
185 { insn_xori, M(xori_op, 0, 0, 0, 0, 0), RS | RT | UIMM },
186 { insn_invalid, 0, 0 }
187};
188
189#undef M
190
191static u32 __init build_rs(u32 arg)
192{
193 if (arg & ~RS_MASK)
194 printk(KERN_WARNING "TLB synthesizer field overflow\n");
195
196 return (arg & RS_MASK) << RS_SH;
197}
198
199static u32 __init build_rt(u32 arg)
200{
201 if (arg & ~RT_MASK)
202 printk(KERN_WARNING "TLB synthesizer field overflow\n");
203
204 return (arg & RT_MASK) << RT_SH;
205}
206
207static u32 __init build_rd(u32 arg)
208{
209 if (arg & ~RD_MASK)
210 printk(KERN_WARNING "TLB synthesizer field overflow\n");
211
212 return (arg & RD_MASK) << RD_SH;
213}
214
215static u32 __init build_re(u32 arg)
216{
217 if (arg & ~RE_MASK)
218 printk(KERN_WARNING "TLB synthesizer field overflow\n");
219
220 return (arg & RE_MASK) << RE_SH;
221}
222
223static u32 __init build_simm(s32 arg)
224{
225 if (arg > 0x7fff || arg < -0x8000)
226 printk(KERN_WARNING "TLB synthesizer field overflow\n");
227
228 return arg & 0xffff;
229}
230
231static u32 __init build_uimm(u32 arg)
232{
233 if (arg & ~IMM_MASK)
234 printk(KERN_WARNING "TLB synthesizer field overflow\n");
235
236 return arg & IMM_MASK;
237}
238
239static u32 __init build_bimm(s32 arg)
240{
241 if (arg > 0x1ffff || arg < -0x20000)
242 printk(KERN_WARNING "TLB synthesizer field overflow\n");
243
244 if (arg & 0x3)
245 printk(KERN_WARNING "Invalid TLB synthesizer branch target\n");
246
247 return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff);
248}
249
250static u32 __init build_jimm(u32 arg)
251{
252 if (arg & ~((JIMM_MASK) << 2))
253 printk(KERN_WARNING "TLB synthesizer field overflow\n");
254
255 return (arg >> 2) & JIMM_MASK;
256}
257
258static u32 __init build_func(u32 arg)
259{
260 if (arg & ~FUNC_MASK)
261 printk(KERN_WARNING "TLB synthesizer field overflow\n");
262
263 return arg & FUNC_MASK;
264}
265
266static u32 __init build_set(u32 arg)
267{
268 if (arg & ~SET_MASK)
269 printk(KERN_WARNING "TLB synthesizer field overflow\n");
270
271 return arg & SET_MASK;
272}
273
274/*
275 * The order of opcode arguments is implicitly left to right,
276 * starting with RS and ending with FUNC or IMM.
277 */
278static void __init build_insn(u32 **buf, enum opcode opc, ...)
279{
280 struct insn *ip = NULL;
281 unsigned int i;
282 va_list ap;
283 u32 op;
284
285 for (i = 0; insn_table[i].opcode != insn_invalid; i++)
286 if (insn_table[i].opcode == opc) {
287 ip = &insn_table[i];
288 break;
289 }
290
291 if (!ip || (opc == insn_daddiu && r4k_daddiu_bug()))
292 panic("Unsupported TLB synthesizer instruction %d", opc);
293
294 op = ip->match;
295 va_start(ap, opc);
296 if (ip->fields & RS) op |= build_rs(va_arg(ap, u32));
297 if (ip->fields & RT) op |= build_rt(va_arg(ap, u32));
298 if (ip->fields & RD) op |= build_rd(va_arg(ap, u32));
299 if (ip->fields & RE) op |= build_re(va_arg(ap, u32));
300 if (ip->fields & SIMM) op |= build_simm(va_arg(ap, s32));
301 if (ip->fields & UIMM) op |= build_uimm(va_arg(ap, u32));
302 if (ip->fields & BIMM) op |= build_bimm(va_arg(ap, s32));
303 if (ip->fields & JIMM) op |= build_jimm(va_arg(ap, u32));
304 if (ip->fields & FUNC) op |= build_func(va_arg(ap, u32));
305 if (ip->fields & SET) op |= build_set(va_arg(ap, u32));
306 va_end(ap);
307
308 **buf = op;
309 (*buf)++;
310}
311
312#define I_u1u2u3(op) \
313 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
314 unsigned int b, unsigned int c) \
315 { \
316 build_insn(buf, insn##op, a, b, c); \
317 }
318
319#define I_u2u1u3(op) \
320 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
321 unsigned int b, unsigned int c) \
322 { \
323 build_insn(buf, insn##op, b, a, c); \
324 }
325
326#define I_u3u1u2(op) \
327 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
328 unsigned int b, unsigned int c) \
329 { \
330 build_insn(buf, insn##op, b, c, a); \
331 }
332
333#define I_u1u2s3(op) \
334 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
335 unsigned int b, signed int c) \
336 { \
337 build_insn(buf, insn##op, a, b, c); \
338 }
339
340#define I_u2s3u1(op) \
341 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
342 signed int b, unsigned int c) \
343 { \
344 build_insn(buf, insn##op, c, a, b); \
345 }
346
347#define I_u2u1s3(op) \
348 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
349 unsigned int b, signed int c) \
350 { \
351 build_insn(buf, insn##op, b, a, c); \
352 }
353
354#define I_u1u2(op) \
355 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
356 unsigned int b) \
357 { \
358 build_insn(buf, insn##op, a, b); \
359 }
360
361#define I_u1s2(op) \
362 static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
363 signed int b) \
364 { \
365 build_insn(buf, insn##op, a, b); \
366 }
367
368#define I_u1(op) \
369 static void __init __maybe_unused i##op(u32 **buf, unsigned int a) \
370 { \
371 build_insn(buf, insn##op, a); \
372 }
373
374#define I_0(op) \
375 static void __init __maybe_unused i##op(u32 **buf) \
376 { \
377 build_insn(buf, insn##op); \
378 }
379
380I_u2u1s3(_addiu);
381I_u3u1u2(_addu);
382I_u2u1u3(_andi);
383I_u3u1u2(_and);
384I_u1u2s3(_beq);
385I_u1u2s3(_beql);
386I_u1s2(_bgez);
387I_u1s2(_bgezl);
388I_u1s2(_bltz);
389I_u1s2(_bltzl);
390I_u1u2s3(_bne);
391I_u1u2u3(_dmfc0);
392I_u1u2u3(_dmtc0);
393I_u2u1s3(_daddiu);
394I_u3u1u2(_daddu);
395I_u2u1u3(_dsll);
396I_u2u1u3(_dsll32);
397I_u2u1u3(_dsra);
398I_u2u1u3(_dsrl);
399I_u2u1u3(_dsrl32);
400I_u3u1u2(_dsubu);
401I_0(_eret);
402I_u1(_j);
403I_u1(_jal);
404I_u1(_jr);
405I_u2s3u1(_ld);
406I_u2s3u1(_ll);
407I_u2s3u1(_lld);
408I_u1s2(_lui);
409I_u2s3u1(_lw);
410I_u1u2u3(_mfc0);
411I_u1u2u3(_mtc0);
412I_u2u1u3(_ori);
413I_0(_rfe);
414I_u2s3u1(_sc);
415I_u2s3u1(_scd);
416I_u2s3u1(_sd);
417I_u2u1u3(_sll);
418I_u2u1u3(_sra);
419I_u2u1u3(_srl);
420I_u3u1u2(_subu);
421I_u2s3u1(_sw);
422I_0(_tlbp);
423I_0(_tlbwi);
424I_0(_tlbwr);
425I_u3u1u2(_xor)
426I_u2u1u3(_xori);
427
428/*
429 * handling labels
430 */
431
69/* Handle labels (which must be positive integers). */
432enum label_id {
70enum label_id {
433 label_invalid,
434 label_second_part,
71 label_second_part = 1,
435 label_leave,
436#ifdef MODULE_START
437 label_module_alloc,
438#endif
439 label_vmalloc,
440 label_vmalloc_done,
441 label_tlbw_hazard,
442 label_split,
443 label_nopage_tlbl,
444 label_nopage_tlbs,
445 label_nopage_tlbm,
446 label_smp_pgtable_change,
447 label_r3000_write_probe_fail,
448};
449
72 label_leave,
73#ifdef MODULE_START
74 label_module_alloc,
75#endif
76 label_vmalloc,
77 label_vmalloc_done,
78 label_tlbw_hazard,
79 label_split,
80 label_nopage_tlbl,
81 label_nopage_tlbs,
82 label_nopage_tlbm,
83 label_smp_pgtable_change,
84 label_r3000_write_probe_fail,
85};
86
450struct label {
451 u32 *addr;
452 enum label_id lab;
453};
454
455static void __init build_label(struct label **lab, u32 *addr,
456 enum label_id l)
457{
458 (*lab)->addr = addr;
459 (*lab)->lab = l;
460 (*lab)++;
461}
462
463#define L_LA(lb) \
464 static inline void __init l##lb(struct label **lab, u32 *addr) \
465 { \
466 build_label(lab, addr, label##lb); \
467 }
468
469L_LA(_second_part)
470L_LA(_leave)
87UASM_L_LA(_second_part)
88UASM_L_LA(_leave)
471#ifdef MODULE_START
89#ifdef MODULE_START
472L_LA(_module_alloc)
90UASM_L_LA(_module_alloc)
473#endif
91#endif
474L_LA(_vmalloc)
475L_LA(_vmalloc_done)
476L_LA(_tlbw_hazard)
477L_LA(_split)
478L_LA(_nopage_tlbl)
479L_LA(_nopage_tlbs)
480L_LA(_nopage_tlbm)
481L_LA(_smp_pgtable_change)
482L_LA(_r3000_write_probe_fail)
92UASM_L_LA(_vmalloc)
93UASM_L_LA(_vmalloc_done)
94UASM_L_LA(_tlbw_hazard)
95UASM_L_LA(_split)
96UASM_L_LA(_nopage_tlbl)
97UASM_L_LA(_nopage_tlbs)
98UASM_L_LA(_nopage_tlbm)
99UASM_L_LA(_smp_pgtable_change)
100UASM_L_LA(_r3000_write_probe_fail)
483
101
484/* convenience macros for instructions */
485#ifdef CONFIG_64BIT
486# define i_LW(buf, rs, rt, off) i_ld(buf, rs, rt, off)
487# define i_SW(buf, rs, rt, off) i_sd(buf, rs, rt, off)
488# define i_SLL(buf, rs, rt, sh) i_dsll(buf, rs, rt, sh)
489# define i_SRA(buf, rs, rt, sh) i_dsra(buf, rs, rt, sh)
490# define i_SRL(buf, rs, rt, sh) i_dsrl(buf, rs, rt, sh)
491# define i_MFC0(buf, rt, rd...) i_dmfc0(buf, rt, rd)
492# define i_MTC0(buf, rt, rd...) i_dmtc0(buf, rt, rd)
493# define i_ADDIU(buf, rs, rt, val) i_daddiu(buf, rs, rt, val)
494# define i_ADDU(buf, rs, rt, rd) i_daddu(buf, rs, rt, rd)
495# define i_SUBU(buf, rs, rt, rd) i_dsubu(buf, rs, rt, rd)
496# define i_LL(buf, rs, rt, off) i_lld(buf, rs, rt, off)
497# define i_SC(buf, rs, rt, off) i_scd(buf, rs, rt, off)
498#else
499# define i_LW(buf, rs, rt, off) i_lw(buf, rs, rt, off)
500# define i_SW(buf, rs, rt, off) i_sw(buf, rs, rt, off)
501# define i_SLL(buf, rs, rt, sh) i_sll(buf, rs, rt, sh)
502# define i_SRA(buf, rs, rt, sh) i_sra(buf, rs, rt, sh)
503# define i_SRL(buf, rs, rt, sh) i_srl(buf, rs, rt, sh)
504# define i_MFC0(buf, rt, rd...) i_mfc0(buf, rt, rd)
505# define i_MTC0(buf, rt, rd...) i_mtc0(buf, rt, rd)
506# define i_ADDIU(buf, rs, rt, val) i_addiu(buf, rs, rt, val)
507# define i_ADDU(buf, rs, rt, rd) i_addu(buf, rs, rt, rd)
508# define i_SUBU(buf, rs, rt, rd) i_subu(buf, rs, rt, rd)
509# define i_LL(buf, rs, rt, off) i_ll(buf, rs, rt, off)
510# define i_SC(buf, rs, rt, off) i_sc(buf, rs, rt, off)
511#endif
512
513#define i_b(buf, off) i_beq(buf, 0, 0, off)
514#define i_beqz(buf, rs, off) i_beq(buf, rs, 0, off)
515#define i_beqzl(buf, rs, off) i_beql(buf, rs, 0, off)
516#define i_bnez(buf, rs, off) i_bne(buf, rs, 0, off)
517#define i_bnezl(buf, rs, off) i_bnel(buf, rs, 0, off)
518#define i_move(buf, a, b) i_ADDU(buf, a, 0, b)
519#define i_nop(buf) i_sll(buf, 0, 0, 0)
520#define i_ssnop(buf) i_sll(buf, 0, 0, 1)
521#define i_ehb(buf) i_sll(buf, 0, 0, 3)
522
523static int __init __maybe_unused in_compat_space_p(long addr)
524{
525 /* Is this address in 32bit compat space? */
526#ifdef CONFIG_64BIT
527 return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L);
528#else
529 return 1;
530#endif
531}
532
533static int __init __maybe_unused rel_highest(long val)
534{
535#ifdef CONFIG_64BIT
536 return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
537#else
538 return 0;
539#endif
540}
541
542static int __init __maybe_unused rel_higher(long val)
543{
544#ifdef CONFIG_64BIT
545 return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
546#else
547 return 0;
548#endif
549}
550
551static int __init rel_hi(long val)
552{
553 return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000;
554}
555
556static int __init rel_lo(long val)
557{
558 return ((val & 0xffff) ^ 0x8000) - 0x8000;
559}
560
561static void __init i_LA_mostly(u32 **buf, unsigned int rs, long addr)
562{
563 if (!in_compat_space_p(addr)) {
564 i_lui(buf, rs, rel_highest(addr));
565 if (rel_higher(addr))
566 i_daddiu(buf, rs, rs, rel_higher(addr));
567 if (rel_hi(addr)) {
568 i_dsll(buf, rs, rs, 16);
569 i_daddiu(buf, rs, rs, rel_hi(addr));
570 i_dsll(buf, rs, rs, 16);
571 } else
572 i_dsll32(buf, rs, rs, 0);
573 } else
574 i_lui(buf, rs, rel_hi(addr));
575}
576
577static void __init __maybe_unused i_LA(u32 **buf, unsigned int rs, long addr)
578{
579 i_LA_mostly(buf, rs, addr);
580 if (rel_lo(addr)) {
581 if (!in_compat_space_p(addr))
582 i_daddiu(buf, rs, rs, rel_lo(addr));
583 else
584 i_addiu(buf, rs, rs, rel_lo(addr));
585 }
586}
587
588/*
102/*
589 * handle relocations
590 */
591
592struct reloc {
593 u32 *addr;
594 unsigned int type;
595 enum label_id lab;
596};
597
598static void __init r_mips_pc16(struct reloc **rel, u32 *addr,
599 enum label_id l)
600{
601 (*rel)->addr = addr;
602 (*rel)->type = R_MIPS_PC16;
603 (*rel)->lab = l;
604 (*rel)++;
605}
606
607static inline void __resolve_relocs(struct reloc *rel, struct label *lab)
608{
609 long laddr = (long)lab->addr;
610 long raddr = (long)rel->addr;
611
612 switch (rel->type) {
613 case R_MIPS_PC16:
614 *rel->addr |= build_bimm(laddr - (raddr + 4));
615 break;
616
617 default:
618 panic("Unsupported TLB synthesizer relocation %d",
619 rel->type);
620 }
621}
622
623static void __init resolve_relocs(struct reloc *rel, struct label *lab)
624{
625 struct label *l;
626
627 for (; rel->lab != label_invalid; rel++)
628 for (l = lab; l->lab != label_invalid; l++)
629 if (rel->lab == l->lab)
630 __resolve_relocs(rel, l);
631}
632
633static void __init move_relocs(struct reloc *rel, u32 *first, u32 *end,
634 long off)
635{
636 for (; rel->lab != label_invalid; rel++)
637 if (rel->addr >= first && rel->addr < end)
638 rel->addr += off;
639}
640
641static void __init move_labels(struct label *lab, u32 *first, u32 *end,
642 long off)
643{
644 for (; lab->lab != label_invalid; lab++)
645 if (lab->addr >= first && lab->addr < end)
646 lab->addr += off;
647}
648
649static void __init copy_handler(struct reloc *rel, struct label *lab,
650 u32 *first, u32 *end, u32 *target)
651{
652 long off = (long)(target - first);
653
654 memcpy(target, first, (end - first) * sizeof(u32));
655
656 move_relocs(rel, first, end, off);
657 move_labels(lab, first, end, off);
658}
659
660static int __init __maybe_unused insn_has_bdelay(struct reloc *rel,
661 u32 *addr)
662{
663 for (; rel->lab != label_invalid; rel++) {
664 if (rel->addr == addr
665 && (rel->type == R_MIPS_PC16
666 || rel->type == R_MIPS_26))
667 return 1;
668 }
669
670 return 0;
671}
672
673/* convenience functions for labeled branches */
674static void __init __maybe_unused
675 il_bltz(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
676{
677 r_mips_pc16(r, *p, l);
678 i_bltz(p, reg, 0);
679}
680
681static void __init __maybe_unused il_b(u32 **p, struct reloc **r,
682 enum label_id l)
683{
684 r_mips_pc16(r, *p, l);
685 i_b(p, 0);
686}
687
688static void __init il_beqz(u32 **p, struct reloc **r, unsigned int reg,
689 enum label_id l)
690{
691 r_mips_pc16(r, *p, l);
692 i_beqz(p, reg, 0);
693}
694
695static void __init __maybe_unused
696il_beqzl(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
697{
698 r_mips_pc16(r, *p, l);
699 i_beqzl(p, reg, 0);
700}
701
702static void __init il_bnez(u32 **p, struct reloc **r, unsigned int reg,
703 enum label_id l)
704{
705 r_mips_pc16(r, *p, l);
706 i_bnez(p, reg, 0);
707}
708
709static void __init il_bgezl(u32 **p, struct reloc **r, unsigned int reg,
710 enum label_id l)
711{
712 r_mips_pc16(r, *p, l);
713 i_bgezl(p, reg, 0);
714}
715
716static void __init __maybe_unused
717il_bgez(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
718{
719 r_mips_pc16(r, *p, l);
720 i_bgez(p, reg, 0);
721}
722
723/*
724 * For debug purposes.
725 */
726static inline void dump_handler(const u32 *handler, int count)
727{
728 int i;
729
730 pr_debug("\t.set push\n");
731 pr_debug("\t.set noreorder\n");

--- 15 unchanged lines hidden (view full) ---

747#define C0_ENTRYLO1 3, 0
748#define C0_CONTEXT 4, 0
749#define C0_BADVADDR 8, 0
750#define C0_ENTRYHI 10, 0
751#define C0_EPC 14, 0
752#define C0_XCONTEXT 20, 0
753
754#ifdef CONFIG_64BIT
103 * For debug purposes.
104 */
105static inline void dump_handler(const u32 *handler, int count)
106{
107 int i;
108
109 pr_debug("\t.set push\n");
110 pr_debug("\t.set noreorder\n");

--- 15 unchanged lines hidden (view full) ---

126#define C0_ENTRYLO1 3, 0
127#define C0_CONTEXT 4, 0
128#define C0_BADVADDR 8, 0
129#define C0_ENTRYHI 10, 0
130#define C0_EPC 14, 0
131#define C0_XCONTEXT 20, 0
132
133#ifdef CONFIG_64BIT
755# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_XCONTEXT)
134# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_XCONTEXT)
756#else
135#else
757# define GET_CONTEXT(buf, reg) i_MFC0(buf, reg, C0_CONTEXT)
136# define GET_CONTEXT(buf, reg) UASM_i_MFC0(buf, reg, C0_CONTEXT)
758#endif
759
760/* The worst case length of the handler is around 18 instructions for
761 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
762 * Maximum space available is 32 instructions for R3000 and 64
763 * instructions for R4000.
764 *
765 * We deliberately chose a buffer size of 128, so we won't scribble
766 * over anything important on overflow before we panic.
767 */
768static u32 tlb_handler[128] __initdata;
769
770/* simply assume worst case size for labels and relocs */
137#endif
138
139/* The worst case length of the handler is around 18 instructions for
140 * R3000-style TLBs and up to 63 instructions for R4000-style TLBs.
141 * Maximum space available is 32 instructions for R3000 and 64
142 * instructions for R4000.
143 *
144 * We deliberately chose a buffer size of 128, so we won't scribble
145 * over anything important on overflow before we panic.
146 */
147static u32 tlb_handler[128] __initdata;
148
149/* simply assume worst case size for labels and relocs */
771static struct label labels[128] __initdata;
772static struct reloc relocs[128] __initdata;
150static struct uasm_label labels[128] __initdata;
151static struct uasm_reloc relocs[128] __initdata;
773
774/*
775 * The R3000 TLB handler is simple.
776 */
777static void __init build_r3000_tlb_refill_handler(void)
778{
779 long pgdc = (long)pgd_current;
780 u32 *p;
781
782 memset(tlb_handler, 0, sizeof(tlb_handler));
783 p = tlb_handler;
784
152
153/*
154 * The R3000 TLB handler is simple.
155 */
156static void __init build_r3000_tlb_refill_handler(void)
157{
158 long pgdc = (long)pgd_current;
159 u32 *p;
160
161 memset(tlb_handler, 0, sizeof(tlb_handler));
162 p = tlb_handler;
163
785 i_mfc0(&p, K0, C0_BADVADDR);
786 i_lui(&p, K1, rel_hi(pgdc)); /* cp0 delay */
787 i_lw(&p, K1, rel_lo(pgdc), K1);
788 i_srl(&p, K0, K0, 22); /* load delay */
789 i_sll(&p, K0, K0, 2);
790 i_addu(&p, K1, K1, K0);
791 i_mfc0(&p, K0, C0_CONTEXT);
792 i_lw(&p, K1, 0, K1); /* cp0 delay */
793 i_andi(&p, K0, K0, 0xffc); /* load delay */
794 i_addu(&p, K1, K1, K0);
795 i_lw(&p, K0, 0, K1);
796 i_nop(&p); /* load delay */
797 i_mtc0(&p, K0, C0_ENTRYLO0);
798 i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
799 i_tlbwr(&p); /* cp0 delay */
800 i_jr(&p, K1);
801 i_rfe(&p); /* branch delay */
164 uasm_i_mfc0(&p, K0, C0_BADVADDR);
165 uasm_i_lui(&p, K1, uasm_rel_hi(pgdc)); /* cp0 delay */
166 uasm_i_lw(&p, K1, uasm_rel_lo(pgdc), K1);
167 uasm_i_srl(&p, K0, K0, 22); /* load delay */
168 uasm_i_sll(&p, K0, K0, 2);
169 uasm_i_addu(&p, K1, K1, K0);
170 uasm_i_mfc0(&p, K0, C0_CONTEXT);
171 uasm_i_lw(&p, K1, 0, K1); /* cp0 delay */
172 uasm_i_andi(&p, K0, K0, 0xffc); /* load delay */
173 uasm_i_addu(&p, K1, K1, K0);
174 uasm_i_lw(&p, K0, 0, K1);
175 uasm_i_nop(&p); /* load delay */
176 uasm_i_mtc0(&p, K0, C0_ENTRYLO0);
177 uasm_i_mfc0(&p, K1, C0_EPC); /* cp0 delay */
178 uasm_i_tlbwr(&p); /* cp0 delay */
179 uasm_i_jr(&p, K1);
180 uasm_i_rfe(&p); /* branch delay */
802
803 if (p > tlb_handler + 32)
804 panic("TLB refill handler space exceeded");
805
181
182 if (p > tlb_handler + 32)
183 panic("TLB refill handler space exceeded");
184
806 pr_info("Synthesized TLB refill handler (%u instructions).\n",
807 (unsigned int)(p - tlb_handler));
185 pr_debug("Wrote TLB refill handler (%u instructions).\n",
186 (unsigned int)(p - tlb_handler));
808
809 memcpy((void *)ebase, tlb_handler, 0x80);
810
811 dump_handler((u32 *)ebase, 32);
812}
813
814/*
815 * The R4000 TLB handler is much more complicated. We have two

--- 29 unchanged lines hidden (view full) ---

845static void __init __maybe_unused build_tlb_probe_entry(u32 **p)
846{
847 switch (current_cpu_type()) {
848 /* Found by experiment: R4600 v2.0 needs this, too. */
849 case CPU_R4600:
850 case CPU_R5000:
851 case CPU_R5000A:
852 case CPU_NEVADA:
187
188 memcpy((void *)ebase, tlb_handler, 0x80);
189
190 dump_handler((u32 *)ebase, 32);
191}
192
193/*
194 * The R4000 TLB handler is much more complicated. We have two

--- 29 unchanged lines hidden (view full) ---

224static void __init __maybe_unused build_tlb_probe_entry(u32 **p)
225{
226 switch (current_cpu_type()) {
227 /* Found by experiment: R4600 v2.0 needs this, too. */
228 case CPU_R4600:
229 case CPU_R5000:
230 case CPU_R5000A:
231 case CPU_NEVADA:
853 i_nop(p);
854 i_tlbp(p);
232 uasm_i_nop(p);
233 uasm_i_tlbp(p);
855 break;
856
857 default:
234 break;
235
236 default:
858 i_tlbp(p);
237 uasm_i_tlbp(p);
859 break;
860 }
861}
862
863/*
864 * Write random or indexed TLB entry, and care about the hazards from
865 * the preceeding mtc0 and for the following eret.
866 */
867enum tlb_write_entry { tlb_random, tlb_indexed };
868
238 break;
239 }
240}
241
242/*
243 * Write random or indexed TLB entry, and care about the hazards from
244 * the preceeding mtc0 and for the following eret.
245 */
246enum tlb_write_entry { tlb_random, tlb_indexed };
247
869static void __init build_tlb_write_entry(u32 **p, struct label **l,
870 struct reloc **r,
248static void __init build_tlb_write_entry(u32 **p, struct uasm_label **l,
249 struct uasm_reloc **r,
871 enum tlb_write_entry wmode)
872{
873 void(*tlbw)(u32 **) = NULL;
874
875 switch (wmode) {
250 enum tlb_write_entry wmode)
251{
252 void(*tlbw)(u32 **) = NULL;
253
254 switch (wmode) {
876 case tlb_random: tlbw = i_tlbwr; break;
877 case tlb_indexed: tlbw = i_tlbwi; break;
255 case tlb_random: tlbw = uasm_i_tlbwr; break;
256 case tlb_indexed: tlbw = uasm_i_tlbwi; break;
878 }
879
880 if (cpu_has_mips_r2) {
257 }
258
259 if (cpu_has_mips_r2) {
881 i_ehb(p);
260 uasm_i_ehb(p);
882 tlbw(p);
883 return;
884 }
885
886 switch (current_cpu_type()) {
887 case CPU_R4000PC:
888 case CPU_R4000SC:
889 case CPU_R4000MC:
890 case CPU_R4400PC:
891 case CPU_R4400SC:
892 case CPU_R4400MC:
893 /*
894 * This branch uses up a mtc0 hazard nop slot and saves
895 * two nops after the tlbw instruction.
896 */
261 tlbw(p);
262 return;
263 }
264
265 switch (current_cpu_type()) {
266 case CPU_R4000PC:
267 case CPU_R4000SC:
268 case CPU_R4000MC:
269 case CPU_R4400PC:
270 case CPU_R4400SC:
271 case CPU_R4400MC:
272 /*
273 * This branch uses up a mtc0 hazard nop slot and saves
274 * two nops after the tlbw instruction.
275 */
897 il_bgezl(p, r, 0, label_tlbw_hazard);
276 uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
898 tlbw(p);
277 tlbw(p);
899 l_tlbw_hazard(l, *p);
900 i_nop(p);
278 uasm_l_tlbw_hazard(l, *p);
279 uasm_i_nop(p);
901 break;
902
903 case CPU_R4600:
904 case CPU_R4700:
905 case CPU_R5000:
906 case CPU_R5000A:
280 break;
281
282 case CPU_R4600:
283 case CPU_R4700:
284 case CPU_R5000:
285 case CPU_R5000A:
907 i_nop(p);
286 uasm_i_nop(p);
908 tlbw(p);
287 tlbw(p);
909 i_nop(p);
288 uasm_i_nop(p);
910 break;
911
912 case CPU_R4300:
913 case CPU_5KC:
914 case CPU_TX49XX:
915 case CPU_AU1000:
916 case CPU_AU1100:
917 case CPU_AU1500:
918 case CPU_AU1550:
919 case CPU_AU1200:
920 case CPU_AU1210:
921 case CPU_AU1250:
922 case CPU_PR4450:
289 break;
290
291 case CPU_R4300:
292 case CPU_5KC:
293 case CPU_TX49XX:
294 case CPU_AU1000:
295 case CPU_AU1100:
296 case CPU_AU1500:
297 case CPU_AU1550:
298 case CPU_AU1200:
299 case CPU_AU1210:
300 case CPU_AU1250:
301 case CPU_PR4450:
923 i_nop(p);
302 uasm_i_nop(p);
924 tlbw(p);
925 break;
926
927 case CPU_R10000:
928 case CPU_R12000:
929 case CPU_R14000:
930 case CPU_4KC:
931 case CPU_SB1:
932 case CPU_SB1A:
933 case CPU_4KSC:
934 case CPU_20KC:
935 case CPU_25KF:
936 case CPU_BCM3302:
937 case CPU_BCM4710:
938 case CPU_LOONGSON2:
939 if (m4kc_tlbp_war())
303 tlbw(p);
304 break;
305
306 case CPU_R10000:
307 case CPU_R12000:
308 case CPU_R14000:
309 case CPU_4KC:
310 case CPU_SB1:
311 case CPU_SB1A:
312 case CPU_4KSC:
313 case CPU_20KC:
314 case CPU_25KF:
315 case CPU_BCM3302:
316 case CPU_BCM4710:
317 case CPU_LOONGSON2:
318 if (m4kc_tlbp_war())
940 i_nop(p);
319 uasm_i_nop(p);
941 tlbw(p);
942 break;
943
944 case CPU_NEVADA:
320 tlbw(p);
321 break;
322
323 case CPU_NEVADA:
945 i_nop(p); /* QED specifies 2 nops hazard */
324 uasm_i_nop(p); /* QED specifies 2 nops hazard */
946 /*
947 * This branch uses up a mtc0 hazard nop slot and saves
948 * a nop after the tlbw instruction.
949 */
325 /*
326 * This branch uses up a mtc0 hazard nop slot and saves
327 * a nop after the tlbw instruction.
328 */
950 il_bgezl(p, r, 0, label_tlbw_hazard);
329 uasm_il_bgezl(p, r, 0, label_tlbw_hazard);
951 tlbw(p);
330 tlbw(p);
952 l_tlbw_hazard(l, *p);
331 uasm_l_tlbw_hazard(l, *p);
953 break;
954
955 case CPU_RM7000:
332 break;
333
334 case CPU_RM7000:
956 i_nop(p);
957 i_nop(p);
958 i_nop(p);
959 i_nop(p);
335 uasm_i_nop(p);
336 uasm_i_nop(p);
337 uasm_i_nop(p);
338 uasm_i_nop(p);
960 tlbw(p);
961 break;
962
963 case CPU_RM9000:
964 /*
965 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
966 * use of the JTLB for instructions should not occur for 4
967 * cpu cycles and use for data translations should not occur
968 * for 3 cpu cycles.
969 */
339 tlbw(p);
340 break;
341
342 case CPU_RM9000:
343 /*
344 * When the JTLB is updated by tlbwi or tlbwr, a subsequent
345 * use of the JTLB for instructions should not occur for 4
346 * cpu cycles and use for data translations should not occur
347 * for 3 cpu cycles.
348 */
970 i_ssnop(p);
971 i_ssnop(p);
972 i_ssnop(p);
973 i_ssnop(p);
349 uasm_i_ssnop(p);
350 uasm_i_ssnop(p);
351 uasm_i_ssnop(p);
352 uasm_i_ssnop(p);
974 tlbw(p);
353 tlbw(p);
975 i_ssnop(p);
976 i_ssnop(p);
977 i_ssnop(p);
978 i_ssnop(p);
354 uasm_i_ssnop(p);
355 uasm_i_ssnop(p);
356 uasm_i_ssnop(p);
357 uasm_i_ssnop(p);
979 break;
980
981 case CPU_VR4111:
982 case CPU_VR4121:
983 case CPU_VR4122:
984 case CPU_VR4181:
985 case CPU_VR4181A:
358 break;
359
360 case CPU_VR4111:
361 case CPU_VR4121:
362 case CPU_VR4122:
363 case CPU_VR4181:
364 case CPU_VR4181A:
986 i_nop(p);
987 i_nop(p);
365 uasm_i_nop(p);
366 uasm_i_nop(p);
988 tlbw(p);
367 tlbw(p);
989 i_nop(p);
990 i_nop(p);
368 uasm_i_nop(p);
369 uasm_i_nop(p);
991 break;
992
993 case CPU_VR4131:
994 case CPU_VR4133:
995 case CPU_R5432:
370 break;
371
372 case CPU_VR4131:
373 case CPU_VR4133:
374 case CPU_R5432:
996 i_nop(p);
997 i_nop(p);
375 uasm_i_nop(p);
376 uasm_i_nop(p);
998 tlbw(p);
999 break;
1000
1001 default:
1002 panic("No TLB refill handler yet (CPU type: %d)",
1003 current_cpu_data.cputype);
1004 break;
1005 }
1006}
1007
1008#ifdef CONFIG_64BIT
1009/*
1010 * TMP and PTR are scratch.
1011 * TMP will be clobbered, PTR will hold the pmd entry.
1012 */
1013static void __init
377 tlbw(p);
378 break;
379
380 default:
381 panic("No TLB refill handler yet (CPU type: %d)",
382 current_cpu_data.cputype);
383 break;
384 }
385}
386
387#ifdef CONFIG_64BIT
388/*
389 * TMP and PTR are scratch.
390 * TMP will be clobbered, PTR will hold the pmd entry.
391 */
392static void __init
1014build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
393build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
1015 unsigned int tmp, unsigned int ptr)
1016{
1017 long pgdc = (long)pgd_current;
1018
1019 /*
1020 * The vmalloc handling is not in the hotpath.
1021 */
394 unsigned int tmp, unsigned int ptr)
395{
396 long pgdc = (long)pgd_current;
397
398 /*
399 * The vmalloc handling is not in the hotpath.
400 */
1022 i_dmfc0(p, tmp, C0_BADVADDR);
401 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
1023#ifdef MODULE_START
402#ifdef MODULE_START
1024 il_bltz(p, r, tmp, label_module_alloc);
403 uasm_il_bltz(p, r, tmp, label_module_alloc);
1025#else
404#else
1026 il_bltz(p, r, tmp, label_vmalloc);
405 uasm_il_bltz(p, r, tmp, label_vmalloc);
1027#endif
406#endif
1028 /* No i_nop needed here, since the next insn doesn't touch TMP. */
407 /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
1029
1030#ifdef CONFIG_SMP
1031# ifdef CONFIG_MIPS_MT_SMTC
1032 /*
1033 * SMTC uses TCBind value as "CPU" index
1034 */
408
409#ifdef CONFIG_SMP
410# ifdef CONFIG_MIPS_MT_SMTC
411 /*
412 * SMTC uses TCBind value as "CPU" index
413 */
1035 i_mfc0(p, ptr, C0_TCBIND);
1036 i_dsrl(p, ptr, ptr, 19);
414 uasm_i_mfc0(p, ptr, C0_TCBIND);
415 uasm_i_dsrl(p, ptr, ptr, 19);
1037# else
1038 /*
1039 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
1040 * stored in CONTEXT.
1041 */
416# else
417 /*
418 * 64 bit SMP running in XKPHYS has smp_processor_id() << 3
419 * stored in CONTEXT.
420 */
1042 i_dmfc0(p, ptr, C0_CONTEXT);
1043 i_dsrl(p, ptr, ptr, 23);
421 uasm_i_dmfc0(p, ptr, C0_CONTEXT);
422 uasm_i_dsrl(p, ptr, ptr, 23);
1044#endif
423#endif
1045 i_LA_mostly(p, tmp, pgdc);
1046 i_daddu(p, ptr, ptr, tmp);
1047 i_dmfc0(p, tmp, C0_BADVADDR);
1048 i_ld(p, ptr, rel_lo(pgdc), ptr);
424 UASM_i_LA_mostly(p, tmp, pgdc);
425 uasm_i_daddu(p, ptr, ptr, tmp);
426 uasm_i_dmfc0(p, tmp, C0_BADVADDR);
427 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
1049#else
428#else
1050 i_LA_mostly(p, ptr, pgdc);
1051 i_ld(p, ptr, rel_lo(pgdc), ptr);
429 UASM_i_LA_mostly(p, ptr, pgdc);
430 uasm_i_ld(p, ptr, uasm_rel_lo(pgdc), ptr);
1052#endif
1053
431#endif
432
1054 l_vmalloc_done(l, *p);
433 uasm_l_vmalloc_done(l, *p);
1055
1056 if (PGDIR_SHIFT - 3 < 32) /* get pgd offset in bytes */
434
435 if (PGDIR_SHIFT - 3 < 32) /* get pgd offset in bytes */
1057 i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
436 uasm_i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
1058 else
437 else
1059 i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);
438 uasm_i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);
1060
439
1061 i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
1062 i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
1063 i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
1064 i_ld(p, ptr, 0, ptr); /* get pmd pointer */
1065 i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
1066 i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
1067 i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
440 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
441 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
442 uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
443 uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
444 uasm_i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
445 uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
446 uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
1068}
1069
1070/*
1071 * BVADDR is the faulting address, PTR is scratch.
1072 * PTR will hold the pgd for vmalloc.
1073 */
1074static void __init
447}
448
449/*
450 * BVADDR is the faulting address, PTR is scratch.
451 * PTR will hold the pgd for vmalloc.
452 */
453static void __init
1075build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
454build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
1076 unsigned int bvaddr, unsigned int ptr)
1077{
1078 long swpd = (long)swapper_pg_dir;
1079
1080#ifdef MODULE_START
1081 long modd = (long)module_pg_dir;
1082
455 unsigned int bvaddr, unsigned int ptr)
456{
457 long swpd = (long)swapper_pg_dir;
458
459#ifdef MODULE_START
460 long modd = (long)module_pg_dir;
461
1083 l_module_alloc(l, *p);
462 uasm_l_module_alloc(l, *p);
1084 /*
1085 * Assumption:
1086 * VMALLOC_START >= 0xc000000000000000UL
1087 * MODULE_START >= 0xe000000000000000UL
1088 */
463 /*
464 * Assumption:
465 * VMALLOC_START >= 0xc000000000000000UL
466 * MODULE_START >= 0xe000000000000000UL
467 */
1089 i_SLL(p, ptr, bvaddr, 2);
1090 il_bgez(p, r, ptr, label_vmalloc);
468 UASM_i_SLL(p, ptr, bvaddr, 2);
469 uasm_il_bgez(p, r, ptr, label_vmalloc);
1091
470
1092 if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START)) {
1093 i_lui(p, ptr, rel_hi(MODULE_START)); /* delay slot */
471 if (uasm_in_compat_space_p(MODULE_START) &&
472 !uasm_rel_lo(MODULE_START)) {
473 uasm_i_lui(p, ptr, uasm_rel_hi(MODULE_START)); /* delay slot */
1094 } else {
1095 /* unlikely configuration */
474 } else {
475 /* unlikely configuration */
1096 i_nop(p); /* delay slot */
1097 i_LA(p, ptr, MODULE_START);
476 uasm_i_nop(p); /* delay slot */
477 UASM_i_LA(p, ptr, MODULE_START);
1098 }
478 }
1099 i_dsubu(p, bvaddr, bvaddr, ptr);
479 uasm_i_dsubu(p, bvaddr, bvaddr, ptr);
1100
480
1101 if (in_compat_space_p(modd) && !rel_lo(modd)) {
1102 il_b(p, r, label_vmalloc_done);
1103 i_lui(p, ptr, rel_hi(modd));
481 if (uasm_in_compat_space_p(modd) && !uasm_rel_lo(modd)) {
482 uasm_il_b(p, r, label_vmalloc_done);
483 uasm_i_lui(p, ptr, uasm_rel_hi(modd));
1104 } else {
484 } else {
1105 i_LA_mostly(p, ptr, modd);
1106 il_b(p, r, label_vmalloc_done);
1107 if (in_compat_space_p(modd))
1108 i_addiu(p, ptr, ptr, rel_lo(modd));
485 UASM_i_LA_mostly(p, ptr, modd);
486 uasm_il_b(p, r, label_vmalloc_done);
487 if (uasm_in_compat_space_p(modd))
488 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(modd));
1109 else
489 else
1110 i_daddiu(p, ptr, ptr, rel_lo(modd));
490 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(modd));
1111 }
1112
491 }
492
1113 l_vmalloc(l, *p);
1114 if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START) &&
493 uasm_l_vmalloc(l, *p);
494 if (uasm_in_compat_space_p(MODULE_START) &&
495 !uasm_rel_lo(MODULE_START) &&
1115 MODULE_START << 32 == VMALLOC_START)
496 MODULE_START << 32 == VMALLOC_START)
1116 i_dsll32(p, ptr, ptr, 0); /* typical case */
497 uasm_i_dsll32(p, ptr, ptr, 0); /* typical case */
1117 else
498 else
1118 i_LA(p, ptr, VMALLOC_START);
499 UASM_i_LA(p, ptr, VMALLOC_START);
1119#else
500#else
1120 l_vmalloc(l, *p);
1121 i_LA(p, ptr, VMALLOC_START);
501 uasm_l_vmalloc(l, *p);
502 UASM_i_LA(p, ptr, VMALLOC_START);
1122#endif
503#endif
1123 i_dsubu(p, bvaddr, bvaddr, ptr);
504 uasm_i_dsubu(p, bvaddr, bvaddr, ptr);
1124
505
1125 if (in_compat_space_p(swpd) && !rel_lo(swpd)) {
1126 il_b(p, r, label_vmalloc_done);
1127 i_lui(p, ptr, rel_hi(swpd));
506 if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
507 uasm_il_b(p, r, label_vmalloc_done);
508 uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
1128 } else {
509 } else {
1129 i_LA_mostly(p, ptr, swpd);
1130 il_b(p, r, label_vmalloc_done);
1131 if (in_compat_space_p(swpd))
1132 i_addiu(p, ptr, ptr, rel_lo(swpd));
510 UASM_i_LA_mostly(p, ptr, swpd);
511 uasm_il_b(p, r, label_vmalloc_done);
512 if (uasm_in_compat_space_p(swpd))
513 uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
1133 else
514 else
1134 i_daddiu(p, ptr, ptr, rel_lo(swpd));
515 uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
1135 }
1136}
1137
1138#else /* !CONFIG_64BIT */
1139
1140/*
1141 * TMP and PTR are scratch.
1142 * TMP will be clobbered, PTR will hold the pgd entry.

--- 4 unchanged lines hidden (view full) ---

1147 long pgdc = (long)pgd_current;
1148
1149 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
1150#ifdef CONFIG_SMP
1151#ifdef CONFIG_MIPS_MT_SMTC
1152 /*
1153 * SMTC uses TCBind value as "CPU" index
1154 */
516 }
517}
518
519#else /* !CONFIG_64BIT */
520
521/*
522 * TMP and PTR are scratch.
523 * TMP will be clobbered, PTR will hold the pgd entry.

--- 4 unchanged lines hidden (view full) ---

528 long pgdc = (long)pgd_current;
529
530 /* 32 bit SMP has smp_processor_id() stored in CONTEXT. */
531#ifdef CONFIG_SMP
532#ifdef CONFIG_MIPS_MT_SMTC
533 /*
534 * SMTC uses TCBind value as "CPU" index
535 */
1155 i_mfc0(p, ptr, C0_TCBIND);
1156 i_LA_mostly(p, tmp, pgdc);
1157 i_srl(p, ptr, ptr, 19);
536 uasm_i_mfc0(p, ptr, C0_TCBIND);
537 UASM_i_LA_mostly(p, tmp, pgdc);
538 uasm_i_srl(p, ptr, ptr, 19);
1158#else
1159 /*
1160 * smp_processor_id() << 3 is stored in CONTEXT.
1161 */
539#else
540 /*
541 * smp_processor_id() << 3 is stored in CONTEXT.
542 */
1162 i_mfc0(p, ptr, C0_CONTEXT);
1163 i_LA_mostly(p, tmp, pgdc);
1164 i_srl(p, ptr, ptr, 23);
543 uasm_i_mfc0(p, ptr, C0_CONTEXT);
544 UASM_i_LA_mostly(p, tmp, pgdc);
545 uasm_i_srl(p, ptr, ptr, 23);
1165#endif
546#endif
1166 i_addu(p, ptr, tmp, ptr);
547 uasm_i_addu(p, ptr, tmp, ptr);
1167#else
548#else
1168 i_LA_mostly(p, ptr, pgdc);
549 UASM_i_LA_mostly(p, ptr, pgdc);
1169#endif
550#endif
1170 i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
1171 i_lw(p, ptr, rel_lo(pgdc), ptr);
1172 i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
1173 i_sll(p, tmp, tmp, PGD_T_LOG2);
1174 i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
551 uasm_i_mfc0(p, tmp, C0_BADVADDR); /* get faulting address */
552 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
553 uasm_i_srl(p, tmp, tmp, PGDIR_SHIFT); /* get pgd only bits */
554 uasm_i_sll(p, tmp, tmp, PGD_T_LOG2);
555 uasm_i_addu(p, ptr, ptr, tmp); /* add in pgd offset */
1175}
1176
1177#endif /* !CONFIG_64BIT */
1178
1179static void __init build_adjust_context(u32 **p, unsigned int ctx)
1180{
1181 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
1182 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);

--- 10 unchanged lines hidden (view full) ---

1193 shift += 2;
1194 break;
1195
1196 default:
1197 break;
1198 }
1199
1200 if (shift)
556}
557
558#endif /* !CONFIG_64BIT */
559
560static void __init build_adjust_context(u32 **p, unsigned int ctx)
561{
562 unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
563 unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);

--- 10 unchanged lines hidden (view full) ---

574 shift += 2;
575 break;
576
577 default:
578 break;
579 }
580
581 if (shift)
1201 i_SRL(p, ctx, ctx, shift);
1202 i_andi(p, ctx, ctx, mask);
582 UASM_i_SRL(p, ctx, ctx, shift);
583 uasm_i_andi(p, ctx, ctx, mask);
1203}
1204
1205static void __init build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
1206{
1207 /*
1208 * Bug workaround for the Nevada. It seems as if under certain
1209 * circumstances the move from cp0_context might produce a
1210 * bogus result when the mfc0 instruction and its consumer are
1211 * in a different cacheline or a load instruction, probably any
1212 * memory reference, is between them.
1213 */
1214 switch (current_cpu_type()) {
1215 case CPU_NEVADA:
584}
585
586static void __init build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr)
587{
588 /*
589 * Bug workaround for the Nevada. It seems as if under certain
590 * circumstances the move from cp0_context might produce a
591 * bogus result when the mfc0 instruction and its consumer are
592 * in a different cacheline or a load instruction, probably any
593 * memory reference, is between them.
594 */
595 switch (current_cpu_type()) {
596 case CPU_NEVADA:
1216 i_LW(p, ptr, 0, ptr);
597 UASM_i_LW(p, ptr, 0, ptr);
1217 GET_CONTEXT(p, tmp); /* get context reg */
1218 break;
1219
1220 default:
1221 GET_CONTEXT(p, tmp); /* get context reg */
598 GET_CONTEXT(p, tmp); /* get context reg */
599 break;
600
601 default:
602 GET_CONTEXT(p, tmp); /* get context reg */
1222 i_LW(p, ptr, 0, ptr);
603 UASM_i_LW(p, ptr, 0, ptr);
1223 break;
1224 }
1225
1226 build_adjust_context(p, tmp);
604 break;
605 }
606
607 build_adjust_context(p, tmp);
1227 i_ADDU(p, ptr, ptr, tmp); /* add in offset */
608 UASM_i_ADDU(p, ptr, ptr, tmp); /* add in offset */
1228}
1229
1230static void __init build_update_entries(u32 **p, unsigned int tmp,
1231 unsigned int ptep)
1232{
1233 /*
1234 * 64bit address support (36bit on a 32bit CPU) in a 32bit
1235 * Kernel is a special case. Only a few CPUs use it.
1236 */
1237#ifdef CONFIG_64BIT_PHYS_ADDR
1238 if (cpu_has_64bits) {
609}
610
611static void __init build_update_entries(u32 **p, unsigned int tmp,
612 unsigned int ptep)
613{
614 /*
615 * 64bit address support (36bit on a 32bit CPU) in a 32bit
616 * Kernel is a special case. Only a few CPUs use it.
617 */
618#ifdef CONFIG_64BIT_PHYS_ADDR
619 if (cpu_has_64bits) {
1239 i_ld(p, tmp, 0, ptep); /* get even pte */
1240 i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
1241 i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
1242 i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
1243 i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
1244 i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
620 uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
621 uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
622 uasm_i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
623 uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
624 uasm_i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
625 uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
1245 } else {
1246 int pte_off_even = sizeof(pte_t) / 2;
1247 int pte_off_odd = pte_off_even + sizeof(pte_t);
1248
1249 /* The pte entries are pre-shifted */
626 } else {
627 int pte_off_even = sizeof(pte_t) / 2;
628 int pte_off_odd = pte_off_even + sizeof(pte_t);
629
630 /* The pte entries are pre-shifted */
1250 i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
1251 i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
1252 i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
1253 i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
631 uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
632 uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
633 uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
634 uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
1254 }
1255#else
635 }
636#else
1256 i_LW(p, tmp, 0, ptep); /* get even pte */
1257 i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
637 UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
638 UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
1258 if (r45k_bvahwbug())
1259 build_tlb_probe_entry(p);
639 if (r45k_bvahwbug())
640 build_tlb_probe_entry(p);
1260 i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
641 UASM_i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
1261 if (r4k_250MHZhwbug())
642 if (r4k_250MHZhwbug())
1262 i_mtc0(p, 0, C0_ENTRYLO0);
1263 i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
1264 i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
643 uasm_i_mtc0(p, 0, C0_ENTRYLO0);
644 uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
645 UASM_i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
1265 if (r45k_bvahwbug())
646 if (r45k_bvahwbug())
1266 i_mfc0(p, tmp, C0_INDEX);
647 uasm_i_mfc0(p, tmp, C0_INDEX);
1267 if (r4k_250MHZhwbug())
648 if (r4k_250MHZhwbug())
1268 i_mtc0(p, 0, C0_ENTRYLO1);
1269 i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
649 uasm_i_mtc0(p, 0, C0_ENTRYLO1);
650 uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
1270#endif
1271}
1272
1273static void __init build_r4000_tlb_refill_handler(void)
1274{
1275 u32 *p = tlb_handler;
651#endif
652}
653
654static void __init build_r4000_tlb_refill_handler(void)
655{
656 u32 *p = tlb_handler;
1276 struct label *l = labels;
1277 struct reloc *r = relocs;
657 struct uasm_label *l = labels;
658 struct uasm_reloc *r = relocs;
1278 u32 *f;
1279 unsigned int final_len;
1280
1281 memset(tlb_handler, 0, sizeof(tlb_handler));
1282 memset(labels, 0, sizeof(labels));
1283 memset(relocs, 0, sizeof(relocs));
1284 memset(final_handler, 0, sizeof(final_handler));
1285
1286 /*
1287 * create the plain linear handler
1288 */
1289 if (bcm1250_m3_war()) {
659 u32 *f;
660 unsigned int final_len;
661
662 memset(tlb_handler, 0, sizeof(tlb_handler));
663 memset(labels, 0, sizeof(labels));
664 memset(relocs, 0, sizeof(relocs));
665 memset(final_handler, 0, sizeof(final_handler));
666
667 /*
668 * create the plain linear handler
669 */
670 if (bcm1250_m3_war()) {
1290 i_MFC0(&p, K0, C0_BADVADDR);
1291 i_MFC0(&p, K1, C0_ENTRYHI);
1292 i_xor(&p, K0, K0, K1);
1293 i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
1294 il_bnez(&p, &r, K0, label_leave);
1295 /* No need for i_nop */
671 UASM_i_MFC0(&p, K0, C0_BADVADDR);
672 UASM_i_MFC0(&p, K1, C0_ENTRYHI);
673 uasm_i_xor(&p, K0, K0, K1);
674 UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
675 uasm_il_bnez(&p, &r, K0, label_leave);
676 /* No need for uasm_i_nop */
1296 }
1297
1298#ifdef CONFIG_64BIT
1299 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
1300#else
1301 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
1302#endif
1303
1304 build_get_ptep(&p, K0, K1);
1305 build_update_entries(&p, K0, K1);
1306 build_tlb_write_entry(&p, &l, &r, tlb_random);
677 }
678
679#ifdef CONFIG_64BIT
680 build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */
681#else
682 build_get_pgde32(&p, K0, K1); /* get pgd in K1 */
683#endif
684
685 build_get_ptep(&p, K0, K1);
686 build_update_entries(&p, K0, K1);
687 build_tlb_write_entry(&p, &l, &r, tlb_random);
1307 l_leave(&l, p);
1308 i_eret(&p); /* return from trap */
688 uasm_l_leave(&l, p);
689 uasm_i_eret(&p); /* return from trap */
1309
1310#ifdef CONFIG_64BIT
1311 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
1312#endif
1313
1314 /*
1315 * Overflow check: For the 64bit handler, we need at least one
1316 * free instruction slot for the wrap-around branch. In worst
1317 * case, if the intended insertion point is a delay slot, we
1318 * need three, with the second nop'ed and the third being
1319 * unused.
1320 */
1321 /* Loongson2 ebase is different than r4k, we have more space */
1322#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
1323 if ((p - tlb_handler) > 64)
1324 panic("TLB refill handler space exceeded");
1325#else
1326 if (((p - tlb_handler) > 63)
1327 || (((p - tlb_handler) > 61)
690
691#ifdef CONFIG_64BIT
692 build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
693#endif
694
695 /*
696 * Overflow check: For the 64bit handler, we need at least one
697 * free instruction slot for the wrap-around branch. In worst
698 * case, if the intended insertion point is a delay slot, we
699 * need three, with the second nop'ed and the third being
700 * unused.
701 */
702 /* Loongson2 ebase is different than r4k, we have more space */
703#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
704 if ((p - tlb_handler) > 64)
705 panic("TLB refill handler space exceeded");
706#else
707 if (((p - tlb_handler) > 63)
708 || (((p - tlb_handler) > 61)
1328 && insn_has_bdelay(relocs, tlb_handler + 29)))
709 && uasm_insn_has_bdelay(relocs, tlb_handler + 29)))
1329 panic("TLB refill handler space exceeded");
1330#endif
1331
1332 /*
1333 * Now fold the handler in the TLB refill handler space.
1334 */
1335#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
1336 f = final_handler;
1337 /* Simplest case, just copy the handler. */
710 panic("TLB refill handler space exceeded");
711#endif
712
713 /*
714 * Now fold the handler in the TLB refill handler space.
715 */
716#if defined(CONFIG_32BIT) || defined(CONFIG_CPU_LOONGSON2)
717 f = final_handler;
718 /* Simplest case, just copy the handler. */
1338 copy_handler(relocs, labels, tlb_handler, p, f);
719 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1339 final_len = p - tlb_handler;
1340#else /* CONFIG_64BIT */
1341 f = final_handler + 32;
1342 if ((p - tlb_handler) <= 32) {
1343 /* Just copy the handler. */
720 final_len = p - tlb_handler;
721#else /* CONFIG_64BIT */
722 f = final_handler + 32;
723 if ((p - tlb_handler) <= 32) {
724 /* Just copy the handler. */
1344 copy_handler(relocs, labels, tlb_handler, p, f);
725 uasm_copy_handler(relocs, labels, tlb_handler, p, f);
1345 final_len = p - tlb_handler;
1346 } else {
1347 u32 *split = tlb_handler + 30;
1348
1349 /*
1350 * Find the split point.
1351 */
726 final_len = p - tlb_handler;
727 } else {
728 u32 *split = tlb_handler + 30;
729
730 /*
731 * Find the split point.
732 */
1352 if (insn_has_bdelay(relocs, split - 1))
733 if (uasm_insn_has_bdelay(relocs, split - 1))
1353 split--;
1354
1355 /* Copy first part of the handler. */
734 split--;
735
736 /* Copy first part of the handler. */
1356 copy_handler(relocs, labels, tlb_handler, split, f);
737 uasm_copy_handler(relocs, labels, tlb_handler, split, f);
1357 f += split - tlb_handler;
1358
1359 /* Insert branch. */
738 f += split - tlb_handler;
739
740 /* Insert branch. */
1360 l_split(&l, final_handler);
1361 il_b(&f, &r, label_split);
1362 if (insn_has_bdelay(relocs, split))
1363 i_nop(&f);
741 uasm_l_split(&l, final_handler);
742 uasm_il_b(&f, &r, label_split);
743 if (uasm_insn_has_bdelay(relocs, split))
744 uasm_i_nop(&f);
1364 else {
745 else {
1365 copy_handler(relocs, labels, split, split + 1, f);
1366 move_labels(labels, f, f + 1, -1);
746 uasm_copy_handler(relocs, labels, split, split + 1, f);
747 uasm_move_labels(labels, f, f + 1, -1);
1367 f++;
1368 split++;
1369 }
1370
1371 /* Copy the rest of the handler. */
748 f++;
749 split++;
750 }
751
752 /* Copy the rest of the handler. */
1372 copy_handler(relocs, labels, split, p, final_handler);
753 uasm_copy_handler(relocs, labels, split, p, final_handler);
1373 final_len = (f - (final_handler + 32)) + (p - split);
1374 }
1375#endif /* CONFIG_64BIT */
1376
754 final_len = (f - (final_handler + 32)) + (p - split);
755 }
756#endif /* CONFIG_64BIT */
757
1377 resolve_relocs(relocs, labels);
1378 pr_info("Synthesized TLB refill handler (%u instructions).\n",
1379 final_len);
758 uasm_resolve_relocs(relocs, labels);
759 pr_debug("Wrote TLB refill handler (%u instructions).\n",
760 final_len);
1380
1381 memcpy((void *)ebase, final_handler, 0x100);
1382
1383 dump_handler((u32 *)ebase, 64);
1384}
1385
1386/*
1387 * TLB load/store/modify handlers.

--- 10 unchanged lines hidden (view full) ---

1398 */
1399#define FASTPATH_SIZE 128
1400
1401u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
1402u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
1403u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
1404
1405static void __init
761
762 memcpy((void *)ebase, final_handler, 0x100);
763
764 dump_handler((u32 *)ebase, 64);
765}
766
767/*
768 * TLB load/store/modify handlers.

--- 10 unchanged lines hidden (view full) ---

779 */
780#define FASTPATH_SIZE 128
781
782u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
783u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
784u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
785
786static void __init
1406iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr)
787iPTE_LW(u32 **p, struct uasm_label **l, unsigned int pte, unsigned int ptr)
1407{
1408#ifdef CONFIG_SMP
1409# ifdef CONFIG_64BIT_PHYS_ADDR
1410 if (cpu_has_64bits)
788{
789#ifdef CONFIG_SMP
790# ifdef CONFIG_64BIT_PHYS_ADDR
791 if (cpu_has_64bits)
1411 i_lld(p, pte, 0, ptr);
792 uasm_i_lld(p, pte, 0, ptr);
1412 else
1413# endif
793 else
794# endif
1414 i_LL(p, pte, 0, ptr);
795 UASM_i_LL(p, pte, 0, ptr);
1415#else
1416# ifdef CONFIG_64BIT_PHYS_ADDR
1417 if (cpu_has_64bits)
796#else
797# ifdef CONFIG_64BIT_PHYS_ADDR
798 if (cpu_has_64bits)
1418 i_ld(p, pte, 0, ptr);
799 uasm_i_ld(p, pte, 0, ptr);
1419 else
1420# endif
800 else
801# endif
1421 i_LW(p, pte, 0, ptr);
802 UASM_i_LW(p, pte, 0, ptr);
1422#endif
1423}
1424
1425static void __init
803#endif
804}
805
806static void __init
1426iPTE_SW(u32 **p, struct reloc **r, unsigned int pte, unsigned int ptr,
807iPTE_SW(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr,
1427 unsigned int mode)
1428{
1429#ifdef CONFIG_64BIT_PHYS_ADDR
1430 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
1431#endif
1432
808 unsigned int mode)
809{
810#ifdef CONFIG_64BIT_PHYS_ADDR
811 unsigned int hwmode = mode & (_PAGE_VALID | _PAGE_DIRTY);
812#endif
813
1433 i_ori(p, pte, pte, mode);
814 uasm_i_ori(p, pte, pte, mode);
1434#ifdef CONFIG_SMP
1435# ifdef CONFIG_64BIT_PHYS_ADDR
1436 if (cpu_has_64bits)
815#ifdef CONFIG_SMP
816# ifdef CONFIG_64BIT_PHYS_ADDR
817 if (cpu_has_64bits)
1437 i_scd(p, pte, 0, ptr);
818 uasm_i_scd(p, pte, 0, ptr);
1438 else
1439# endif
819 else
820# endif
1440 i_SC(p, pte, 0, ptr);
821 UASM_i_SC(p, pte, 0, ptr);
1441
1442 if (r10000_llsc_war())
822
823 if (r10000_llsc_war())
1443 il_beqzl(p, r, pte, label_smp_pgtable_change);
824 uasm_il_beqzl(p, r, pte, label_smp_pgtable_change);
1444 else
825 else
1445 il_beqz(p, r, pte, label_smp_pgtable_change);
826 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
1446
1447# ifdef CONFIG_64BIT_PHYS_ADDR
1448 if (!cpu_has_64bits) {
827
828# ifdef CONFIG_64BIT_PHYS_ADDR
829 if (!cpu_has_64bits) {
1449 /* no i_nop needed */
1450 i_ll(p, pte, sizeof(pte_t) / 2, ptr);
1451 i_ori(p, pte, pte, hwmode);
1452 i_sc(p, pte, sizeof(pte_t) / 2, ptr);
1453 il_beqz(p, r, pte, label_smp_pgtable_change);
1454 /* no i_nop needed */
1455 i_lw(p, pte, 0, ptr);
830 /* no uasm_i_nop needed */
831 uasm_i_ll(p, pte, sizeof(pte_t) / 2, ptr);
832 uasm_i_ori(p, pte, pte, hwmode);
833 uasm_i_sc(p, pte, sizeof(pte_t) / 2, ptr);
834 uasm_il_beqz(p, r, pte, label_smp_pgtable_change);
835 /* no uasm_i_nop needed */
836 uasm_i_lw(p, pte, 0, ptr);
1456 } else
837 } else
1457 i_nop(p);
838 uasm_i_nop(p);
1458# else
839# else
1459 i_nop(p);
840 uasm_i_nop(p);
1460# endif
1461#else
1462# ifdef CONFIG_64BIT_PHYS_ADDR
1463 if (cpu_has_64bits)
841# endif
842#else
843# ifdef CONFIG_64BIT_PHYS_ADDR
844 if (cpu_has_64bits)
1464 i_sd(p, pte, 0, ptr);
845 uasm_i_sd(p, pte, 0, ptr);
1465 else
1466# endif
846 else
847# endif
1467 i_SW(p, pte, 0, ptr);
848 UASM_i_SW(p, pte, 0, ptr);
1468
1469# ifdef CONFIG_64BIT_PHYS_ADDR
1470 if (!cpu_has_64bits) {
849
850# ifdef CONFIG_64BIT_PHYS_ADDR
851 if (!cpu_has_64bits) {
1471 i_lw(p, pte, sizeof(pte_t) / 2, ptr);
1472 i_ori(p, pte, pte, hwmode);
1473 i_sw(p, pte, sizeof(pte_t) / 2, ptr);
1474 i_lw(p, pte, 0, ptr);
852 uasm_i_lw(p, pte, sizeof(pte_t) / 2, ptr);
853 uasm_i_ori(p, pte, pte, hwmode);
854 uasm_i_sw(p, pte, sizeof(pte_t) / 2, ptr);
855 uasm_i_lw(p, pte, 0, ptr);
1475 }
1476# endif
1477#endif
1478}
1479
1480/*
1481 * Check if PTE is present, if not then jump to LABEL. PTR points to
1482 * the page table where this PTE is located, PTE will be re-loaded
1483 * with it's original value.
1484 */
1485static void __init
856 }
857# endif
858#endif
859}
860
861/*
862 * Check if PTE is present, if not then jump to LABEL. PTR points to
863 * the page table where this PTE is located, PTE will be re-loaded
864 * with it's original value.
865 */
866static void __init
1486build_pte_present(u32 **p, struct label **l, struct reloc **r,
867build_pte_present(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
1487 unsigned int pte, unsigned int ptr, enum label_id lid)
1488{
868 unsigned int pte, unsigned int ptr, enum label_id lid)
869{
1489 i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1490 i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
1491 il_bnez(p, r, pte, lid);
870 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
871 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
872 uasm_il_bnez(p, r, pte, lid);
1492 iPTE_LW(p, l, pte, ptr);
1493}
1494
1495/* Make PTE valid, store result in PTR. */
1496static void __init
873 iPTE_LW(p, l, pte, ptr);
874}
875
876/* Make PTE valid, store result in PTR. */
877static void __init
1497build_make_valid(u32 **p, struct reloc **r, unsigned int pte,
878build_make_valid(u32 **p, struct uasm_reloc **r, unsigned int pte,
1498 unsigned int ptr)
1499{
1500 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
1501
1502 iPTE_SW(p, r, pte, ptr, mode);
1503}
1504
1505/*
1506 * Check if PTE can be written to, if not branch to LABEL. Regardless
1507 * restore PTE with value from PTR when done.
1508 */
1509static void __init
879 unsigned int ptr)
880{
881 unsigned int mode = _PAGE_VALID | _PAGE_ACCESSED;
882
883 iPTE_SW(p, r, pte, ptr, mode);
884}
885
886/*
887 * Check if PTE can be written to, if not branch to LABEL. Regardless
888 * restore PTE with value from PTR when done.
889 */
890static void __init
1510build_pte_writable(u32 **p, struct label **l, struct reloc **r,
891build_pte_writable(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
1511 unsigned int pte, unsigned int ptr, enum label_id lid)
1512{
892 unsigned int pte, unsigned int ptr, enum label_id lid)
893{
1513 i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1514 i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
1515 il_bnez(p, r, pte, lid);
894 uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
895 uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE);
896 uasm_il_bnez(p, r, pte, lid);
1516 iPTE_LW(p, l, pte, ptr);
1517}
1518
1519/* Make PTE writable, update software status bits as well, then store
1520 * at PTR.
1521 */
1522static void __init
897 iPTE_LW(p, l, pte, ptr);
898}
899
900/* Make PTE writable, update software status bits as well, then store
901 * at PTR.
902 */
903static void __init
1523build_make_write(u32 **p, struct reloc **r, unsigned int pte,
904build_make_write(u32 **p, struct uasm_reloc **r, unsigned int pte,
1524 unsigned int ptr)
1525{
1526 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
1527 | _PAGE_DIRTY);
1528
1529 iPTE_SW(p, r, pte, ptr, mode);
1530}
1531
1532/*
1533 * Check if PTE can be modified, if not branch to LABEL. Regardless
1534 * restore PTE with value from PTR when done.
1535 */
1536static void __init
905 unsigned int ptr)
906{
907 unsigned int mode = (_PAGE_ACCESSED | _PAGE_MODIFIED | _PAGE_VALID
908 | _PAGE_DIRTY);
909
910 iPTE_SW(p, r, pte, ptr, mode);
911}
912
913/*
914 * Check if PTE can be modified, if not branch to LABEL. Regardless
915 * restore PTE with value from PTR when done.
916 */
917static void __init
1537build_pte_modifiable(u32 **p, struct label **l, struct reloc **r,
918build_pte_modifiable(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
1538 unsigned int pte, unsigned int ptr, enum label_id lid)
1539{
919 unsigned int pte, unsigned int ptr, enum label_id lid)
920{
1540 i_andi(p, pte, pte, _PAGE_WRITE);
1541 il_beqz(p, r, pte, lid);
921 uasm_i_andi(p, pte, pte, _PAGE_WRITE);
922 uasm_il_beqz(p, r, pte, lid);
1542 iPTE_LW(p, l, pte, ptr);
1543}
1544
1545/*
1546 * R3000 style TLB load/store/modify handlers.
1547 */
1548
1549/*
1550 * This places the pte into ENTRYLO0 and writes it with tlbwi.
1551 * Then it returns.
1552 */
1553static void __init
1554build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
1555{
923 iPTE_LW(p, l, pte, ptr);
924}
925
926/*
927 * R3000 style TLB load/store/modify handlers.
928 */
929
930/*
931 * This places the pte into ENTRYLO0 and writes it with tlbwi.
932 * Then it returns.
933 */
934static void __init
935build_r3000_pte_reload_tlbwi(u32 **p, unsigned int pte, unsigned int tmp)
936{
1556 i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1557 i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
1558 i_tlbwi(p);
1559 i_jr(p, tmp);
1560 i_rfe(p); /* branch delay */
937 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
938 uasm_i_mfc0(p, tmp, C0_EPC); /* cp0 delay */
939 uasm_i_tlbwi(p);
940 uasm_i_jr(p, tmp);
941 uasm_i_rfe(p); /* branch delay */
1561}
1562
1563/*
1564 * This places the pte into ENTRYLO0 and writes it with tlbwi
1565 * or tlbwr as appropriate. This is because the index register
1566 * may have the probe fail bit set as a result of a trap on a
1567 * kseg2 access, i.e. without refill. Then it returns.
1568 */
1569static void __init
942}
943
944/*
945 * This places the pte into ENTRYLO0 and writes it with tlbwi
946 * or tlbwr as appropriate. This is because the index register
947 * may have the probe fail bit set as a result of a trap on a
948 * kseg2 access, i.e. without refill. Then it returns.
949 */
950static void __init
1570build_r3000_tlb_reload_write(u32 **p, struct label **l, struct reloc **r,
1571 unsigned int pte, unsigned int tmp)
951build_r3000_tlb_reload_write(u32 **p, struct uasm_label **l,
952 struct uasm_reloc **r, unsigned int pte,
953 unsigned int tmp)
1572{
954{
1573 i_mfc0(p, tmp, C0_INDEX);
1574 i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
1575 il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
1576 i_mfc0(p, tmp, C0_EPC); /* branch delay */
1577 i_tlbwi(p); /* cp0 delay */
1578 i_jr(p, tmp);
1579 i_rfe(p); /* branch delay */
1580 l_r3000_write_probe_fail(l, *p);
1581 i_tlbwr(p); /* cp0 delay */
1582 i_jr(p, tmp);
1583 i_rfe(p); /* branch delay */
955 uasm_i_mfc0(p, tmp, C0_INDEX);
956 uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* cp0 delay */
957 uasm_il_bltz(p, r, tmp, label_r3000_write_probe_fail); /* cp0 delay */
958 uasm_i_mfc0(p, tmp, C0_EPC); /* branch delay */
959 uasm_i_tlbwi(p); /* cp0 delay */
960 uasm_i_jr(p, tmp);
961 uasm_i_rfe(p); /* branch delay */
962 uasm_l_r3000_write_probe_fail(l, *p);
963 uasm_i_tlbwr(p); /* cp0 delay */
964 uasm_i_jr(p, tmp);
965 uasm_i_rfe(p); /* branch delay */
1584}
1585
1586static void __init
1587build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
1588 unsigned int ptr)
1589{
1590 long pgdc = (long)pgd_current;
1591
966}
967
968static void __init
969build_r3000_tlbchange_handler_head(u32 **p, unsigned int pte,
970 unsigned int ptr)
971{
972 long pgdc = (long)pgd_current;
973
1592 i_mfc0(p, pte, C0_BADVADDR);
1593 i_lui(p, ptr, rel_hi(pgdc)); /* cp0 delay */
1594 i_lw(p, ptr, rel_lo(pgdc), ptr);
1595 i_srl(p, pte, pte, 22); /* load delay */
1596 i_sll(p, pte, pte, 2);
1597 i_addu(p, ptr, ptr, pte);
1598 i_mfc0(p, pte, C0_CONTEXT);
1599 i_lw(p, ptr, 0, ptr); /* cp0 delay */
1600 i_andi(p, pte, pte, 0xffc); /* load delay */
1601 i_addu(p, ptr, ptr, pte);
1602 i_lw(p, pte, 0, ptr);
1603 i_tlbp(p); /* load delay */
974 uasm_i_mfc0(p, pte, C0_BADVADDR);
975 uasm_i_lui(p, ptr, uasm_rel_hi(pgdc)); /* cp0 delay */
976 uasm_i_lw(p, ptr, uasm_rel_lo(pgdc), ptr);
977 uasm_i_srl(p, pte, pte, 22); /* load delay */
978 uasm_i_sll(p, pte, pte, 2);
979 uasm_i_addu(p, ptr, ptr, pte);
980 uasm_i_mfc0(p, pte, C0_CONTEXT);
981 uasm_i_lw(p, ptr, 0, ptr); /* cp0 delay */
982 uasm_i_andi(p, pte, pte, 0xffc); /* load delay */
983 uasm_i_addu(p, ptr, ptr, pte);
984 uasm_i_lw(p, pte, 0, ptr);
985 uasm_i_tlbp(p); /* load delay */
1604}
1605
1606static void __init build_r3000_tlb_load_handler(void)
1607{
1608 u32 *p = handle_tlbl;
986}
987
988static void __init build_r3000_tlb_load_handler(void)
989{
990 u32 *p = handle_tlbl;
1609 struct label *l = labels;
1610 struct reloc *r = relocs;
991 struct uasm_label *l = labels;
992 struct uasm_reloc *r = relocs;
1611
1612 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1613 memset(labels, 0, sizeof(labels));
1614 memset(relocs, 0, sizeof(relocs));
1615
1616 build_r3000_tlbchange_handler_head(&p, K0, K1);
1617 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
993
994 memset(handle_tlbl, 0, sizeof(handle_tlbl));
995 memset(labels, 0, sizeof(labels));
996 memset(relocs, 0, sizeof(relocs));
997
998 build_r3000_tlbchange_handler_head(&p, K0, K1);
999 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
1618 i_nop(&p); /* load delay */
1000 uasm_i_nop(&p); /* load delay */
1619 build_make_valid(&p, &r, K0, K1);
1620 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1621
1001 build_make_valid(&p, &r, K0, K1);
1002 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1003
1622 l_nopage_tlbl(&l, p);
1623 i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1624 i_nop(&p);
1004 uasm_l_nopage_tlbl(&l, p);
1005 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1006 uasm_i_nop(&p);
1625
1626 if ((p - handle_tlbl) > FASTPATH_SIZE)
1627 panic("TLB load handler fastpath space exceeded");
1628
1007
1008 if ((p - handle_tlbl) > FASTPATH_SIZE)
1009 panic("TLB load handler fastpath space exceeded");
1010
1629 resolve_relocs(relocs, labels);
1630 pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
1631 (unsigned int)(p - handle_tlbl));
1011 uasm_resolve_relocs(relocs, labels);
1012 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1013 (unsigned int)(p - handle_tlbl));
1632
1633 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
1634}
1635
1636static void __init build_r3000_tlb_store_handler(void)
1637{
1638 u32 *p = handle_tlbs;
1014
1015 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
1016}
1017
1018static void __init build_r3000_tlb_store_handler(void)
1019{
1020 u32 *p = handle_tlbs;
1639 struct label *l = labels;
1640 struct reloc *r = relocs;
1021 struct uasm_label *l = labels;
1022 struct uasm_reloc *r = relocs;
1641
1642 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1643 memset(labels, 0, sizeof(labels));
1644 memset(relocs, 0, sizeof(relocs));
1645
1646 build_r3000_tlbchange_handler_head(&p, K0, K1);
1647 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1023
1024 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1025 memset(labels, 0, sizeof(labels));
1026 memset(relocs, 0, sizeof(relocs));
1027
1028 build_r3000_tlbchange_handler_head(&p, K0, K1);
1029 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1648 i_nop(&p); /* load delay */
1030 uasm_i_nop(&p); /* load delay */
1649 build_make_write(&p, &r, K0, K1);
1650 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1651
1031 build_make_write(&p, &r, K0, K1);
1032 build_r3000_tlb_reload_write(&p, &l, &r, K0, K1);
1033
1652 l_nopage_tlbs(&l, p);
1653 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1654 i_nop(&p);
1034 uasm_l_nopage_tlbs(&l, p);
1035 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1036 uasm_i_nop(&p);
1655
1656 if ((p - handle_tlbs) > FASTPATH_SIZE)
1657 panic("TLB store handler fastpath space exceeded");
1658
1037
1038 if ((p - handle_tlbs) > FASTPATH_SIZE)
1039 panic("TLB store handler fastpath space exceeded");
1040
1659 resolve_relocs(relocs, labels);
1660 pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
1661 (unsigned int)(p - handle_tlbs));
1041 uasm_resolve_relocs(relocs, labels);
1042 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1043 (unsigned int)(p - handle_tlbs));
1662
1663 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
1664}
1665
1666static void __init build_r3000_tlb_modify_handler(void)
1667{
1668 u32 *p = handle_tlbm;
1044
1045 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
1046}
1047
1048static void __init build_r3000_tlb_modify_handler(void)
1049{
1050 u32 *p = handle_tlbm;
1669 struct label *l = labels;
1670 struct reloc *r = relocs;
1051 struct uasm_label *l = labels;
1052 struct uasm_reloc *r = relocs;
1671
1672 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1673 memset(labels, 0, sizeof(labels));
1674 memset(relocs, 0, sizeof(relocs));
1675
1676 build_r3000_tlbchange_handler_head(&p, K0, K1);
1677 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1053
1054 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1055 memset(labels, 0, sizeof(labels));
1056 memset(relocs, 0, sizeof(relocs));
1057
1058 build_r3000_tlbchange_handler_head(&p, K0, K1);
1059 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1678 i_nop(&p); /* load delay */
1060 uasm_i_nop(&p); /* load delay */
1679 build_make_write(&p, &r, K0, K1);
1680 build_r3000_pte_reload_tlbwi(&p, K0, K1);
1681
1061 build_make_write(&p, &r, K0, K1);
1062 build_r3000_pte_reload_tlbwi(&p, K0, K1);
1063
1682 l_nopage_tlbm(&l, p);
1683 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1684 i_nop(&p);
1064 uasm_l_nopage_tlbm(&l, p);
1065 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1066 uasm_i_nop(&p);
1685
1686 if ((p - handle_tlbm) > FASTPATH_SIZE)
1687 panic("TLB modify handler fastpath space exceeded");
1688
1067
1068 if ((p - handle_tlbm) > FASTPATH_SIZE)
1069 panic("TLB modify handler fastpath space exceeded");
1070
1689 resolve_relocs(relocs, labels);
1690 pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
1691 (unsigned int)(p - handle_tlbm));
1071 uasm_resolve_relocs(relocs, labels);
1072 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1073 (unsigned int)(p - handle_tlbm));
1692
1693 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
1694}
1695
1696/*
1697 * R4000 style TLB load/store/modify handlers.
1698 */
1699static void __init
1074
1075 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
1076}
1077
1078/*
1079 * R4000 style TLB load/store/modify handlers.
1080 */
1081static void __init
1700build_r4000_tlbchange_handler_head(u32 **p, struct label **l,
1701 struct reloc **r, unsigned int pte,
1082build_r4000_tlbchange_handler_head(u32 **p, struct uasm_label **l,
1083 struct uasm_reloc **r, unsigned int pte,
1702 unsigned int ptr)
1703{
1704#ifdef CONFIG_64BIT
1705 build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
1706#else
1707 build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
1708#endif
1709
1084 unsigned int ptr)
1085{
1086#ifdef CONFIG_64BIT
1087 build_get_pmde64(p, l, r, pte, ptr); /* get pmd in ptr */
1088#else
1089 build_get_pgde32(p, pte, ptr); /* get pgd in ptr */
1090#endif
1091
1710 i_MFC0(p, pte, C0_BADVADDR);
1711 i_LW(p, ptr, 0, ptr);
1712 i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
1713 i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
1714 i_ADDU(p, ptr, ptr, pte);
1092 UASM_i_MFC0(p, pte, C0_BADVADDR);
1093 UASM_i_LW(p, ptr, 0, ptr);
1094 UASM_i_SRL(p, pte, pte, PAGE_SHIFT + PTE_ORDER - PTE_T_LOG2);
1095 uasm_i_andi(p, pte, pte, (PTRS_PER_PTE - 1) << PTE_T_LOG2);
1096 UASM_i_ADDU(p, ptr, ptr, pte);
1715
1716#ifdef CONFIG_SMP
1097
1098#ifdef CONFIG_SMP
1717 l_smp_pgtable_change(l, *p);
1718# endif
1099 uasm_l_smp_pgtable_change(l, *p);
1100#endif
1719 iPTE_LW(p, l, pte, ptr); /* get even pte */
1720 if (!m4kc_tlbp_war())
1721 build_tlb_probe_entry(p);
1722}
1723
1724static void __init
1101 iPTE_LW(p, l, pte, ptr); /* get even pte */
1102 if (!m4kc_tlbp_war())
1103 build_tlb_probe_entry(p);
1104}
1105
1106static void __init
1725build_r4000_tlbchange_handler_tail(u32 **p, struct label **l,
1726 struct reloc **r, unsigned int tmp,
1107build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
1108 struct uasm_reloc **r, unsigned int tmp,
1727 unsigned int ptr)
1728{
1109 unsigned int ptr)
1110{
1729 i_ori(p, ptr, ptr, sizeof(pte_t));
1730 i_xori(p, ptr, ptr, sizeof(pte_t));
1111 uasm_i_ori(p, ptr, ptr, sizeof(pte_t));
1112 uasm_i_xori(p, ptr, ptr, sizeof(pte_t));
1731 build_update_entries(p, tmp, ptr);
1732 build_tlb_write_entry(p, l, r, tlb_indexed);
1113 build_update_entries(p, tmp, ptr);
1114 build_tlb_write_entry(p, l, r, tlb_indexed);
1733 l_leave(l, *p);
1734 i_eret(p); /* return from trap */
1115 uasm_l_leave(l, *p);
1116 uasm_i_eret(p); /* return from trap */
1735
1736#ifdef CONFIG_64BIT
1737 build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
1738#endif
1739}
1740
1741static void __init build_r4000_tlb_load_handler(void)
1742{
1743 u32 *p = handle_tlbl;
1117
1118#ifdef CONFIG_64BIT
1119 build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
1120#endif
1121}
1122
1123static void __init build_r4000_tlb_load_handler(void)
1124{
1125 u32 *p = handle_tlbl;
1744 struct label *l = labels;
1745 struct reloc *r = relocs;
1126 struct uasm_label *l = labels;
1127 struct uasm_reloc *r = relocs;
1746
1747 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1748 memset(labels, 0, sizeof(labels));
1749 memset(relocs, 0, sizeof(relocs));
1750
1751 if (bcm1250_m3_war()) {
1128
1129 memset(handle_tlbl, 0, sizeof(handle_tlbl));
1130 memset(labels, 0, sizeof(labels));
1131 memset(relocs, 0, sizeof(relocs));
1132
1133 if (bcm1250_m3_war()) {
1752 i_MFC0(&p, K0, C0_BADVADDR);
1753 i_MFC0(&p, K1, C0_ENTRYHI);
1754 i_xor(&p, K0, K0, K1);
1755 i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
1756 il_bnez(&p, &r, K0, label_leave);
1757 /* No need for i_nop */
1134 UASM_i_MFC0(&p, K0, C0_BADVADDR);
1135 UASM_i_MFC0(&p, K1, C0_ENTRYHI);
1136 uasm_i_xor(&p, K0, K0, K1);
1137 UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
1138 uasm_il_bnez(&p, &r, K0, label_leave);
1139 /* No need for uasm_i_nop */
1758 }
1759
1760 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1761 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
1762 if (m4kc_tlbp_war())
1763 build_tlb_probe_entry(&p);
1764 build_make_valid(&p, &r, K0, K1);
1765 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1766
1140 }
1141
1142 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1143 build_pte_present(&p, &l, &r, K0, K1, label_nopage_tlbl);
1144 if (m4kc_tlbp_war())
1145 build_tlb_probe_entry(&p);
1146 build_make_valid(&p, &r, K0, K1);
1147 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1148
1767 l_nopage_tlbl(&l, p);
1768 i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1769 i_nop(&p);
1149 uasm_l_nopage_tlbl(&l, p);
1150 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_0 & 0x0fffffff);
1151 uasm_i_nop(&p);
1770
1771 if ((p - handle_tlbl) > FASTPATH_SIZE)
1772 panic("TLB load handler fastpath space exceeded");
1773
1152
1153 if ((p - handle_tlbl) > FASTPATH_SIZE)
1154 panic("TLB load handler fastpath space exceeded");
1155
1774 resolve_relocs(relocs, labels);
1775 pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
1776 (unsigned int)(p - handle_tlbl));
1156 uasm_resolve_relocs(relocs, labels);
1157 pr_debug("Wrote TLB load handler fastpath (%u instructions).\n",
1158 (unsigned int)(p - handle_tlbl));
1777
1778 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
1779}
1780
1781static void __init build_r4000_tlb_store_handler(void)
1782{
1783 u32 *p = handle_tlbs;
1159
1160 dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
1161}
1162
1163static void __init build_r4000_tlb_store_handler(void)
1164{
1165 u32 *p = handle_tlbs;
1784 struct label *l = labels;
1785 struct reloc *r = relocs;
1166 struct uasm_label *l = labels;
1167 struct uasm_reloc *r = relocs;
1786
1787 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1788 memset(labels, 0, sizeof(labels));
1789 memset(relocs, 0, sizeof(relocs));
1790
1791 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1792 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1793 if (m4kc_tlbp_war())
1794 build_tlb_probe_entry(&p);
1795 build_make_write(&p, &r, K0, K1);
1796 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1797
1168
1169 memset(handle_tlbs, 0, sizeof(handle_tlbs));
1170 memset(labels, 0, sizeof(labels));
1171 memset(relocs, 0, sizeof(relocs));
1172
1173 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1174 build_pte_writable(&p, &l, &r, K0, K1, label_nopage_tlbs);
1175 if (m4kc_tlbp_war())
1176 build_tlb_probe_entry(&p);
1177 build_make_write(&p, &r, K0, K1);
1178 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1179
1798 l_nopage_tlbs(&l, p);
1799 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1800 i_nop(&p);
1180 uasm_l_nopage_tlbs(&l, p);
1181 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1182 uasm_i_nop(&p);
1801
1802 if ((p - handle_tlbs) > FASTPATH_SIZE)
1803 panic("TLB store handler fastpath space exceeded");
1804
1183
1184 if ((p - handle_tlbs) > FASTPATH_SIZE)
1185 panic("TLB store handler fastpath space exceeded");
1186
1805 resolve_relocs(relocs, labels);
1806 pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
1807 (unsigned int)(p - handle_tlbs));
1187 uasm_resolve_relocs(relocs, labels);
1188 pr_debug("Wrote TLB store handler fastpath (%u instructions).\n",
1189 (unsigned int)(p - handle_tlbs));
1808
1809 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
1810}
1811
1812static void __init build_r4000_tlb_modify_handler(void)
1813{
1814 u32 *p = handle_tlbm;
1190
1191 dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
1192}
1193
1194static void __init build_r4000_tlb_modify_handler(void)
1195{
1196 u32 *p = handle_tlbm;
1815 struct label *l = labels;
1816 struct reloc *r = relocs;
1197 struct uasm_label *l = labels;
1198 struct uasm_reloc *r = relocs;
1817
1818 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1819 memset(labels, 0, sizeof(labels));
1820 memset(relocs, 0, sizeof(relocs));
1821
1822 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1823 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1824 if (m4kc_tlbp_war())
1825 build_tlb_probe_entry(&p);
1826 /* Present and writable bits set, set accessed and dirty bits. */
1827 build_make_write(&p, &r, K0, K1);
1828 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1829
1199
1200 memset(handle_tlbm, 0, sizeof(handle_tlbm));
1201 memset(labels, 0, sizeof(labels));
1202 memset(relocs, 0, sizeof(relocs));
1203
1204 build_r4000_tlbchange_handler_head(&p, &l, &r, K0, K1);
1205 build_pte_modifiable(&p, &l, &r, K0, K1, label_nopage_tlbm);
1206 if (m4kc_tlbp_war())
1207 build_tlb_probe_entry(&p);
1208 /* Present and writable bits set, set accessed and dirty bits. */
1209 build_make_write(&p, &r, K0, K1);
1210 build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
1211
1830 l_nopage_tlbm(&l, p);
1831 i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1832 i_nop(&p);
1212 uasm_l_nopage_tlbm(&l, p);
1213 uasm_i_j(&p, (unsigned long)tlb_do_page_fault_1 & 0x0fffffff);
1214 uasm_i_nop(&p);
1833
1834 if ((p - handle_tlbm) > FASTPATH_SIZE)
1835 panic("TLB modify handler fastpath space exceeded");
1836
1215
1216 if ((p - handle_tlbm) > FASTPATH_SIZE)
1217 panic("TLB modify handler fastpath space exceeded");
1218
1837 resolve_relocs(relocs, labels);
1838 pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
1839 (unsigned int)(p - handle_tlbm));
1219 uasm_resolve_relocs(relocs, labels);
1220 pr_debug("Wrote TLB modify handler fastpath (%u instructions).\n",
1221 (unsigned int)(p - handle_tlbm));
1840
1841 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
1842}
1843
1844void __init build_tlb_refill_handler(void)
1845{
1846 /*
1847 * The refill handler is generated per-CPU, multi-node systems

--- 51 unchanged lines hidden --- 		1222
1223 dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
1224}
1225
1226void __init build_tlb_refill_handler(void)
1227{
1228 /*
1229 * The refill handler is generated per-CPU, multi-node systems

--- 51 unchanged lines hidden ---