xref: /linux/arch/parisc/kernel/pacache.S (revision f3d9478b2ce468c3115b02ecae7e975990697f15) 
1/*
2 *  PARISC TLB and cache flushing support
3 *  Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
4 *  Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
5 *  Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
6 *
7 *    This program is free software; you can redistribute it and/or modify
8 *    it under the terms of the GNU General Public License as published by
9 *    the Free Software Foundation; either version 2, or (at your option)
10 *    any later version.
11 *
12 *    This program is distributed in the hope that it will be useful,
13 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
14 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 *    GNU General Public License for more details.
16 *
17 *    You should have received a copy of the GNU General Public License
18 *    along with this program; if not, write to the Free Software
19 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20 */
21
22/*
23 * NOTE: fdc,fic, and pdc instructions that use base register modification
24 *       should only use index and base registers that are not shadowed,
25 *       so that the fast path emulation in the non access miss handler
26 *       can be used.
27 */
28
29#ifdef CONFIG_64BIT
30#define ADDIB	addib,*
31#define CMPB	cmpb,*
32#define ANDCM	andcm,*
33
34	.level	2.0w
35#else
36#define ADDIB	addib,
37#define CMPB	cmpb,
38#define ANDCM	andcm
39
40	.level	2.0
41#endif
42
43#include <linux/config.h>
44
45#include <asm/psw.h>
46#include <asm/assembly.h>
47#include <asm/pgtable.h>
48#include <asm/cache.h>
49
50	.text
51	.align	128
52
53	.export flush_tlb_all_local,code
54
55flush_tlb_all_local:
56	.proc
57	.callinfo NO_CALLS
58	.entry
59
60	/*
61	 * The pitlbe and pdtlbe instructions should only be used to
62	 * flush the entire tlb. Also, there needs to be no intervening
63	 * tlb operations, e.g. tlb misses, so the operation needs
64	 * to happen in real mode with all interruptions disabled.
65	 */
66
67	/* pcxt_ssm_bug	- relied upon translation! PA 2.0 Arch. F-4 and F-5 */
68	rsm		PSW_SM_I, %r19		/* save I-bit state */
69	load32		PA(1f), %r1
70	nop
71	nop
72	nop
73	nop
74	nop
75
76	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
77	mtctl		%r0, %cr17		/* Clear IIASQ tail */
78	mtctl		%r0, %cr17		/* Clear IIASQ head */
79	mtctl		%r1, %cr18		/* IIAOQ head */
80	ldo		4(%r1), %r1
81	mtctl		%r1, %cr18		/* IIAOQ tail */
82	load32		REAL_MODE_PSW, %r1
83	mtctl           %r1, %ipsw
84	rfi
85	nop
86
871:      load32		PA(cache_info), %r1
88
89	/* Flush Instruction Tlb */
90
91	LDREG		ITLB_SID_BASE(%r1), %r20
92	LDREG		ITLB_SID_STRIDE(%r1), %r21
93	LDREG		ITLB_SID_COUNT(%r1), %r22
94	LDREG		ITLB_OFF_BASE(%r1), %arg0
95	LDREG		ITLB_OFF_STRIDE(%r1), %arg1
96	LDREG		ITLB_OFF_COUNT(%r1), %arg2
97	LDREG		ITLB_LOOP(%r1), %arg3
98
99	ADDIB=		-1, %arg3, fitoneloop	/* Preadjust and test */
100	movb,<,n	%arg3, %r31, fitdone	/* If loop < 0, skip */
101	copy		%arg0, %r28		/* Init base addr */
102
103fitmanyloop:					/* Loop if LOOP >= 2 */
104	mtsp		%r20, %sr1
105	add		%r21, %r20, %r20	/* increment space */
106	copy		%arg2, %r29		/* Init middle loop count */
107
108fitmanymiddle:					/* Loop if LOOP >= 2 */
109	ADDIB>		-1, %r31, fitmanymiddle	/* Adjusted inner loop decr */
110	pitlbe		0(%sr1, %r28)
111	pitlbe,m	%arg1(%sr1, %r28)	/* Last pitlbe and addr adjust */
112	ADDIB>		-1, %r29, fitmanymiddle	/* Middle loop decr */
113	copy		%arg3, %r31		/* Re-init inner loop count */
114
115	movb,tr		%arg0, %r28, fitmanyloop /* Re-init base addr */
116	ADDIB<=,n	-1, %r22, fitdone	/* Outer loop count decr */
117
118fitoneloop:					/* Loop if LOOP = 1 */
119	mtsp		%r20, %sr1
120	copy		%arg0, %r28		/* init base addr */
121	copy		%arg2, %r29		/* init middle loop count */
122
123fitonemiddle:					/* Loop if LOOP = 1 */
124	ADDIB>		-1, %r29, fitonemiddle	/* Middle loop count decr */
125	pitlbe,m	%arg1(%sr1, %r28)	/* pitlbe for one loop */
126
127	ADDIB>		-1, %r22, fitoneloop	/* Outer loop count decr */
128	add		%r21, %r20, %r20		/* increment space */
129
130fitdone:
131
132	/* Flush Data Tlb */
133
134	LDREG		DTLB_SID_BASE(%r1), %r20
135	LDREG		DTLB_SID_STRIDE(%r1), %r21
136	LDREG		DTLB_SID_COUNT(%r1), %r22
137	LDREG		DTLB_OFF_BASE(%r1), %arg0
138	LDREG		DTLB_OFF_STRIDE(%r1), %arg1
139	LDREG		DTLB_OFF_COUNT(%r1), %arg2
140	LDREG		DTLB_LOOP(%r1), %arg3
141
142	ADDIB=		-1, %arg3, fdtoneloop	/* Preadjust and test */
143	movb,<,n	%arg3, %r31, fdtdone	/* If loop < 0, skip */
144	copy		%arg0, %r28		/* Init base addr */
145
146fdtmanyloop:					/* Loop if LOOP >= 2 */
147	mtsp		%r20, %sr1
148	add		%r21, %r20, %r20	/* increment space */
149	copy		%arg2, %r29		/* Init middle loop count */
150
151fdtmanymiddle:					/* Loop if LOOP >= 2 */
152	ADDIB>		-1, %r31, fdtmanymiddle	/* Adjusted inner loop decr */
153	pdtlbe		0(%sr1, %r28)
154	pdtlbe,m	%arg1(%sr1, %r28)	/* Last pdtlbe and addr adjust */
155	ADDIB>		-1, %r29, fdtmanymiddle	/* Middle loop decr */
156	copy		%arg3, %r31		/* Re-init inner loop count */
157
158	movb,tr		%arg0, %r28, fdtmanyloop /* Re-init base addr */
159	ADDIB<=,n	-1, %r22,fdtdone	/* Outer loop count decr */
160
161fdtoneloop:					/* Loop if LOOP = 1 */
162	mtsp		%r20, %sr1
163	copy		%arg0, %r28		/* init base addr */
164	copy		%arg2, %r29		/* init middle loop count */
165
166fdtonemiddle:					/* Loop if LOOP = 1 */
167	ADDIB>		-1, %r29, fdtonemiddle	/* Middle loop count decr */
168	pdtlbe,m	%arg1(%sr1, %r28)	/* pdtlbe for one loop */
169
170	ADDIB>		-1, %r22, fdtoneloop	/* Outer loop count decr */
171	add		%r21, %r20, %r20	/* increment space */
172
173
174fdtdone:
175	/*
176	 * Switch back to virtual mode
177	 */
178	/* pcxt_ssm_bug */
179	rsm		PSW_SM_I, %r0
180	load32		2f, %r1
181	nop
182	nop
183	nop
184	nop
185	nop
186
187	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
188	mtctl		%r0, %cr17		/* Clear IIASQ tail */
189	mtctl		%r0, %cr17		/* Clear IIASQ head */
190	mtctl		%r1, %cr18		/* IIAOQ head */
191	ldo		4(%r1), %r1
192	mtctl		%r1, %cr18		/* IIAOQ tail */
193	load32		KERNEL_PSW, %r1
194	or		%r1, %r19, %r1	/* I-bit to state on entry */
195	mtctl		%r1, %ipsw	/* restore I-bit (entire PSW) */
196	rfi
197	nop
198
1992:      bv		%r0(%r2)
200	nop
201
202	.exit
203	.procend
204
205	.export flush_instruction_cache_local,code
206	.import cache_info,data
207
208flush_instruction_cache_local:
209	.proc
210	.callinfo NO_CALLS
211	.entry
212
213	mtsp		%r0, %sr1
214	load32		cache_info, %r1
215
216	/* Flush Instruction Cache */
217
218	LDREG		ICACHE_BASE(%r1), %arg0
219	LDREG		ICACHE_STRIDE(%r1), %arg1
220	LDREG		ICACHE_COUNT(%r1), %arg2
221	LDREG		ICACHE_LOOP(%r1), %arg3
222	rsm             PSW_SM_I, %r22		/* No mmgt ops during loop*/
223	ADDIB=		-1, %arg3, fioneloop	/* Preadjust and test */
224	movb,<,n	%arg3, %r31, fisync	/* If loop < 0, do sync */
225
226fimanyloop:					/* Loop if LOOP >= 2 */
227	ADDIB>		-1, %r31, fimanyloop	/* Adjusted inner loop decr */
228	fice            %r0(%sr1, %arg0)
229	fice,m		%arg1(%sr1, %arg0)	/* Last fice and addr adjust */
230	movb,tr		%arg3, %r31, fimanyloop	/* Re-init inner loop count */
231	ADDIB<=,n	-1, %arg2, fisync	/* Outer loop decr */
232
233fioneloop:					/* Loop if LOOP = 1 */
234	ADDIB>		-1, %arg2, fioneloop	/* Outer loop count decr */
235	fice,m		%arg1(%sr1, %arg0)	/* Fice for one loop */
236
237fisync:
238	sync
239	mtsm		%r22			/* restore I-bit */
240	bv		%r0(%r2)
241	nop
242	.exit
243
244	.procend
245
246	.export flush_data_cache_local, code
247	.import cache_info, data
248
249flush_data_cache_local:
250	.proc
251	.callinfo NO_CALLS
252	.entry
253
254	mtsp		%r0, %sr1
255	load32 		cache_info, %r1
256
257	/* Flush Data Cache */
258
259	LDREG		DCACHE_BASE(%r1), %arg0
260	LDREG		DCACHE_STRIDE(%r1), %arg1
261	LDREG		DCACHE_COUNT(%r1), %arg2
262	LDREG		DCACHE_LOOP(%r1), %arg3
263	rsm		PSW_SM_I, %r22
264	ADDIB=		-1, %arg3, fdoneloop	/* Preadjust and test */
265	movb,<,n	%arg3, %r31, fdsync	/* If loop < 0, do sync */
266
267fdmanyloop:					/* Loop if LOOP >= 2 */
268	ADDIB>		-1, %r31, fdmanyloop	/* Adjusted inner loop decr */
269	fdce		%r0(%sr1, %arg0)
270	fdce,m		%arg1(%sr1, %arg0)	/* Last fdce and addr adjust */
271	movb,tr		%arg3, %r31, fdmanyloop	/* Re-init inner loop count */
272	ADDIB<=,n	-1, %arg2, fdsync	/* Outer loop decr */
273
274fdoneloop:					/* Loop if LOOP = 1 */
275	ADDIB>		-1, %arg2, fdoneloop	/* Outer loop count decr */
276	fdce,m		%arg1(%sr1, %arg0)	/* Fdce for one loop */
277
278fdsync:
279	syncdma
280	sync
281	mtsm		%r22			/* restore I-bit */
282	bv		%r0(%r2)
283	nop
284	.exit
285
286	.procend
287
288	.export copy_user_page_asm,code
289	.align	16
290
291copy_user_page_asm:
292	.proc
293	.callinfo NO_CALLS
294	.entry
295
296#ifdef CONFIG_64BIT
297	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
298	 * Unroll the loop by hand and arrange insn appropriately.
299	 * GCC probably can do this just as well.
300	 */
301
302	ldd		0(%r25), %r19
303	ldi		ASM_PAGE_SIZE_DIV128, %r1
304
305	ldw		64(%r25), %r0		/* prefetch 1 cacheline ahead */
306	ldw		128(%r25), %r0		/* prefetch 2 */
307
3081:	ldd		8(%r25), %r20
309	ldw		192(%r25), %r0		/* prefetch 3 */
310	ldw		256(%r25), %r0		/* prefetch 4 */
311
312	ldd		16(%r25), %r21
313	ldd		24(%r25), %r22
314	std		%r19, 0(%r26)
315	std		%r20, 8(%r26)
316
317	ldd		32(%r25), %r19
318	ldd		40(%r25), %r20
319	std		%r21, 16(%r26)
320	std		%r22, 24(%r26)
321
322	ldd		48(%r25), %r21
323	ldd		56(%r25), %r22
324	std		%r19, 32(%r26)
325	std		%r20, 40(%r26)
326
327	ldd		64(%r25), %r19
328	ldd		72(%r25), %r20
329	std		%r21, 48(%r26)
330	std		%r22, 56(%r26)
331
332	ldd		80(%r25), %r21
333	ldd		88(%r25), %r22
334	std		%r19, 64(%r26)
335	std		%r20, 72(%r26)
336
337	ldd		 96(%r25), %r19
338	ldd		104(%r25), %r20
339	std		%r21, 80(%r26)
340	std		%r22, 88(%r26)
341
342	ldd		112(%r25), %r21
343	ldd		120(%r25), %r22
344	std		%r19, 96(%r26)
345	std		%r20, 104(%r26)
346
347	ldo		128(%r25), %r25
348	std		%r21, 112(%r26)
349	std		%r22, 120(%r26)
350	ldo		128(%r26), %r26
351
352	/* conditional branches nullify on forward taken branch, and on
353	 * non-taken backward branch. Note that .+4 is a backwards branch.
354	 * The ldd should only get executed if the branch is taken.
355	 */
356	ADDIB>,n	-1, %r1, 1b		/* bundle 10 */
357	ldd		0(%r25), %r19		/* start next loads */
358
359#else
360
361	/*
362	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
363	 * bundles (very restricted rules for bundling).
364	 * Note that until (if) we start saving
365	 * the full 64 bit register values on interrupt, we can't
366	 * use ldd/std on a 32 bit kernel.
367	 */
368	ldw		0(%r25), %r19
369	ldi		ASM_PAGE_SIZE_DIV64, %r1
370
3711:
372	ldw		4(%r25), %r20
373	ldw		8(%r25), %r21
374	ldw		12(%r25), %r22
375	stw		%r19, 0(%r26)
376	stw		%r20, 4(%r26)
377	stw		%r21, 8(%r26)
378	stw		%r22, 12(%r26)
379	ldw		16(%r25), %r19
380	ldw		20(%r25), %r20
381	ldw		24(%r25), %r21
382	ldw		28(%r25), %r22
383	stw		%r19, 16(%r26)
384	stw		%r20, 20(%r26)
385	stw		%r21, 24(%r26)
386	stw		%r22, 28(%r26)
387	ldw		32(%r25), %r19
388	ldw		36(%r25), %r20
389	ldw		40(%r25), %r21
390	ldw		44(%r25), %r22
391	stw		%r19, 32(%r26)
392	stw		%r20, 36(%r26)
393	stw		%r21, 40(%r26)
394	stw		%r22, 44(%r26)
395	ldw		48(%r25), %r19
396	ldw		52(%r25), %r20
397	ldw		56(%r25), %r21
398	ldw		60(%r25), %r22
399	stw		%r19, 48(%r26)
400	stw		%r20, 52(%r26)
401	ldo		64(%r25), %r25
402	stw		%r21, 56(%r26)
403	stw		%r22, 60(%r26)
404	ldo		64(%r26), %r26
405	ADDIB>,n	-1, %r1, 1b
406	ldw		0(%r25), %r19
407#endif
408	bv		%r0(%r2)
409	nop
410	.exit
411
412	.procend
413
414/*
415 * NOTE: Code in clear_user_page has a hard coded dependency on the
416 *       maximum alias boundary being 4 Mb. We've been assured by the
417 *       parisc chip designers that there will not ever be a parisc
418 *       chip with a larger alias boundary (Never say never :-) ).
419 *
420 *       Subtle: the dtlb miss handlers support the temp alias region by
421 *       "knowing" that if a dtlb miss happens within the temp alias
422 *       region it must have occurred while in clear_user_page. Since
423 *       this routine makes use of processor local translations, we
424 *       don't want to insert them into the kernel page table. Instead,
425 *       we load up some general registers (they need to be registers
426 *       which aren't shadowed) with the physical page numbers (preshifted
427 *       for tlb insertion) needed to insert the translations. When we
428 *       miss on the translation, the dtlb miss handler inserts the
429 *       translation into the tlb using these values:
430 *
431 *          %r26 physical page (shifted for tlb insert) of "to" translation
432 *          %r23 physical page (shifted for tlb insert) of "from" translation
433 */
434
435#if 0
436
437	/*
438	 * We can't do this since copy_user_page is used to bring in
439	 * file data that might have instructions. Since the data would
440	 * then need to be flushed out so the i-fetch can see it, it
441	 * makes more sense to just copy through the kernel translation
442	 * and flush it.
443	 *
444	 * I'm still keeping this around because it may be possible to
445	 * use it if more information is passed into copy_user_page().
446	 * Have to do some measurements to see if it is worthwhile to
447	 * lobby for such a change.
448	 */
449
450	.export copy_user_page_asm,code
451
452copy_user_page_asm:
453	.proc
454	.callinfo NO_CALLS
455	.entry
456
457	ldil		L%(__PAGE_OFFSET), %r1
458	sub		%r26, %r1, %r26
459	sub		%r25, %r1, %r23		/* move physical addr into non shadowed reg */
460
461	ldil		L%(TMPALIAS_MAP_START), %r28
462	/* FIXME for different page sizes != 4k */
463#ifdef CONFIG_64BIT
464	extrd,u		%r26,56,32, %r26		/* convert phys addr to tlb insert format */
465	extrd,u		%r23,56,32, %r23		/* convert phys addr to tlb insert format */
466	depd		%r24,63,22, %r28		/* Form aliased virtual address 'to' */
467	depdi		0, 63,12, %r28		/* Clear any offset bits */
468	copy		%r28, %r29
469	depdi		1, 41,1, %r29		/* Form aliased virtual address 'from' */
470#else
471	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
472	extrw,u		%r23, 24,25, %r23	/* convert phys addr to tlb insert format */
473	depw		%r24, 31,22, %r28	/* Form aliased virtual address 'to' */
474	depwi		0, 31,12, %r28		/* Clear any offset bits */
475	copy		%r28, %r29
476	depwi		1, 9,1, %r29		/* Form aliased virtual address 'from' */
477#endif
478
479	/* Purge any old translations */
480
481	pdtlb		0(%r28)
482	pdtlb		0(%r29)
483
484	ldi		64, %r1
485
486	/*
487	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
488	 * bundles (very restricted rules for bundling). It probably
489	 * does OK on PCXU and better, but we could do better with
490	 * ldd/std instructions. Note that until (if) we start saving
491	 * the full 64 bit register values on interrupt, we can't
492	 * use ldd/std on a 32 bit kernel.
493	 */
494
495
4961:
497	ldw		0(%r29), %r19
498	ldw		4(%r29), %r20
499	ldw		8(%r29), %r21
500	ldw		12(%r29), %r22
501	stw		%r19, 0(%r28)
502	stw		%r20, 4(%r28)
503	stw		%r21, 8(%r28)
504	stw		%r22, 12(%r28)
505	ldw		16(%r29), %r19
506	ldw		20(%r29), %r20
507	ldw		24(%r29), %r21
508	ldw		28(%r29), %r22
509	stw		%r19, 16(%r28)
510	stw		%r20, 20(%r28)
511	stw		%r21, 24(%r28)
512	stw		%r22, 28(%r28)
513	ldw		32(%r29), %r19
514	ldw		36(%r29), %r20
515	ldw		40(%r29), %r21
516	ldw		44(%r29), %r22
517	stw		%r19, 32(%r28)
518	stw		%r20, 36(%r28)
519	stw		%r21, 40(%r28)
520	stw		%r22, 44(%r28)
521	ldw		48(%r29), %r19
522	ldw		52(%r29), %r20
523	ldw		56(%r29), %r21
524	ldw		60(%r29), %r22
525	stw		%r19, 48(%r28)
526	stw		%r20, 52(%r28)
527	stw		%r21, 56(%r28)
528	stw		%r22, 60(%r28)
529	ldo		64(%r28), %r28
530	ADDIB>		-1, %r1,1b
531	ldo		64(%r29), %r29
532
533	bv		%r0(%r2)
534	nop
535	.exit
536
537	.procend
538#endif
539
540	.export __clear_user_page_asm,code
541
542__clear_user_page_asm:
543	.proc
544	.callinfo NO_CALLS
545	.entry
546
547	tophys_r1	%r26
548
549	ldil		L%(TMPALIAS_MAP_START), %r28
550#ifdef CONFIG_64BIT
551#if (TMPALIAS_MAP_START >= 0x80000000)
552	depdi		0, 31,32, %r28		/* clear any sign extension */
553	/* FIXME: page size dependend */
554#endif
555	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
556	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
557	depdi		0, 63,12, %r28		/* Clear any offset bits */
558#else
559	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
560	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
561	depwi		0, 31,12, %r28		/* Clear any offset bits */
562#endif
563
564	/* Purge any old translation */
565
566	pdtlb		0(%r28)
567
568#ifdef CONFIG_64BIT
569	ldi		ASM_PAGE_SIZE_DIV128, %r1
570
571	/* PREFETCH (Write) has not (yet) been proven to help here */
572	/* #define	PREFETCHW_OP	ldd		256(%0), %r0 */
573
5741:	std		%r0, 0(%r28)
575	std		%r0, 8(%r28)
576	std		%r0, 16(%r28)
577	std		%r0, 24(%r28)
578	std		%r0, 32(%r28)
579	std		%r0, 40(%r28)
580	std		%r0, 48(%r28)
581	std		%r0, 56(%r28)
582	std		%r0, 64(%r28)
583	std		%r0, 72(%r28)
584	std		%r0, 80(%r28)
585	std		%r0, 88(%r28)
586	std		%r0, 96(%r28)
587	std		%r0, 104(%r28)
588	std		%r0, 112(%r28)
589	std		%r0, 120(%r28)
590	ADDIB>		-1, %r1, 1b
591	ldo		128(%r28), %r28
592
593#else	/* ! CONFIG_64BIT */
594	ldi		ASM_PAGE_SIZE_DIV64, %r1
595
5961:
597	stw		%r0, 0(%r28)
598	stw		%r0, 4(%r28)
599	stw		%r0, 8(%r28)
600	stw		%r0, 12(%r28)
601	stw		%r0, 16(%r28)
602	stw		%r0, 20(%r28)
603	stw		%r0, 24(%r28)
604	stw		%r0, 28(%r28)
605	stw		%r0, 32(%r28)
606	stw		%r0, 36(%r28)
607	stw		%r0, 40(%r28)
608	stw		%r0, 44(%r28)
609	stw		%r0, 48(%r28)
610	stw		%r0, 52(%r28)
611	stw		%r0, 56(%r28)
612	stw		%r0, 60(%r28)
613	ADDIB>		-1, %r1, 1b
614	ldo		64(%r28), %r28
615#endif	/* CONFIG_64BIT */
616
617	bv		%r0(%r2)
618	nop
619	.exit
620
621	.procend
622
623	.export flush_kernel_dcache_page_asm
624
625flush_kernel_dcache_page_asm:
626	.proc
627	.callinfo NO_CALLS
628	.entry
629
630	ldil		L%dcache_stride, %r1
631	ldw		R%dcache_stride(%r1), %r23
632
633#ifdef CONFIG_64BIT
634	depdi,z		1, 63-PAGE_SHIFT,1, %r25
635#else
636	depwi,z		1, 31-PAGE_SHIFT,1, %r25
637#endif
638	add		%r26, %r25, %r25
639	sub		%r25, %r23, %r25
640
641
6421:      fdc,m		%r23(%r26)
643	fdc,m		%r23(%r26)
644	fdc,m		%r23(%r26)
645	fdc,m		%r23(%r26)
646	fdc,m		%r23(%r26)
647	fdc,m		%r23(%r26)
648	fdc,m		%r23(%r26)
649	fdc,m		%r23(%r26)
650	fdc,m		%r23(%r26)
651	fdc,m		%r23(%r26)
652	fdc,m		%r23(%r26)
653	fdc,m		%r23(%r26)
654	fdc,m		%r23(%r26)
655	fdc,m		%r23(%r26)
656	fdc,m		%r23(%r26)
657	CMPB<<		%r26, %r25,1b
658	fdc,m		%r23(%r26)
659
660	sync
661	bv		%r0(%r2)
662	nop
663	.exit
664
665	.procend
666
667	.export flush_user_dcache_page
668
669flush_user_dcache_page:
670	.proc
671	.callinfo NO_CALLS
672	.entry
673
674	ldil		L%dcache_stride, %r1
675	ldw		R%dcache_stride(%r1), %r23
676
677#ifdef CONFIG_64BIT
678	depdi,z		1,63-PAGE_SHIFT,1, %r25
679#else
680	depwi,z		1,31-PAGE_SHIFT,1, %r25
681#endif
682	add		%r26, %r25, %r25
683	sub		%r25, %r23, %r25
684
685
6861:      fdc,m		%r23(%sr3, %r26)
687	fdc,m		%r23(%sr3, %r26)
688	fdc,m		%r23(%sr3, %r26)
689	fdc,m		%r23(%sr3, %r26)
690	fdc,m		%r23(%sr3, %r26)
691	fdc,m		%r23(%sr3, %r26)
692	fdc,m		%r23(%sr3, %r26)
693	fdc,m		%r23(%sr3, %r26)
694	fdc,m		%r23(%sr3, %r26)
695	fdc,m		%r23(%sr3, %r26)
696	fdc,m		%r23(%sr3, %r26)
697	fdc,m		%r23(%sr3, %r26)
698	fdc,m		%r23(%sr3, %r26)
699	fdc,m		%r23(%sr3, %r26)
700	fdc,m		%r23(%sr3, %r26)
701	CMPB<<		%r26, %r25,1b
702	fdc,m		%r23(%sr3, %r26)
703
704	sync
705	bv		%r0(%r2)
706	nop
707	.exit
708
709	.procend
710
711	.export flush_user_icache_page
712
713flush_user_icache_page:
714	.proc
715	.callinfo NO_CALLS
716	.entry
717
718	ldil		L%dcache_stride, %r1
719	ldw		R%dcache_stride(%r1), %r23
720
721#ifdef CONFIG_64BIT
722	depdi,z		1, 63-PAGE_SHIFT,1, %r25
723#else
724	depwi,z		1, 31-PAGE_SHIFT,1, %r25
725#endif
726	add		%r26, %r25, %r25
727	sub		%r25, %r23, %r25
728
729
7301:      fic,m		%r23(%sr3, %r26)
731	fic,m		%r23(%sr3, %r26)
732	fic,m		%r23(%sr3, %r26)
733	fic,m		%r23(%sr3, %r26)
734	fic,m		%r23(%sr3, %r26)
735	fic,m		%r23(%sr3, %r26)
736	fic,m		%r23(%sr3, %r26)
737	fic,m		%r23(%sr3, %r26)
738	fic,m		%r23(%sr3, %r26)
739	fic,m		%r23(%sr3, %r26)
740	fic,m		%r23(%sr3, %r26)
741	fic,m		%r23(%sr3, %r26)
742	fic,m		%r23(%sr3, %r26)
743	fic,m		%r23(%sr3, %r26)
744	fic,m		%r23(%sr3, %r26)
745	CMPB<<		%r26, %r25,1b
746	fic,m		%r23(%sr3, %r26)
747
748	sync
749	bv		%r0(%r2)
750	nop
751	.exit
752
753	.procend
754
755
756	.export purge_kernel_dcache_page
757
758purge_kernel_dcache_page:
759	.proc
760	.callinfo NO_CALLS
761	.entry
762
763	ldil		L%dcache_stride, %r1
764	ldw		R%dcache_stride(%r1), %r23
765
766#ifdef CONFIG_64BIT
767	depdi,z		1, 63-PAGE_SHIFT,1, %r25
768#else
769	depwi,z		1, 31-PAGE_SHIFT,1, %r25
770#endif
771	add		%r26, %r25, %r25
772	sub		%r25, %r23, %r25
773
7741:      pdc,m		%r23(%r26)
775	pdc,m		%r23(%r26)
776	pdc,m		%r23(%r26)
777	pdc,m		%r23(%r26)
778	pdc,m		%r23(%r26)
779	pdc,m		%r23(%r26)
780	pdc,m		%r23(%r26)
781	pdc,m		%r23(%r26)
782	pdc,m		%r23(%r26)
783	pdc,m		%r23(%r26)
784	pdc,m		%r23(%r26)
785	pdc,m		%r23(%r26)
786	pdc,m		%r23(%r26)
787	pdc,m		%r23(%r26)
788	pdc,m		%r23(%r26)
789	CMPB<<		%r26, %r25, 1b
790	pdc,m		%r23(%r26)
791
792	sync
793	bv		%r0(%r2)
794	nop
795	.exit
796
797	.procend
798
799#if 0
800	/* Currently not used, but it still is a possible alternate
801	 * solution.
802	 */
803
804	.export flush_alias_page
805
806flush_alias_page:
807	.proc
808	.callinfo NO_CALLS
809	.entry
810
811	tophys_r1		%r26
812
813	ldil		L%(TMPALIAS_MAP_START), %r28
814#ifdef CONFIG_64BIT
815	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
816	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
817	depdi		0, 63,12, %r28		/* Clear any offset bits */
818#else
819	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
820	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
821	depwi		0, 31,12, %r28		/* Clear any offset bits */
822#endif
823
824	/* Purge any old translation */
825
826	pdtlb		0(%r28)
827
828	ldil		L%dcache_stride, %r1
829	ldw		R%dcache_stride(%r1), %r23
830
831#ifdef CONFIG_64BIT
832	depdi,z		1, 63-PAGE_SHIFT,1, %r29
833#else
834	depwi,z		1, 31-PAGE_SHIFT,1, %r29
835#endif
836	add		%r28, %r29, %r29
837	sub		%r29, %r23, %r29
838
8391:      fdc,m		%r23(%r28)
840	fdc,m		%r23(%r28)
841	fdc,m		%r23(%r28)
842	fdc,m		%r23(%r28)
843	fdc,m		%r23(%r28)
844	fdc,m		%r23(%r28)
845	fdc,m		%r23(%r28)
846	fdc,m		%r23(%r28)
847	fdc,m		%r23(%r28)
848	fdc,m		%r23(%r28)
849	fdc,m		%r23(%r28)
850	fdc,m		%r23(%r28)
851	fdc,m		%r23(%r28)
852	fdc,m		%r23(%r28)
853	fdc,m		%r23(%r28)
854	CMPB<<		%r28, %r29, 1b
855	fdc,m		%r23(%r28)
856
857	sync
858	bv		%r0(%r2)
859	nop
860	.exit
861
862	.procend
863#endif
864
865	.export flush_user_dcache_range_asm
866
867flush_user_dcache_range_asm:
868	.proc
869	.callinfo NO_CALLS
870	.entry
871
872	ldil		L%dcache_stride, %r1
873	ldw		R%dcache_stride(%r1), %r23
874	ldo		-1(%r23), %r21
875	ANDCM		%r26, %r21, %r26
876
8771:      CMPB<<,n	%r26, %r25, 1b
878	fdc,m		%r23(%sr3, %r26)
879
880	sync
881	bv		%r0(%r2)
882	nop
883	.exit
884
885	.procend
886
887	.export flush_kernel_dcache_range_asm
888
889flush_kernel_dcache_range_asm:
890	.proc
891	.callinfo NO_CALLS
892	.entry
893
894	ldil		L%dcache_stride, %r1
895	ldw		R%dcache_stride(%r1), %r23
896	ldo		-1(%r23), %r21
897	ANDCM		%r26, %r21, %r26
898
8991:      CMPB<<,n	%r26, %r25,1b
900	fdc,m		%r23(%r26)
901
902	sync
903	syncdma
904	bv		%r0(%r2)
905	nop
906	.exit
907
908	.procend
909
910	.export flush_user_icache_range_asm
911
912flush_user_icache_range_asm:
913	.proc
914	.callinfo NO_CALLS
915	.entry
916
917	ldil		L%icache_stride, %r1
918	ldw		R%icache_stride(%r1), %r23
919	ldo		-1(%r23), %r21
920	ANDCM		%r26, %r21, %r26
921
9221:      CMPB<<,n	%r26, %r25,1b
923	fic,m		%r23(%sr3, %r26)
924
925	sync
926	bv		%r0(%r2)
927	nop
928	.exit
929
930	.procend
931
932	.export flush_kernel_icache_page
933
934flush_kernel_icache_page:
935	.proc
936	.callinfo NO_CALLS
937	.entry
938
939	ldil		L%icache_stride, %r1
940	ldw		R%icache_stride(%r1), %r23
941
942#ifdef CONFIG_64BIT
943	depdi,z		1, 63-PAGE_SHIFT,1, %r25
944#else
945	depwi,z		1, 31-PAGE_SHIFT,1, %r25
946#endif
947	add		%r26, %r25, %r25
948	sub		%r25, %r23, %r25
949
950
9511:      fic,m		%r23(%sr4, %r26)
952	fic,m		%r23(%sr4, %r26)
953	fic,m		%r23(%sr4, %r26)
954	fic,m		%r23(%sr4, %r26)
955	fic,m		%r23(%sr4, %r26)
956	fic,m		%r23(%sr4, %r26)
957	fic,m		%r23(%sr4, %r26)
958	fic,m		%r23(%sr4, %r26)
959	fic,m		%r23(%sr4, %r26)
960	fic,m		%r23(%sr4, %r26)
961	fic,m		%r23(%sr4, %r26)
962	fic,m		%r23(%sr4, %r26)
963	fic,m		%r23(%sr4, %r26)
964	fic,m		%r23(%sr4, %r26)
965	fic,m		%r23(%sr4, %r26)
966	CMPB<<		%r26, %r25, 1b
967	fic,m		%r23(%sr4, %r26)
968
969	sync
970	bv		%r0(%r2)
971	nop
972	.exit
973
974	.procend
975
976	.export flush_kernel_icache_range_asm
977
978flush_kernel_icache_range_asm:
979	.proc
980	.callinfo NO_CALLS
981	.entry
982
983	ldil		L%icache_stride, %r1
984	ldw		R%icache_stride(%r1), %r23
985	ldo		-1(%r23), %r21
986	ANDCM		%r26, %r21, %r26
987
9881:      CMPB<<,n	%r26, %r25, 1b
989	fic,m		%r23(%sr4, %r26)
990
991	sync
992	bv		%r0(%r2)
993	nop
994	.exit
995	.procend
996
997	/* align should cover use of rfi in disable_sr_hashing_asm and
998	 * srdis_done.
999	 */
1000	.align	256
1001	.export disable_sr_hashing_asm,code
1002
1003disable_sr_hashing_asm:
1004	.proc
1005	.callinfo NO_CALLS
1006	.entry
1007
1008	/*
1009	 * Switch to real mode
1010	 */
1011	/* pcxt_ssm_bug */
1012	rsm		PSW_SM_I, %r0
1013	load32		PA(1f), %r1
1014	nop
1015	nop
1016	nop
1017	nop
1018	nop
1019
1020	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1021	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1022	mtctl		%r0, %cr17		/* Clear IIASQ head */
1023	mtctl		%r1, %cr18		/* IIAOQ head */
1024	ldo		4(%r1), %r1
1025	mtctl		%r1, %cr18		/* IIAOQ tail */
1026	load32		REAL_MODE_PSW, %r1
1027	mtctl		%r1, %ipsw
1028	rfi
1029	nop
1030
10311:      cmpib,=,n	SRHASH_PCXST, %r26,srdis_pcxs
1032	cmpib,=,n	SRHASH_PCXL, %r26,srdis_pcxl
1033	cmpib,=,n	SRHASH_PA20, %r26,srdis_pa20
1034	b,n		srdis_done
1035
1036srdis_pcxs:
1037
1038	/* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1039
1040	.word		0x141c1a00		/* mfdiag %dr0, %r28 */
1041	.word		0x141c1a00		/* must issue twice */
1042	depwi		0,18,1, %r28		/* Clear DHE (dcache hash enable) */
1043	depwi		0,20,1, %r28		/* Clear IHE (icache hash enable) */
1044	.word		0x141c1600		/* mtdiag %r28, %dr0 */
1045	.word		0x141c1600		/* must issue twice */
1046	b,n		srdis_done
1047
1048srdis_pcxl:
1049
1050	/* Disable Space Register Hashing for PCXL */
1051
1052	.word		0x141c0600		/* mfdiag %dr0, %r28 */
1053	depwi           0,28,2, %r28		/* Clear DHASH_EN & IHASH_EN */
1054	.word		0x141c0240		/* mtdiag %r28, %dr0 */
1055	b,n		srdis_done
1056
1057srdis_pa20:
1058
1059	/* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1060
1061	.word		0x144008bc		/* mfdiag %dr2, %r28 */
1062	depdi		0, 54,1, %r28		/* clear DIAG_SPHASH_ENAB (bit 54) */
1063	.word		0x145c1840		/* mtdiag %r28, %dr2 */
1064
1065
1066srdis_done:
1067	/* Switch back to virtual mode */
1068	rsm		PSW_SM_I, %r0		/* prep to load iia queue */
1069	load32 	   	2f, %r1
1070	nop
1071	nop
1072	nop
1073	nop
1074	nop
1075
1076	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1077	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1078	mtctl		%r0, %cr17		/* Clear IIASQ head */
1079	mtctl		%r1, %cr18		/* IIAOQ head */
1080	ldo		4(%r1), %r1
1081	mtctl		%r1, %cr18		/* IIAOQ tail */
1082	load32		KERNEL_PSW, %r1
1083	mtctl		%r1, %ipsw
1084	rfi
1085	nop
1086
10872:      bv		%r0(%r2)
1088	nop
1089	.exit
1090
1091	.procend
1092
1093	.end
1094