xref: /linux/arch/parisc/kernel/pacache.S (revision 14b42963f64b98ab61fa9723c03d71aa5ef4f862) 
1/*
2 *  PARISC TLB and cache flushing support
3 *  Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
4 *  Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
5 *  Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
6 *
7 *    This program is free software; you can redistribute it and/or modify
8 *    it under the terms of the GNU General Public License as published by
9 *    the Free Software Foundation; either version 2, or (at your option)
10 *    any later version.
11 *
12 *    This program is distributed in the hope that it will be useful,
13 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
14 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15 *    GNU General Public License for more details.
16 *
17 *    You should have received a copy of the GNU General Public License
18 *    along with this program; if not, write to the Free Software
19 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20 */
21
22/*
23 * NOTE: fdc,fic, and pdc instructions that use base register modification
24 *       should only use index and base registers that are not shadowed,
25 *       so that the fast path emulation in the non access miss handler
26 *       can be used.
27 */
28
29#ifdef CONFIG_64BIT
30#define ADDIB	addib,*
31#define CMPB	cmpb,*
32#define ANDCM	andcm,*
33
34	.level	2.0w
35#else
36#define ADDIB	addib,
37#define CMPB	cmpb,
38#define ANDCM	andcm
39
40	.level	2.0
41#endif
42
43
44#include <asm/psw.h>
45#include <asm/assembly.h>
46#include <asm/pgtable.h>
47#include <asm/cache.h>
48
49	.text
50	.align	128
51
52	.export flush_tlb_all_local,code
53
54flush_tlb_all_local:
55	.proc
56	.callinfo NO_CALLS
57	.entry
58
59	/*
60	 * The pitlbe and pdtlbe instructions should only be used to
61	 * flush the entire tlb. Also, there needs to be no intervening
62	 * tlb operations, e.g. tlb misses, so the operation needs
63	 * to happen in real mode with all interruptions disabled.
64	 */
65
66	/* pcxt_ssm_bug	- relied upon translation! PA 2.0 Arch. F-4 and F-5 */
67	rsm		PSW_SM_I, %r19		/* save I-bit state */
68	load32		PA(1f), %r1
69	nop
70	nop
71	nop
72	nop
73	nop
74
75	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
76	mtctl		%r0, %cr17		/* Clear IIASQ tail */
77	mtctl		%r0, %cr17		/* Clear IIASQ head */
78	mtctl		%r1, %cr18		/* IIAOQ head */
79	ldo		4(%r1), %r1
80	mtctl		%r1, %cr18		/* IIAOQ tail */
81	load32		REAL_MODE_PSW, %r1
82	mtctl           %r1, %ipsw
83	rfi
84	nop
85
861:      load32		PA(cache_info), %r1
87
88	/* Flush Instruction Tlb */
89
90	LDREG		ITLB_SID_BASE(%r1), %r20
91	LDREG		ITLB_SID_STRIDE(%r1), %r21
92	LDREG		ITLB_SID_COUNT(%r1), %r22
93	LDREG		ITLB_OFF_BASE(%r1), %arg0
94	LDREG		ITLB_OFF_STRIDE(%r1), %arg1
95	LDREG		ITLB_OFF_COUNT(%r1), %arg2
96	LDREG		ITLB_LOOP(%r1), %arg3
97
98	ADDIB=		-1, %arg3, fitoneloop	/* Preadjust and test */
99	movb,<,n	%arg3, %r31, fitdone	/* If loop < 0, skip */
100	copy		%arg0, %r28		/* Init base addr */
101
102fitmanyloop:					/* Loop if LOOP >= 2 */
103	mtsp		%r20, %sr1
104	add		%r21, %r20, %r20	/* increment space */
105	copy		%arg2, %r29		/* Init middle loop count */
106
107fitmanymiddle:					/* Loop if LOOP >= 2 */
108	ADDIB>		-1, %r31, fitmanymiddle	/* Adjusted inner loop decr */
109	pitlbe		0(%sr1, %r28)
110	pitlbe,m	%arg1(%sr1, %r28)	/* Last pitlbe and addr adjust */
111	ADDIB>		-1, %r29, fitmanymiddle	/* Middle loop decr */
112	copy		%arg3, %r31		/* Re-init inner loop count */
113
114	movb,tr		%arg0, %r28, fitmanyloop /* Re-init base addr */
115	ADDIB<=,n	-1, %r22, fitdone	/* Outer loop count decr */
116
117fitoneloop:					/* Loop if LOOP = 1 */
118	mtsp		%r20, %sr1
119	copy		%arg0, %r28		/* init base addr */
120	copy		%arg2, %r29		/* init middle loop count */
121
122fitonemiddle:					/* Loop if LOOP = 1 */
123	ADDIB>		-1, %r29, fitonemiddle	/* Middle loop count decr */
124	pitlbe,m	%arg1(%sr1, %r28)	/* pitlbe for one loop */
125
126	ADDIB>		-1, %r22, fitoneloop	/* Outer loop count decr */
127	add		%r21, %r20, %r20		/* increment space */
128
129fitdone:
130
131	/* Flush Data Tlb */
132
133	LDREG		DTLB_SID_BASE(%r1), %r20
134	LDREG		DTLB_SID_STRIDE(%r1), %r21
135	LDREG		DTLB_SID_COUNT(%r1), %r22
136	LDREG		DTLB_OFF_BASE(%r1), %arg0
137	LDREG		DTLB_OFF_STRIDE(%r1), %arg1
138	LDREG		DTLB_OFF_COUNT(%r1), %arg2
139	LDREG		DTLB_LOOP(%r1), %arg3
140
141	ADDIB=		-1, %arg3, fdtoneloop	/* Preadjust and test */
142	movb,<,n	%arg3, %r31, fdtdone	/* If loop < 0, skip */
143	copy		%arg0, %r28		/* Init base addr */
144
145fdtmanyloop:					/* Loop if LOOP >= 2 */
146	mtsp		%r20, %sr1
147	add		%r21, %r20, %r20	/* increment space */
148	copy		%arg2, %r29		/* Init middle loop count */
149
150fdtmanymiddle:					/* Loop if LOOP >= 2 */
151	ADDIB>		-1, %r31, fdtmanymiddle	/* Adjusted inner loop decr */
152	pdtlbe		0(%sr1, %r28)
153	pdtlbe,m	%arg1(%sr1, %r28)	/* Last pdtlbe and addr adjust */
154	ADDIB>		-1, %r29, fdtmanymiddle	/* Middle loop decr */
155	copy		%arg3, %r31		/* Re-init inner loop count */
156
157	movb,tr		%arg0, %r28, fdtmanyloop /* Re-init base addr */
158	ADDIB<=,n	-1, %r22,fdtdone	/* Outer loop count decr */
159
160fdtoneloop:					/* Loop if LOOP = 1 */
161	mtsp		%r20, %sr1
162	copy		%arg0, %r28		/* init base addr */
163	copy		%arg2, %r29		/* init middle loop count */
164
165fdtonemiddle:					/* Loop if LOOP = 1 */
166	ADDIB>		-1, %r29, fdtonemiddle	/* Middle loop count decr */
167	pdtlbe,m	%arg1(%sr1, %r28)	/* pdtlbe for one loop */
168
169	ADDIB>		-1, %r22, fdtoneloop	/* Outer loop count decr */
170	add		%r21, %r20, %r20	/* increment space */
171
172
173fdtdone:
174	/*
175	 * Switch back to virtual mode
176	 */
177	/* pcxt_ssm_bug */
178	rsm		PSW_SM_I, %r0
179	load32		2f, %r1
180	nop
181	nop
182	nop
183	nop
184	nop
185
186	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
187	mtctl		%r0, %cr17		/* Clear IIASQ tail */
188	mtctl		%r0, %cr17		/* Clear IIASQ head */
189	mtctl		%r1, %cr18		/* IIAOQ head */
190	ldo		4(%r1), %r1
191	mtctl		%r1, %cr18		/* IIAOQ tail */
192	load32		KERNEL_PSW, %r1
193	or		%r1, %r19, %r1	/* I-bit to state on entry */
194	mtctl		%r1, %ipsw	/* restore I-bit (entire PSW) */
195	rfi
196	nop
197
1982:      bv		%r0(%r2)
199	nop
200
201	.exit
202	.procend
203
204	.export flush_instruction_cache_local,code
205	.import cache_info,data
206
207flush_instruction_cache_local:
208	.proc
209	.callinfo NO_CALLS
210	.entry
211
212	mtsp		%r0, %sr1
213	load32		cache_info, %r1
214
215	/* Flush Instruction Cache */
216
217	LDREG		ICACHE_BASE(%r1), %arg0
218	LDREG		ICACHE_STRIDE(%r1), %arg1
219	LDREG		ICACHE_COUNT(%r1), %arg2
220	LDREG		ICACHE_LOOP(%r1), %arg3
221	rsm             PSW_SM_I, %r22		/* No mmgt ops during loop*/
222	ADDIB=		-1, %arg3, fioneloop	/* Preadjust and test */
223	movb,<,n	%arg3, %r31, fisync	/* If loop < 0, do sync */
224
225fimanyloop:					/* Loop if LOOP >= 2 */
226	ADDIB>		-1, %r31, fimanyloop	/* Adjusted inner loop decr */
227	fice            %r0(%sr1, %arg0)
228	fice,m		%arg1(%sr1, %arg0)	/* Last fice and addr adjust */
229	movb,tr		%arg3, %r31, fimanyloop	/* Re-init inner loop count */
230	ADDIB<=,n	-1, %arg2, fisync	/* Outer loop decr */
231
232fioneloop:					/* Loop if LOOP = 1 */
233	ADDIB>		-1, %arg2, fioneloop	/* Outer loop count decr */
234	fice,m		%arg1(%sr1, %arg0)	/* Fice for one loop */
235
236fisync:
237	sync
238	mtsm		%r22			/* restore I-bit */
239	bv		%r0(%r2)
240	nop
241	.exit
242
243	.procend
244
245	.export flush_data_cache_local, code
246	.import cache_info, data
247
248flush_data_cache_local:
249	.proc
250	.callinfo NO_CALLS
251	.entry
252
253	mtsp		%r0, %sr1
254	load32 		cache_info, %r1
255
256	/* Flush Data Cache */
257
258	LDREG		DCACHE_BASE(%r1), %arg0
259	LDREG		DCACHE_STRIDE(%r1), %arg1
260	LDREG		DCACHE_COUNT(%r1), %arg2
261	LDREG		DCACHE_LOOP(%r1), %arg3
262	rsm		PSW_SM_I, %r22
263	ADDIB=		-1, %arg3, fdoneloop	/* Preadjust and test */
264	movb,<,n	%arg3, %r31, fdsync	/* If loop < 0, do sync */
265
266fdmanyloop:					/* Loop if LOOP >= 2 */
267	ADDIB>		-1, %r31, fdmanyloop	/* Adjusted inner loop decr */
268	fdce		%r0(%sr1, %arg0)
269	fdce,m		%arg1(%sr1, %arg0)	/* Last fdce and addr adjust */
270	movb,tr		%arg3, %r31, fdmanyloop	/* Re-init inner loop count */
271	ADDIB<=,n	-1, %arg2, fdsync	/* Outer loop decr */
272
273fdoneloop:					/* Loop if LOOP = 1 */
274	ADDIB>		-1, %arg2, fdoneloop	/* Outer loop count decr */
275	fdce,m		%arg1(%sr1, %arg0)	/* Fdce for one loop */
276
277fdsync:
278	syncdma
279	sync
280	mtsm		%r22			/* restore I-bit */
281	bv		%r0(%r2)
282	nop
283	.exit
284
285	.procend
286
287	.export copy_user_page_asm,code
288	.align	16
289
290copy_user_page_asm:
291	.proc
292	.callinfo NO_CALLS
293	.entry
294
295#ifdef CONFIG_64BIT
296	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
297	 * Unroll the loop by hand and arrange insn appropriately.
298	 * GCC probably can do this just as well.
299	 */
300
301	ldd		0(%r25), %r19
302	ldi		ASM_PAGE_SIZE_DIV128, %r1
303
304	ldw		64(%r25), %r0		/* prefetch 1 cacheline ahead */
305	ldw		128(%r25), %r0		/* prefetch 2 */
306
3071:	ldd		8(%r25), %r20
308	ldw		192(%r25), %r0		/* prefetch 3 */
309	ldw		256(%r25), %r0		/* prefetch 4 */
310
311	ldd		16(%r25), %r21
312	ldd		24(%r25), %r22
313	std		%r19, 0(%r26)
314	std		%r20, 8(%r26)
315
316	ldd		32(%r25), %r19
317	ldd		40(%r25), %r20
318	std		%r21, 16(%r26)
319	std		%r22, 24(%r26)
320
321	ldd		48(%r25), %r21
322	ldd		56(%r25), %r22
323	std		%r19, 32(%r26)
324	std		%r20, 40(%r26)
325
326	ldd		64(%r25), %r19
327	ldd		72(%r25), %r20
328	std		%r21, 48(%r26)
329	std		%r22, 56(%r26)
330
331	ldd		80(%r25), %r21
332	ldd		88(%r25), %r22
333	std		%r19, 64(%r26)
334	std		%r20, 72(%r26)
335
336	ldd		 96(%r25), %r19
337	ldd		104(%r25), %r20
338	std		%r21, 80(%r26)
339	std		%r22, 88(%r26)
340
341	ldd		112(%r25), %r21
342	ldd		120(%r25), %r22
343	std		%r19, 96(%r26)
344	std		%r20, 104(%r26)
345
346	ldo		128(%r25), %r25
347	std		%r21, 112(%r26)
348	std		%r22, 120(%r26)
349	ldo		128(%r26), %r26
350
351	/* conditional branches nullify on forward taken branch, and on
352	 * non-taken backward branch. Note that .+4 is a backwards branch.
353	 * The ldd should only get executed if the branch is taken.
354	 */
355	ADDIB>,n	-1, %r1, 1b		/* bundle 10 */
356	ldd		0(%r25), %r19		/* start next loads */
357
358#else
359
360	/*
361	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
362	 * bundles (very restricted rules for bundling).
363	 * Note that until (if) we start saving
364	 * the full 64 bit register values on interrupt, we can't
365	 * use ldd/std on a 32 bit kernel.
366	 */
367	ldw		0(%r25), %r19
368	ldi		ASM_PAGE_SIZE_DIV64, %r1
369
3701:
371	ldw		4(%r25), %r20
372	ldw		8(%r25), %r21
373	ldw		12(%r25), %r22
374	stw		%r19, 0(%r26)
375	stw		%r20, 4(%r26)
376	stw		%r21, 8(%r26)
377	stw		%r22, 12(%r26)
378	ldw		16(%r25), %r19
379	ldw		20(%r25), %r20
380	ldw		24(%r25), %r21
381	ldw		28(%r25), %r22
382	stw		%r19, 16(%r26)
383	stw		%r20, 20(%r26)
384	stw		%r21, 24(%r26)
385	stw		%r22, 28(%r26)
386	ldw		32(%r25), %r19
387	ldw		36(%r25), %r20
388	ldw		40(%r25), %r21
389	ldw		44(%r25), %r22
390	stw		%r19, 32(%r26)
391	stw		%r20, 36(%r26)
392	stw		%r21, 40(%r26)
393	stw		%r22, 44(%r26)
394	ldw		48(%r25), %r19
395	ldw		52(%r25), %r20
396	ldw		56(%r25), %r21
397	ldw		60(%r25), %r22
398	stw		%r19, 48(%r26)
399	stw		%r20, 52(%r26)
400	ldo		64(%r25), %r25
401	stw		%r21, 56(%r26)
402	stw		%r22, 60(%r26)
403	ldo		64(%r26), %r26
404	ADDIB>,n	-1, %r1, 1b
405	ldw		0(%r25), %r19
406#endif
407	bv		%r0(%r2)
408	nop
409	.exit
410
411	.procend
412
413/*
414 * NOTE: Code in clear_user_page has a hard coded dependency on the
415 *       maximum alias boundary being 4 Mb. We've been assured by the
416 *       parisc chip designers that there will not ever be a parisc
417 *       chip with a larger alias boundary (Never say never :-) ).
418 *
419 *       Subtle: the dtlb miss handlers support the temp alias region by
420 *       "knowing" that if a dtlb miss happens within the temp alias
421 *       region it must have occurred while in clear_user_page. Since
422 *       this routine makes use of processor local translations, we
423 *       don't want to insert them into the kernel page table. Instead,
424 *       we load up some general registers (they need to be registers
425 *       which aren't shadowed) with the physical page numbers (preshifted
426 *       for tlb insertion) needed to insert the translations. When we
427 *       miss on the translation, the dtlb miss handler inserts the
428 *       translation into the tlb using these values:
429 *
430 *          %r26 physical page (shifted for tlb insert) of "to" translation
431 *          %r23 physical page (shifted for tlb insert) of "from" translation
432 */
433
434#if 0
435
436	/*
437	 * We can't do this since copy_user_page is used to bring in
438	 * file data that might have instructions. Since the data would
439	 * then need to be flushed out so the i-fetch can see it, it
440	 * makes more sense to just copy through the kernel translation
441	 * and flush it.
442	 *
443	 * I'm still keeping this around because it may be possible to
444	 * use it if more information is passed into copy_user_page().
445	 * Have to do some measurements to see if it is worthwhile to
446	 * lobby for such a change.
447	 */
448
449	.export copy_user_page_asm,code
450
451copy_user_page_asm:
452	.proc
453	.callinfo NO_CALLS
454	.entry
455
456	ldil		L%(__PAGE_OFFSET), %r1
457	sub		%r26, %r1, %r26
458	sub		%r25, %r1, %r23		/* move physical addr into non shadowed reg */
459
460	ldil		L%(TMPALIAS_MAP_START), %r28
461	/* FIXME for different page sizes != 4k */
462#ifdef CONFIG_64BIT
463	extrd,u		%r26,56,32, %r26		/* convert phys addr to tlb insert format */
464	extrd,u		%r23,56,32, %r23		/* convert phys addr to tlb insert format */
465	depd		%r24,63,22, %r28		/* Form aliased virtual address 'to' */
466	depdi		0, 63,12, %r28		/* Clear any offset bits */
467	copy		%r28, %r29
468	depdi		1, 41,1, %r29		/* Form aliased virtual address 'from' */
469#else
470	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
471	extrw,u		%r23, 24,25, %r23	/* convert phys addr to tlb insert format */
472	depw		%r24, 31,22, %r28	/* Form aliased virtual address 'to' */
473	depwi		0, 31,12, %r28		/* Clear any offset bits */
474	copy		%r28, %r29
475	depwi		1, 9,1, %r29		/* Form aliased virtual address 'from' */
476#endif
477
478	/* Purge any old translations */
479
480	pdtlb		0(%r28)
481	pdtlb		0(%r29)
482
483	ldi		64, %r1
484
485	/*
486	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
487	 * bundles (very restricted rules for bundling). It probably
488	 * does OK on PCXU and better, but we could do better with
489	 * ldd/std instructions. Note that until (if) we start saving
490	 * the full 64 bit register values on interrupt, we can't
491	 * use ldd/std on a 32 bit kernel.
492	 */
493
494
4951:
496	ldw		0(%r29), %r19
497	ldw		4(%r29), %r20
498	ldw		8(%r29), %r21
499	ldw		12(%r29), %r22
500	stw		%r19, 0(%r28)
501	stw		%r20, 4(%r28)
502	stw		%r21, 8(%r28)
503	stw		%r22, 12(%r28)
504	ldw		16(%r29), %r19
505	ldw		20(%r29), %r20
506	ldw		24(%r29), %r21
507	ldw		28(%r29), %r22
508	stw		%r19, 16(%r28)
509	stw		%r20, 20(%r28)
510	stw		%r21, 24(%r28)
511	stw		%r22, 28(%r28)
512	ldw		32(%r29), %r19
513	ldw		36(%r29), %r20
514	ldw		40(%r29), %r21
515	ldw		44(%r29), %r22
516	stw		%r19, 32(%r28)
517	stw		%r20, 36(%r28)
518	stw		%r21, 40(%r28)
519	stw		%r22, 44(%r28)
520	ldw		48(%r29), %r19
521	ldw		52(%r29), %r20
522	ldw		56(%r29), %r21
523	ldw		60(%r29), %r22
524	stw		%r19, 48(%r28)
525	stw		%r20, 52(%r28)
526	stw		%r21, 56(%r28)
527	stw		%r22, 60(%r28)
528	ldo		64(%r28), %r28
529	ADDIB>		-1, %r1,1b
530	ldo		64(%r29), %r29
531
532	bv		%r0(%r2)
533	nop
534	.exit
535
536	.procend
537#endif
538
539	.export __clear_user_page_asm,code
540
541__clear_user_page_asm:
542	.proc
543	.callinfo NO_CALLS
544	.entry
545
546	tophys_r1	%r26
547
548	ldil		L%(TMPALIAS_MAP_START), %r28
549#ifdef CONFIG_64BIT
550#if (TMPALIAS_MAP_START >= 0x80000000)
551	depdi		0, 31,32, %r28		/* clear any sign extension */
552	/* FIXME: page size dependend */
553#endif
554	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
555	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
556	depdi		0, 63,12, %r28		/* Clear any offset bits */
557#else
558	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
559	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
560	depwi		0, 31,12, %r28		/* Clear any offset bits */
561#endif
562
563	/* Purge any old translation */
564
565	pdtlb		0(%r28)
566
567#ifdef CONFIG_64BIT
568	ldi		ASM_PAGE_SIZE_DIV128, %r1
569
570	/* PREFETCH (Write) has not (yet) been proven to help here */
571	/* #define	PREFETCHW_OP	ldd		256(%0), %r0 */
572
5731:	std		%r0, 0(%r28)
574	std		%r0, 8(%r28)
575	std		%r0, 16(%r28)
576	std		%r0, 24(%r28)
577	std		%r0, 32(%r28)
578	std		%r0, 40(%r28)
579	std		%r0, 48(%r28)
580	std		%r0, 56(%r28)
581	std		%r0, 64(%r28)
582	std		%r0, 72(%r28)
583	std		%r0, 80(%r28)
584	std		%r0, 88(%r28)
585	std		%r0, 96(%r28)
586	std		%r0, 104(%r28)
587	std		%r0, 112(%r28)
588	std		%r0, 120(%r28)
589	ADDIB>		-1, %r1, 1b
590	ldo		128(%r28), %r28
591
592#else	/* ! CONFIG_64BIT */
593	ldi		ASM_PAGE_SIZE_DIV64, %r1
594
5951:
596	stw		%r0, 0(%r28)
597	stw		%r0, 4(%r28)
598	stw		%r0, 8(%r28)
599	stw		%r0, 12(%r28)
600	stw		%r0, 16(%r28)
601	stw		%r0, 20(%r28)
602	stw		%r0, 24(%r28)
603	stw		%r0, 28(%r28)
604	stw		%r0, 32(%r28)
605	stw		%r0, 36(%r28)
606	stw		%r0, 40(%r28)
607	stw		%r0, 44(%r28)
608	stw		%r0, 48(%r28)
609	stw		%r0, 52(%r28)
610	stw		%r0, 56(%r28)
611	stw		%r0, 60(%r28)
612	ADDIB>		-1, %r1, 1b
613	ldo		64(%r28), %r28
614#endif	/* CONFIG_64BIT */
615
616	bv		%r0(%r2)
617	nop
618	.exit
619
620	.procend
621
622	.export flush_kernel_dcache_page_asm
623
624flush_kernel_dcache_page_asm:
625	.proc
626	.callinfo NO_CALLS
627	.entry
628
629	ldil		L%dcache_stride, %r1
630	ldw		R%dcache_stride(%r1), %r23
631
632#ifdef CONFIG_64BIT
633	depdi,z		1, 63-PAGE_SHIFT,1, %r25
634#else
635	depwi,z		1, 31-PAGE_SHIFT,1, %r25
636#endif
637	add		%r26, %r25, %r25
638	sub		%r25, %r23, %r25
639
640
6411:      fdc,m		%r23(%r26)
642	fdc,m		%r23(%r26)
643	fdc,m		%r23(%r26)
644	fdc,m		%r23(%r26)
645	fdc,m		%r23(%r26)
646	fdc,m		%r23(%r26)
647	fdc,m		%r23(%r26)
648	fdc,m		%r23(%r26)
649	fdc,m		%r23(%r26)
650	fdc,m		%r23(%r26)
651	fdc,m		%r23(%r26)
652	fdc,m		%r23(%r26)
653	fdc,m		%r23(%r26)
654	fdc,m		%r23(%r26)
655	fdc,m		%r23(%r26)
656	CMPB<<		%r26, %r25,1b
657	fdc,m		%r23(%r26)
658
659	sync
660	bv		%r0(%r2)
661	nop
662	.exit
663
664	.procend
665
666	.export flush_user_dcache_page
667
668flush_user_dcache_page:
669	.proc
670	.callinfo NO_CALLS
671	.entry
672
673	ldil		L%dcache_stride, %r1
674	ldw		R%dcache_stride(%r1), %r23
675
676#ifdef CONFIG_64BIT
677	depdi,z		1,63-PAGE_SHIFT,1, %r25
678#else
679	depwi,z		1,31-PAGE_SHIFT,1, %r25
680#endif
681	add		%r26, %r25, %r25
682	sub		%r25, %r23, %r25
683
684
6851:      fdc,m		%r23(%sr3, %r26)
686	fdc,m		%r23(%sr3, %r26)
687	fdc,m		%r23(%sr3, %r26)
688	fdc,m		%r23(%sr3, %r26)
689	fdc,m		%r23(%sr3, %r26)
690	fdc,m		%r23(%sr3, %r26)
691	fdc,m		%r23(%sr3, %r26)
692	fdc,m		%r23(%sr3, %r26)
693	fdc,m		%r23(%sr3, %r26)
694	fdc,m		%r23(%sr3, %r26)
695	fdc,m		%r23(%sr3, %r26)
696	fdc,m		%r23(%sr3, %r26)
697	fdc,m		%r23(%sr3, %r26)
698	fdc,m		%r23(%sr3, %r26)
699	fdc,m		%r23(%sr3, %r26)
700	CMPB<<		%r26, %r25,1b
701	fdc,m		%r23(%sr3, %r26)
702
703	sync
704	bv		%r0(%r2)
705	nop
706	.exit
707
708	.procend
709
710	.export flush_user_icache_page
711
712flush_user_icache_page:
713	.proc
714	.callinfo NO_CALLS
715	.entry
716
717	ldil		L%dcache_stride, %r1
718	ldw		R%dcache_stride(%r1), %r23
719
720#ifdef CONFIG_64BIT
721	depdi,z		1, 63-PAGE_SHIFT,1, %r25
722#else
723	depwi,z		1, 31-PAGE_SHIFT,1, %r25
724#endif
725	add		%r26, %r25, %r25
726	sub		%r25, %r23, %r25
727
728
7291:      fic,m		%r23(%sr3, %r26)
730	fic,m		%r23(%sr3, %r26)
731	fic,m		%r23(%sr3, %r26)
732	fic,m		%r23(%sr3, %r26)
733	fic,m		%r23(%sr3, %r26)
734	fic,m		%r23(%sr3, %r26)
735	fic,m		%r23(%sr3, %r26)
736	fic,m		%r23(%sr3, %r26)
737	fic,m		%r23(%sr3, %r26)
738	fic,m		%r23(%sr3, %r26)
739	fic,m		%r23(%sr3, %r26)
740	fic,m		%r23(%sr3, %r26)
741	fic,m		%r23(%sr3, %r26)
742	fic,m		%r23(%sr3, %r26)
743	fic,m		%r23(%sr3, %r26)
744	CMPB<<		%r26, %r25,1b
745	fic,m		%r23(%sr3, %r26)
746
747	sync
748	bv		%r0(%r2)
749	nop
750	.exit
751
752	.procend
753
754
755	.export purge_kernel_dcache_page
756
757purge_kernel_dcache_page:
758	.proc
759	.callinfo NO_CALLS
760	.entry
761
762	ldil		L%dcache_stride, %r1
763	ldw		R%dcache_stride(%r1), %r23
764
765#ifdef CONFIG_64BIT
766	depdi,z		1, 63-PAGE_SHIFT,1, %r25
767#else
768	depwi,z		1, 31-PAGE_SHIFT,1, %r25
769#endif
770	add		%r26, %r25, %r25
771	sub		%r25, %r23, %r25
772
7731:      pdc,m		%r23(%r26)
774	pdc,m		%r23(%r26)
775	pdc,m		%r23(%r26)
776	pdc,m		%r23(%r26)
777	pdc,m		%r23(%r26)
778	pdc,m		%r23(%r26)
779	pdc,m		%r23(%r26)
780	pdc,m		%r23(%r26)
781	pdc,m		%r23(%r26)
782	pdc,m		%r23(%r26)
783	pdc,m		%r23(%r26)
784	pdc,m		%r23(%r26)
785	pdc,m		%r23(%r26)
786	pdc,m		%r23(%r26)
787	pdc,m		%r23(%r26)
788	CMPB<<		%r26, %r25, 1b
789	pdc,m		%r23(%r26)
790
791	sync
792	bv		%r0(%r2)
793	nop
794	.exit
795
796	.procend
797
798#if 0
799	/* Currently not used, but it still is a possible alternate
800	 * solution.
801	 */
802
803	.export flush_alias_page
804
805flush_alias_page:
806	.proc
807	.callinfo NO_CALLS
808	.entry
809
810	tophys_r1		%r26
811
812	ldil		L%(TMPALIAS_MAP_START), %r28
813#ifdef CONFIG_64BIT
814	extrd,u		%r26, 56,32, %r26	/* convert phys addr to tlb insert format */
815	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
816	depdi		0, 63,12, %r28		/* Clear any offset bits */
817#else
818	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
819	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
820	depwi		0, 31,12, %r28		/* Clear any offset bits */
821#endif
822
823	/* Purge any old translation */
824
825	pdtlb		0(%r28)
826
827	ldil		L%dcache_stride, %r1
828	ldw		R%dcache_stride(%r1), %r23
829
830#ifdef CONFIG_64BIT
831	depdi,z		1, 63-PAGE_SHIFT,1, %r29
832#else
833	depwi,z		1, 31-PAGE_SHIFT,1, %r29
834#endif
835	add		%r28, %r29, %r29
836	sub		%r29, %r23, %r29
837
8381:      fdc,m		%r23(%r28)
839	fdc,m		%r23(%r28)
840	fdc,m		%r23(%r28)
841	fdc,m		%r23(%r28)
842	fdc,m		%r23(%r28)
843	fdc,m		%r23(%r28)
844	fdc,m		%r23(%r28)
845	fdc,m		%r23(%r28)
846	fdc,m		%r23(%r28)
847	fdc,m		%r23(%r28)
848	fdc,m		%r23(%r28)
849	fdc,m		%r23(%r28)
850	fdc,m		%r23(%r28)
851	fdc,m		%r23(%r28)
852	fdc,m		%r23(%r28)
853	CMPB<<		%r28, %r29, 1b
854	fdc,m		%r23(%r28)
855
856	sync
857	bv		%r0(%r2)
858	nop
859	.exit
860
861	.procend
862#endif
863
864	.export flush_user_dcache_range_asm
865
866flush_user_dcache_range_asm:
867	.proc
868	.callinfo NO_CALLS
869	.entry
870
871	ldil		L%dcache_stride, %r1
872	ldw		R%dcache_stride(%r1), %r23
873	ldo		-1(%r23), %r21
874	ANDCM		%r26, %r21, %r26
875
8761:      CMPB<<,n	%r26, %r25, 1b
877	fdc,m		%r23(%sr3, %r26)
878
879	sync
880	bv		%r0(%r2)
881	nop
882	.exit
883
884	.procend
885
886	.export flush_kernel_dcache_range_asm
887
888flush_kernel_dcache_range_asm:
889	.proc
890	.callinfo NO_CALLS
891	.entry
892
893	ldil		L%dcache_stride, %r1
894	ldw		R%dcache_stride(%r1), %r23
895	ldo		-1(%r23), %r21
896	ANDCM		%r26, %r21, %r26
897
8981:      CMPB<<,n	%r26, %r25,1b
899	fdc,m		%r23(%r26)
900
901	sync
902	syncdma
903	bv		%r0(%r2)
904	nop
905	.exit
906
907	.procend
908
909	.export flush_user_icache_range_asm
910
911flush_user_icache_range_asm:
912	.proc
913	.callinfo NO_CALLS
914	.entry
915
916	ldil		L%icache_stride, %r1
917	ldw		R%icache_stride(%r1), %r23
918	ldo		-1(%r23), %r21
919	ANDCM		%r26, %r21, %r26
920
9211:      CMPB<<,n	%r26, %r25,1b
922	fic,m		%r23(%sr3, %r26)
923
924	sync
925	bv		%r0(%r2)
926	nop
927	.exit
928
929	.procend
930
931	.export flush_kernel_icache_page
932
933flush_kernel_icache_page:
934	.proc
935	.callinfo NO_CALLS
936	.entry
937
938	ldil		L%icache_stride, %r1
939	ldw		R%icache_stride(%r1), %r23
940
941#ifdef CONFIG_64BIT
942	depdi,z		1, 63-PAGE_SHIFT,1, %r25
943#else
944	depwi,z		1, 31-PAGE_SHIFT,1, %r25
945#endif
946	add		%r26, %r25, %r25
947	sub		%r25, %r23, %r25
948
949
9501:      fic,m		%r23(%sr4, %r26)
951	fic,m		%r23(%sr4, %r26)
952	fic,m		%r23(%sr4, %r26)
953	fic,m		%r23(%sr4, %r26)
954	fic,m		%r23(%sr4, %r26)
955	fic,m		%r23(%sr4, %r26)
956	fic,m		%r23(%sr4, %r26)
957	fic,m		%r23(%sr4, %r26)
958	fic,m		%r23(%sr4, %r26)
959	fic,m		%r23(%sr4, %r26)
960	fic,m		%r23(%sr4, %r26)
961	fic,m		%r23(%sr4, %r26)
962	fic,m		%r23(%sr4, %r26)
963	fic,m		%r23(%sr4, %r26)
964	fic,m		%r23(%sr4, %r26)
965	CMPB<<		%r26, %r25, 1b
966	fic,m		%r23(%sr4, %r26)
967
968	sync
969	bv		%r0(%r2)
970	nop
971	.exit
972
973	.procend
974
975	.export flush_kernel_icache_range_asm
976
977flush_kernel_icache_range_asm:
978	.proc
979	.callinfo NO_CALLS
980	.entry
981
982	ldil		L%icache_stride, %r1
983	ldw		R%icache_stride(%r1), %r23
984	ldo		-1(%r23), %r21
985	ANDCM		%r26, %r21, %r26
986
9871:      CMPB<<,n	%r26, %r25, 1b
988	fic,m		%r23(%sr4, %r26)
989
990	sync
991	bv		%r0(%r2)
992	nop
993	.exit
994	.procend
995
996	/* align should cover use of rfi in disable_sr_hashing_asm and
997	 * srdis_done.
998	 */
999	.align	256
1000	.export disable_sr_hashing_asm,code
1001
1002disable_sr_hashing_asm:
1003	.proc
1004	.callinfo NO_CALLS
1005	.entry
1006
1007	/*
1008	 * Switch to real mode
1009	 */
1010	/* pcxt_ssm_bug */
1011	rsm		PSW_SM_I, %r0
1012	load32		PA(1f), %r1
1013	nop
1014	nop
1015	nop
1016	nop
1017	nop
1018
1019	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1020	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1021	mtctl		%r0, %cr17		/* Clear IIASQ head */
1022	mtctl		%r1, %cr18		/* IIAOQ head */
1023	ldo		4(%r1), %r1
1024	mtctl		%r1, %cr18		/* IIAOQ tail */
1025	load32		REAL_MODE_PSW, %r1
1026	mtctl		%r1, %ipsw
1027	rfi
1028	nop
1029
10301:      cmpib,=,n	SRHASH_PCXST, %r26,srdis_pcxs
1031	cmpib,=,n	SRHASH_PCXL, %r26,srdis_pcxl
1032	cmpib,=,n	SRHASH_PA20, %r26,srdis_pa20
1033	b,n		srdis_done
1034
1035srdis_pcxs:
1036
1037	/* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1038
1039	.word		0x141c1a00		/* mfdiag %dr0, %r28 */
1040	.word		0x141c1a00		/* must issue twice */
1041	depwi		0,18,1, %r28		/* Clear DHE (dcache hash enable) */
1042	depwi		0,20,1, %r28		/* Clear IHE (icache hash enable) */
1043	.word		0x141c1600		/* mtdiag %r28, %dr0 */
1044	.word		0x141c1600		/* must issue twice */
1045	b,n		srdis_done
1046
1047srdis_pcxl:
1048
1049	/* Disable Space Register Hashing for PCXL */
1050
1051	.word		0x141c0600		/* mfdiag %dr0, %r28 */
1052	depwi           0,28,2, %r28		/* Clear DHASH_EN & IHASH_EN */
1053	.word		0x141c0240		/* mtdiag %r28, %dr0 */
1054	b,n		srdis_done
1055
1056srdis_pa20:
1057
1058	/* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1059
1060	.word		0x144008bc		/* mfdiag %dr2, %r28 */
1061	depdi		0, 54,1, %r28		/* clear DIAG_SPHASH_ENAB (bit 54) */
1062	.word		0x145c1840		/* mtdiag %r28, %dr2 */
1063
1064
1065srdis_done:
1066	/* Switch back to virtual mode */
1067	rsm		PSW_SM_I, %r0		/* prep to load iia queue */
1068	load32 	   	2f, %r1
1069	nop
1070	nop
1071	nop
1072	nop
1073	nop
1074
1075	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1076	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1077	mtctl		%r0, %cr17		/* Clear IIASQ head */
1078	mtctl		%r1, %cr18		/* IIAOQ head */
1079	ldo		4(%r1), %r1
1080	mtctl		%r1, %cr18		/* IIAOQ tail */
1081	load32		KERNEL_PSW, %r1
1082	mtctl		%r1, %ipsw
1083	rfi
1084	nop
1085
10862:      bv		%r0(%r2)
1087	nop
1088	.exit
1089
1090	.procend
1091
1092	.end
1093