xref: /linux/arch/powerpc/kernel/misc_64.S (revision 4f58e6dceb0e44ca8f21568ed81e1df24e55964c) 
1/*
2 * This file contains miscellaneous low-level functions.
3 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
6 * and Paul Mackerras.
7 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
8 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 *
15 */
16
17#include <linux/sys.h>
18#include <asm/unistd.h>
19#include <asm/errno.h>
20#include <asm/processor.h>
21#include <asm/page.h>
22#include <asm/cache.h>
23#include <asm/ppc_asm.h>
24#include <asm/asm-offsets.h>
25#include <asm/cputable.h>
26#include <asm/thread_info.h>
27#include <asm/kexec.h>
28#include <asm/ptrace.h>
29#include <asm/mmu.h>
30
31	.text
32
33_GLOBAL(call_do_softirq)
34	mflr	r0
35	std	r0,16(r1)
36	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
37	mr	r1,r3
38	bl	__do_softirq
39	ld	r1,0(r1)
40	ld	r0,16(r1)
41	mtlr	r0
42	blr
43
44_GLOBAL(call_do_irq)
45	mflr	r0
46	std	r0,16(r1)
47	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r4)
48	mr	r1,r4
49	bl	__do_irq
50	ld	r1,0(r1)
51	ld	r0,16(r1)
52	mtlr	r0
53	blr
54
55	.section	".toc","aw"
56PPC64_CACHES:
57	.tc		ppc64_caches[TC],ppc64_caches
58	.section	".text"
59
60/*
61 * Write any modified data cache blocks out to memory
62 * and invalidate the corresponding instruction cache blocks.
63 *
64 * flush_icache_range(unsigned long start, unsigned long stop)
65 *
66 *   flush all bytes from start through stop-1 inclusive
67 */
68
69_GLOBAL(flush_icache_range)
70BEGIN_FTR_SECTION
71	PURGE_PREFETCHED_INS
72	blr
73END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
74/*
75 * Flush the data cache to memory
76 *
77 * Different systems have different cache line sizes
78 * and in some cases i-cache and d-cache line sizes differ from
79 * each other.
80 */
81 	ld	r10,PPC64_CACHES@toc(r2)
82	lwz	r7,DCACHEL1LINESIZE(r10)/* Get cache line size */
83	addi	r5,r7,-1
84	andc	r6,r3,r5		/* round low to line bdy */
85	subf	r8,r6,r4		/* compute length */
86	add	r8,r8,r5		/* ensure we get enough */
87	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of cache line size */
88	srw.	r8,r8,r9		/* compute line count */
89	beqlr				/* nothing to do? */
90	mtctr	r8
911:	dcbst	0,r6
92	add	r6,r6,r7
93	bdnz	1b
94	sync
95
96/* Now invalidate the instruction cache */
97
98	lwz	r7,ICACHEL1LINESIZE(r10)	/* Get Icache line size */
99	addi	r5,r7,-1
100	andc	r6,r3,r5		/* round low to line bdy */
101	subf	r8,r6,r4		/* compute length */
102	add	r8,r8,r5
103	lwz	r9,ICACHEL1LOGLINESIZE(r10)	/* Get log-2 of Icache line size */
104	srw.	r8,r8,r9		/* compute line count */
105	beqlr				/* nothing to do? */
106	mtctr	r8
1072:	icbi	0,r6
108	add	r6,r6,r7
109	bdnz	2b
110	isync
111	blr
112_ASM_NOKPROBE_SYMBOL(flush_icache_range)
113
114/*
115 * Like above, but only do the D-cache.
116 *
117 * flush_dcache_range(unsigned long start, unsigned long stop)
118 *
119 *    flush all bytes from start to stop-1 inclusive
120 */
121_GLOBAL(flush_dcache_range)
122
123/*
124 * Flush the data cache to memory
125 *
126 * Different systems have different cache line sizes
127 */
128 	ld	r10,PPC64_CACHES@toc(r2)
129	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
130	addi	r5,r7,-1
131	andc	r6,r3,r5		/* round low to line bdy */
132	subf	r8,r6,r4		/* compute length */
133	add	r8,r8,r5		/* ensure we get enough */
134	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of dcache line size */
135	srw.	r8,r8,r9		/* compute line count */
136	beqlr				/* nothing to do? */
137	mtctr	r8
1380:	dcbst	0,r6
139	add	r6,r6,r7
140	bdnz	0b
141	sync
142	blr
143
144/*
145 * Like above, but works on non-mapped physical addresses.
146 * Use only for non-LPAR setups ! It also assumes real mode
147 * is cacheable. Used for flushing out the DART before using
148 * it as uncacheable memory
149 *
150 * flush_dcache_phys_range(unsigned long start, unsigned long stop)
151 *
152 *    flush all bytes from start to stop-1 inclusive
153 */
154_GLOBAL(flush_dcache_phys_range)
155 	ld	r10,PPC64_CACHES@toc(r2)
156	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
157	addi	r5,r7,-1
158	andc	r6,r3,r5		/* round low to line bdy */
159	subf	r8,r6,r4		/* compute length */
160	add	r8,r8,r5		/* ensure we get enough */
161	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of dcache line size */
162	srw.	r8,r8,r9		/* compute line count */
163	beqlr				/* nothing to do? */
164	mfmsr	r5			/* Disable MMU Data Relocation */
165	ori	r0,r5,MSR_DR
166	xori	r0,r0,MSR_DR
167	sync
168	mtmsr	r0
169	sync
170	isync
171	mtctr	r8
1720:	dcbst	0,r6
173	add	r6,r6,r7
174	bdnz	0b
175	sync
176	isync
177	mtmsr	r5			/* Re-enable MMU Data Relocation */
178	sync
179	isync
180	blr
181
182_GLOBAL(flush_inval_dcache_range)
183 	ld	r10,PPC64_CACHES@toc(r2)
184	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
185	addi	r5,r7,-1
186	andc	r6,r3,r5		/* round low to line bdy */
187	subf	r8,r6,r4		/* compute length */
188	add	r8,r8,r5		/* ensure we get enough */
189	lwz	r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */
190	srw.	r8,r8,r9		/* compute line count */
191	beqlr				/* nothing to do? */
192	sync
193	isync
194	mtctr	r8
1950:	dcbf	0,r6
196	add	r6,r6,r7
197	bdnz	0b
198	sync
199	isync
200	blr
201
202
203/*
204 * Flush a particular page from the data cache to RAM.
205 * Note: this is necessary because the instruction cache does *not*
206 * snoop from the data cache.
207 *
208 *	void __flush_dcache_icache(void *page)
209 */
210_GLOBAL(__flush_dcache_icache)
211/*
212 * Flush the data cache to memory
213 *
214 * Different systems have different cache line sizes
215 */
216
217BEGIN_FTR_SECTION
218	PURGE_PREFETCHED_INS
219	blr
220END_FTR_SECTION_IFSET(CPU_FTR_COHERENT_ICACHE)
221
222/* Flush the dcache */
223 	ld	r7,PPC64_CACHES@toc(r2)
224	clrrdi	r3,r3,PAGE_SHIFT           	    /* Page align */
225	lwz	r4,DCACHEL1LINESPERPAGE(r7)	/* Get # dcache lines per page */
226	lwz	r5,DCACHEL1LINESIZE(r7)		/* Get dcache line size */
227	mr	r6,r3
228	mtctr	r4
2290:	dcbst	0,r6
230	add	r6,r6,r5
231	bdnz	0b
232	sync
233
234/* Now invalidate the icache */
235
236	lwz	r4,ICACHEL1LINESPERPAGE(r7)	/* Get # icache lines per page */
237	lwz	r5,ICACHEL1LINESIZE(r7)		/* Get icache line size */
238	mtctr	r4
2391:	icbi	0,r3
240	add	r3,r3,r5
241	bdnz	1b
242	isync
243	blr
244
245_GLOBAL(__bswapdi2)
246	srdi	r8,r3,32
247	rlwinm	r7,r3,8,0xffffffff
248	rlwimi	r7,r3,24,0,7
249	rlwinm	r9,r8,8,0xffffffff
250	rlwimi	r7,r3,24,16,23
251	rlwimi	r9,r8,24,0,7
252	rlwimi	r9,r8,24,16,23
253	sldi	r7,r7,32
254	or	r3,r7,r9
255	blr
256
257
258#ifdef CONFIG_PPC_EARLY_DEBUG_BOOTX
259_GLOBAL(rmci_on)
260	sync
261	isync
262	li	r3,0x100
263	rldicl	r3,r3,32,0
264	mfspr	r5,SPRN_HID4
265	or	r5,r5,r3
266	sync
267	mtspr	SPRN_HID4,r5
268	isync
269	slbia
270	isync
271	sync
272	blr
273
274_GLOBAL(rmci_off)
275	sync
276	isync
277	li	r3,0x100
278	rldicl	r3,r3,32,0
279	mfspr	r5,SPRN_HID4
280	andc	r5,r5,r3
281	sync
282	mtspr	SPRN_HID4,r5
283	isync
284	slbia
285	isync
286	sync
287	blr
288#endif /* CONFIG_PPC_EARLY_DEBUG_BOOTX */
289
290#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE)
291
292/*
293 * Do an IO access in real mode
294 */
295_GLOBAL(real_readb)
296	mfmsr	r7
297	ori	r0,r7,MSR_DR
298	xori	r0,r0,MSR_DR
299	sync
300	mtmsrd	r0
301	sync
302	isync
303	mfspr	r6,SPRN_HID4
304	rldicl	r5,r6,32,0
305	ori	r5,r5,0x100
306	rldicl	r5,r5,32,0
307	sync
308	mtspr	SPRN_HID4,r5
309	isync
310	slbia
311	isync
312	lbz	r3,0(r3)
313	sync
314	mtspr	SPRN_HID4,r6
315	isync
316	slbia
317	isync
318	mtmsrd	r7
319	sync
320	isync
321	blr
322
323	/*
324 * Do an IO access in real mode
325 */
326_GLOBAL(real_writeb)
327	mfmsr	r7
328	ori	r0,r7,MSR_DR
329	xori	r0,r0,MSR_DR
330	sync
331	mtmsrd	r0
332	sync
333	isync
334	mfspr	r6,SPRN_HID4
335	rldicl	r5,r6,32,0
336	ori	r5,r5,0x100
337	rldicl	r5,r5,32,0
338	sync
339	mtspr	SPRN_HID4,r5
340	isync
341	slbia
342	isync
343	stb	r3,0(r4)
344	sync
345	mtspr	SPRN_HID4,r6
346	isync
347	slbia
348	isync
349	mtmsrd	r7
350	sync
351	isync
352	blr
353#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */
354
355#ifdef CONFIG_PPC_PASEMI
356
357_GLOBAL(real_205_readb)
358	mfmsr	r7
359	ori	r0,r7,MSR_DR
360	xori	r0,r0,MSR_DR
361	sync
362	mtmsrd	r0
363	sync
364	isync
365	LBZCIX(R3,R0,R3)
366	isync
367	mtmsrd	r7
368	sync
369	isync
370	blr
371
372_GLOBAL(real_205_writeb)
373	mfmsr	r7
374	ori	r0,r7,MSR_DR
375	xori	r0,r0,MSR_DR
376	sync
377	mtmsrd	r0
378	sync
379	isync
380	STBCIX(R3,R0,R4)
381	isync
382	mtmsrd	r7
383	sync
384	isync
385	blr
386
387#endif /* CONFIG_PPC_PASEMI */
388
389
390#if defined(CONFIG_CPU_FREQ_PMAC64) || defined(CONFIG_CPU_FREQ_MAPLE)
391/*
392 * SCOM access functions for 970 (FX only for now)
393 *
394 * unsigned long scom970_read(unsigned int address);
395 * void scom970_write(unsigned int address, unsigned long value);
396 *
397 * The address passed in is the 24 bits register address. This code
398 * is 970 specific and will not check the status bits, so you should
399 * know what you are doing.
400 */
401_GLOBAL(scom970_read)
402	/* interrupts off */
403	mfmsr	r4
404	ori	r0,r4,MSR_EE
405	xori	r0,r0,MSR_EE
406	mtmsrd	r0,1
407
408	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
409	 * (including parity). On current CPUs they must be 0'd,
410	 * and finally or in RW bit
411	 */
412	rlwinm	r3,r3,8,0,15
413	ori	r3,r3,0x8000
414
415	/* do the actual scom read */
416	sync
417	mtspr	SPRN_SCOMC,r3
418	isync
419	mfspr	r3,SPRN_SCOMD
420	isync
421	mfspr	r0,SPRN_SCOMC
422	isync
423
424	/* XXX:	fixup result on some buggy 970's (ouch ! we lost a bit, bah
425	 * that's the best we can do). Not implemented yet as we don't use
426	 * the scom on any of the bogus CPUs yet, but may have to be done
427	 * ultimately
428	 */
429
430	/* restore interrupts */
431	mtmsrd	r4,1
432	blr
433
434
435_GLOBAL(scom970_write)
436	/* interrupts off */
437	mfmsr	r5
438	ori	r0,r5,MSR_EE
439	xori	r0,r0,MSR_EE
440	mtmsrd	r0,1
441
442	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
443	 * (including parity). On current CPUs they must be 0'd.
444	 */
445
446	rlwinm	r3,r3,8,0,15
447
448	sync
449	mtspr	SPRN_SCOMD,r4      /* write data */
450	isync
451	mtspr	SPRN_SCOMC,r3      /* write command */
452	isync
453	mfspr	3,SPRN_SCOMC
454	isync
455
456	/* restore interrupts */
457	mtmsrd	r5,1
458	blr
459#endif /* CONFIG_CPU_FREQ_PMAC64 || CONFIG_CPU_FREQ_MAPLE */
460
461/* kexec_wait(phys_cpu)
462 *
463 * wait for the flag to change, indicating this kernel is going away but
464 * the slave code for the next one is at addresses 0 to 100.
465 *
466 * This is used by all slaves, even those that did not find a matching
467 * paca in the secondary startup code.
468 *
469 * Physical (hardware) cpu id should be in r3.
470 */
471_GLOBAL(kexec_wait)
472	bl	1f
4731:	mflr	r5
474	addi	r5,r5,kexec_flag-1b
475
47699:	HMT_LOW
477#ifdef CONFIG_KEXEC		/* use no memory without kexec */
478	lwz	r4,0(r5)
479	cmpwi	0,r4,0
480	beq	99b
481#ifdef CONFIG_PPC_BOOK3S_64
482	li	r10,0x60
483	mfmsr	r11
484	clrrdi	r11,r11,1	/* Clear MSR_LE */
485	mtsrr0	r10
486	mtsrr1	r11
487	rfid
488#else
489	/* Create TLB entry in book3e_secondary_core_init */
490	li	r4,0
491	ba	0x60
492#endif
493#endif
494
495/* this can be in text because we won't change it until we are
496 * running in real anyways
497 */
498kexec_flag:
499	.long	0
500
501
502#ifdef CONFIG_KEXEC
503#ifdef CONFIG_PPC_BOOK3E
504/*
505 * BOOK3E has no real MMU mode, so we have to setup the initial TLB
506 * for a core to identity map v:0 to p:0.  This current implementation
507 * assumes that 1G is enough for kexec.
508 */
509kexec_create_tlb:
510	/*
511	 * Invalidate all non-IPROT TLB entries to avoid any TLB conflict.
512	 * IPROT TLB entries should be >= PAGE_OFFSET and thus not conflict.
513	 */
514	PPC_TLBILX_ALL(0,R0)
515	sync
516	isync
517
518	mfspr	r10,SPRN_TLB1CFG
519	andi.	r10,r10,TLBnCFG_N_ENTRY	/* Extract # entries */
520	subi	r10,r10,1	/* Last entry: no conflict with kernel text */
521	lis	r9,MAS0_TLBSEL(1)@h
522	rlwimi	r9,r10,16,4,15		/* Setup MAS0 = TLBSEL | ESEL(r9) */
523
524/* Set up a temp identity mapping v:0 to p:0 and return to it. */
525#if defined(CONFIG_SMP) || defined(CONFIG_PPC_E500MC)
526#define M_IF_NEEDED	MAS2_M
527#else
528#define M_IF_NEEDED	0
529#endif
530	mtspr	SPRN_MAS0,r9
531
532	lis	r9,(MAS1_VALID|MAS1_IPROT)@h
533	ori	r9,r9,(MAS1_TSIZE(BOOK3E_PAGESZ_1GB))@l
534	mtspr	SPRN_MAS1,r9
535
536	LOAD_REG_IMMEDIATE(r9, 0x0 | M_IF_NEEDED)
537	mtspr	SPRN_MAS2,r9
538
539	LOAD_REG_IMMEDIATE(r9, 0x0 | MAS3_SR | MAS3_SW | MAS3_SX)
540	mtspr	SPRN_MAS3,r9
541	li	r9,0
542	mtspr	SPRN_MAS7,r9
543
544	tlbwe
545	isync
546	blr
547#endif
548
549/* kexec_smp_wait(void)
550 *
551 * call with interrupts off
552 * note: this is a terminal routine, it does not save lr
553 *
554 * get phys id from paca
555 * switch to real mode
556 * mark the paca as no longer used
557 * join other cpus in kexec_wait(phys_id)
558 */
559_GLOBAL(kexec_smp_wait)
560	lhz	r3,PACAHWCPUID(r13)
561	bl	real_mode
562
563	li	r4,KEXEC_STATE_REAL_MODE
564	stb	r4,PACAKEXECSTATE(r13)
565	SYNC
566
567	b	kexec_wait
568
569/*
570 * switch to real mode (turn mmu off)
571 * we use the early kernel trick that the hardware ignores bits
572 * 0 and 1 (big endian) of the effective address in real mode
573 *
574 * don't overwrite r3 here, it is live for kexec_wait above.
575 */
576real_mode:	/* assume normal blr return */
577#ifdef CONFIG_PPC_BOOK3E
578	/* Create an identity mapping. */
579	b	kexec_create_tlb
580#else
5811:	li	r9,MSR_RI
582	li	r10,MSR_DR|MSR_IR
583	mflr	r11		/* return address to SRR0 */
584	mfmsr	r12
585	andc	r9,r12,r9
586	andc	r10,r12,r10
587
588	mtmsrd	r9,1
589	mtspr	SPRN_SRR1,r10
590	mtspr	SPRN_SRR0,r11
591	rfid
592#endif
593
594/*
595 * kexec_sequence(newstack, start, image, control, clear_all(),
596	          copy_with_mmu_off)
597 *
598 * does the grungy work with stack switching and real mode switches
599 * also does simple calls to other code
600 */
601
602_GLOBAL(kexec_sequence)
603	mflr	r0
604	std	r0,16(r1)
605
606	/* switch stacks to newstack -- &kexec_stack.stack */
607	stdu	r1,THREAD_SIZE-STACK_FRAME_OVERHEAD(r3)
608	mr	r1,r3
609
610	li	r0,0
611	std	r0,16(r1)
612
613	/* save regs for local vars on new stack.
614	 * yes, we won't go back, but ...
615	 */
616	std	r31,-8(r1)
617	std	r30,-16(r1)
618	std	r29,-24(r1)
619	std	r28,-32(r1)
620	std	r27,-40(r1)
621	std	r26,-48(r1)
622	std	r25,-56(r1)
623
624	stdu	r1,-STACK_FRAME_OVERHEAD-64(r1)
625
626	/* save args into preserved regs */
627	mr	r31,r3			/* newstack (both) */
628	mr	r30,r4			/* start (real) */
629	mr	r29,r5			/* image (virt) */
630	mr	r28,r6			/* control, unused */
631	mr	r27,r7			/* clear_all() fn desc */
632	mr	r26,r8			/* copy_with_mmu_off */
633	lhz	r25,PACAHWCPUID(r13)	/* get our phys cpu from paca */
634
635	/* disable interrupts, we are overwriting kernel data next */
636#ifdef CONFIG_PPC_BOOK3E
637	wrteei	0
638#else
639	mfmsr	r3
640	rlwinm	r3,r3,0,17,15
641	mtmsrd	r3,1
642#endif
643
644	/* We need to turn the MMU off unless we are in hash mode
645	 * under a hypervisor
646	 */
647	cmpdi	r26,0
648	beq	1f
649	bl	real_mode
6501:
651	/* copy dest pages, flush whole dest image */
652	mr	r3,r29
653	bl	kexec_copy_flush	/* (image) */
654
655	/* turn off mmu now if not done earlier */
656	cmpdi	r26,0
657	bne	1f
658	bl	real_mode
659
660	/* copy  0x100 bytes starting at start to 0 */
6611:	li	r3,0
662	mr	r4,r30		/* start, aka phys mem offset */
663	li	r5,0x100
664	li	r6,0
665	bl	copy_and_flush	/* (dest, src, copy limit, start offset) */
6661:	/* assume normal blr return */
667
668	/* release other cpus to the new kernel secondary start at 0x60 */
669	mflr	r5
670	li	r6,1
671	stw	r6,kexec_flag-1b(5)
672
673	cmpdi	r27,0
674	beq	1f
675
676	/* clear out hardware hash page table and tlb */
677#ifdef PPC64_ELF_ABI_v1
678	ld	r12,0(r27)		/* deref function descriptor */
679#else
680	mr	r12,r27
681#endif
682	mtctr	r12
683	bctrl				/* mmu_hash_ops.hpte_clear_all(void); */
684
685/*
686 *   kexec image calling is:
687 *      the first 0x100 bytes of the entry point are copied to 0
688 *
689 *      all slaves branch to slave = 0x60 (absolute)
690 *              slave(phys_cpu_id);
691 *
692 *      master goes to start = entry point
693 *              start(phys_cpu_id, start, 0);
694 *
695 *
696 *   a wrapper is needed to call existing kernels, here is an approximate
697 *   description of one method:
698 *
699 * v2: (2.6.10)
700 *   start will be near the boot_block (maybe 0x100 bytes before it?)
701 *   it will have a 0x60, which will b to boot_block, where it will wait
702 *   and 0 will store phys into struct boot-block and load r3 from there,
703 *   copy kernel 0-0x100 and tell slaves to back down to 0x60 again
704 *
705 * v1: (2.6.9)
706 *    boot block will have all cpus scanning device tree to see if they
707 *    are the boot cpu ?????
708 *    other device tree differences (prop sizes, va vs pa, etc)...
709 */
7101:	mr	r3,r25	# my phys cpu
711	mr	r4,r30	# start, aka phys mem offset
712	mtlr	4
713	li	r5,0
714	blr	/* image->start(physid, image->start, 0); */
715#endif /* CONFIG_KEXEC */
716