xref: /linux/arch/powerpc/kernel/misc_64.S (revision 606d099cdd1080bbb50ea50dc52d98252f8f10a1) 
1/*
2 * This file contains miscellaneous low-level functions.
3 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 *
5 * Largely rewritten by Cort Dougan (cort@cs.nmt.edu)
6 * and Paul Mackerras.
7 * Adapted for iSeries by Mike Corrigan (mikejc@us.ibm.com)
8 * PPC64 updates by Dave Engebretsen (engebret@us.ibm.com)
9 *
10 * This program is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU General Public License
12 * as published by the Free Software Foundation; either version
13 * 2 of the License, or (at your option) any later version.
14 *
15 */
16
17#include <linux/sys.h>
18#include <asm/unistd.h>
19#include <asm/errno.h>
20#include <asm/processor.h>
21#include <asm/page.h>
22#include <asm/cache.h>
23#include <asm/ppc_asm.h>
24#include <asm/asm-offsets.h>
25#include <asm/cputable.h>
26#include <asm/thread_info.h>
27
28	.text
29
30_GLOBAL(get_msr)
31	mfmsr	r3
32	blr
33
34_GLOBAL(get_srr0)
35	mfsrr0  r3
36	blr
37
38_GLOBAL(get_srr1)
39	mfsrr1  r3
40	blr
41
42#ifdef CONFIG_IRQSTACKS
43_GLOBAL(call_do_softirq)
44	mflr	r0
45	std	r0,16(r1)
46	stdu	r1,THREAD_SIZE-112(r3)
47	mr	r1,r3
48	bl	.__do_softirq
49	ld	r1,0(r1)
50	ld	r0,16(r1)
51	mtlr	r0
52	blr
53
54_GLOBAL(call_handle_irq)
55	ld	r8,0(r6)
56	mflr	r0
57	std	r0,16(r1)
58	mtctr	r8
59	stdu	r1,THREAD_SIZE-112(r5)
60	mr	r1,r5
61	bctrl
62	ld	r1,0(r1)
63	ld	r0,16(r1)
64	mtlr	r0
65	blr
66#endif /* CONFIG_IRQSTACKS */
67
68	.section	".toc","aw"
69PPC64_CACHES:
70	.tc		ppc64_caches[TC],ppc64_caches
71	.section	".text"
72
73/*
74 * Write any modified data cache blocks out to memory
75 * and invalidate the corresponding instruction cache blocks.
76 *
77 * flush_icache_range(unsigned long start, unsigned long stop)
78 *
79 *   flush all bytes from start through stop-1 inclusive
80 */
81
82_KPROBE(__flush_icache_range)
83
84/*
85 * Flush the data cache to memory
86 *
87 * Different systems have different cache line sizes
88 * and in some cases i-cache and d-cache line sizes differ from
89 * each other.
90 */
91 	ld	r10,PPC64_CACHES@toc(r2)
92	lwz	r7,DCACHEL1LINESIZE(r10)/* Get cache line size */
93	addi	r5,r7,-1
94	andc	r6,r3,r5		/* round low to line bdy */
95	subf	r8,r6,r4		/* compute length */
96	add	r8,r8,r5		/* ensure we get enough */
97	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of cache line size */
98	srw.	r8,r8,r9		/* compute line count */
99	beqlr				/* nothing to do? */
100	mtctr	r8
1011:	dcbst	0,r6
102	add	r6,r6,r7
103	bdnz	1b
104	sync
105
106/* Now invalidate the instruction cache */
107
108	lwz	r7,ICACHEL1LINESIZE(r10)	/* Get Icache line size */
109	addi	r5,r7,-1
110	andc	r6,r3,r5		/* round low to line bdy */
111	subf	r8,r6,r4		/* compute length */
112	add	r8,r8,r5
113	lwz	r9,ICACHEL1LOGLINESIZE(r10)	/* Get log-2 of Icache line size */
114	srw.	r8,r8,r9		/* compute line count */
115	beqlr				/* nothing to do? */
116	mtctr	r8
1172:	icbi	0,r6
118	add	r6,r6,r7
119	bdnz	2b
120	isync
121	blr
122	.previous .text
123/*
124 * Like above, but only do the D-cache.
125 *
126 * flush_dcache_range(unsigned long start, unsigned long stop)
127 *
128 *    flush all bytes from start to stop-1 inclusive
129 */
130_GLOBAL(flush_dcache_range)
131
132/*
133 * Flush the data cache to memory
134 *
135 * Different systems have different cache line sizes
136 */
137 	ld	r10,PPC64_CACHES@toc(r2)
138	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
139	addi	r5,r7,-1
140	andc	r6,r3,r5		/* round low to line bdy */
141	subf	r8,r6,r4		/* compute length */
142	add	r8,r8,r5		/* ensure we get enough */
143	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of dcache line size */
144	srw.	r8,r8,r9		/* compute line count */
145	beqlr				/* nothing to do? */
146	mtctr	r8
1470:	dcbst	0,r6
148	add	r6,r6,r7
149	bdnz	0b
150	sync
151	blr
152
153/*
154 * Like above, but works on non-mapped physical addresses.
155 * Use only for non-LPAR setups ! It also assumes real mode
156 * is cacheable. Used for flushing out the DART before using
157 * it as uncacheable memory
158 *
159 * flush_dcache_phys_range(unsigned long start, unsigned long stop)
160 *
161 *    flush all bytes from start to stop-1 inclusive
162 */
163_GLOBAL(flush_dcache_phys_range)
164 	ld	r10,PPC64_CACHES@toc(r2)
165	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
166	addi	r5,r7,-1
167	andc	r6,r3,r5		/* round low to line bdy */
168	subf	r8,r6,r4		/* compute length */
169	add	r8,r8,r5		/* ensure we get enough */
170	lwz	r9,DCACHEL1LOGLINESIZE(r10)	/* Get log-2 of dcache line size */
171	srw.	r8,r8,r9		/* compute line count */
172	beqlr				/* nothing to do? */
173	mfmsr	r5			/* Disable MMU Data Relocation */
174	ori	r0,r5,MSR_DR
175	xori	r0,r0,MSR_DR
176	sync
177	mtmsr	r0
178	sync
179	isync
180	mtctr	r8
1810:	dcbst	0,r6
182	add	r6,r6,r7
183	bdnz	0b
184	sync
185	isync
186	mtmsr	r5			/* Re-enable MMU Data Relocation */
187	sync
188	isync
189	blr
190
191_GLOBAL(flush_inval_dcache_range)
192 	ld	r10,PPC64_CACHES@toc(r2)
193	lwz	r7,DCACHEL1LINESIZE(r10)	/* Get dcache line size */
194	addi	r5,r7,-1
195	andc	r6,r3,r5		/* round low to line bdy */
196	subf	r8,r6,r4		/* compute length */
197	add	r8,r8,r5		/* ensure we get enough */
198	lwz	r9,DCACHEL1LOGLINESIZE(r10)/* Get log-2 of dcache line size */
199	srw.	r8,r8,r9		/* compute line count */
200	beqlr				/* nothing to do? */
201	sync
202	isync
203	mtctr	r8
2040:	dcbf	0,r6
205	add	r6,r6,r7
206	bdnz	0b
207	sync
208	isync
209	blr
210
211
212/*
213 * Flush a particular page from the data cache to RAM.
214 * Note: this is necessary because the instruction cache does *not*
215 * snoop from the data cache.
216 *
217 *	void __flush_dcache_icache(void *page)
218 */
219_GLOBAL(__flush_dcache_icache)
220/*
221 * Flush the data cache to memory
222 *
223 * Different systems have different cache line sizes
224 */
225
226/* Flush the dcache */
227 	ld	r7,PPC64_CACHES@toc(r2)
228	clrrdi	r3,r3,PAGE_SHIFT           	    /* Page align */
229	lwz	r4,DCACHEL1LINESPERPAGE(r7)	/* Get # dcache lines per page */
230	lwz	r5,DCACHEL1LINESIZE(r7)		/* Get dcache line size */
231	mr	r6,r3
232	mtctr	r4
2330:	dcbst	0,r6
234	add	r6,r6,r5
235	bdnz	0b
236	sync
237
238/* Now invalidate the icache */
239
240	lwz	r4,ICACHEL1LINESPERPAGE(r7)	/* Get # icache lines per page */
241	lwz	r5,ICACHEL1LINESIZE(r7)		/* Get icache line size */
242	mtctr	r4
2431:	icbi	0,r3
244	add	r3,r3,r5
245	bdnz	1b
246	isync
247	blr
248
249
250#if defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE)
251/*
252 * Do an IO access in real mode
253 */
254_GLOBAL(real_readb)
255	mfmsr	r7
256	ori	r0,r7,MSR_DR
257	xori	r0,r0,MSR_DR
258	sync
259	mtmsrd	r0
260	sync
261	isync
262	mfspr	r6,SPRN_HID4
263	rldicl	r5,r6,32,0
264	ori	r5,r5,0x100
265	rldicl	r5,r5,32,0
266	sync
267	mtspr	SPRN_HID4,r5
268	isync
269	slbia
270	isync
271	lbz	r3,0(r3)
272	sync
273	mtspr	SPRN_HID4,r6
274	isync
275	slbia
276	isync
277	mtmsrd	r7
278	sync
279	isync
280	blr
281
282	/*
283 * Do an IO access in real mode
284 */
285_GLOBAL(real_writeb)
286	mfmsr	r7
287	ori	r0,r7,MSR_DR
288	xori	r0,r0,MSR_DR
289	sync
290	mtmsrd	r0
291	sync
292	isync
293	mfspr	r6,SPRN_HID4
294	rldicl	r5,r6,32,0
295	ori	r5,r5,0x100
296	rldicl	r5,r5,32,0
297	sync
298	mtspr	SPRN_HID4,r5
299	isync
300	slbia
301	isync
302	stb	r3,0(r4)
303	sync
304	mtspr	SPRN_HID4,r6
305	isync
306	slbia
307	isync
308	mtmsrd	r7
309	sync
310	isync
311	blr
312#endif /* defined(CONFIG_PPC_PMAC) || defined(CONFIG_PPC_MAPLE) */
313
314#ifdef CONFIG_CPU_FREQ_PMAC64
315/*
316 * SCOM access functions for 970 (FX only for now)
317 *
318 * unsigned long scom970_read(unsigned int address);
319 * void scom970_write(unsigned int address, unsigned long value);
320 *
321 * The address passed in is the 24 bits register address. This code
322 * is 970 specific and will not check the status bits, so you should
323 * know what you are doing.
324 */
325_GLOBAL(scom970_read)
326	/* interrupts off */
327	mfmsr	r4
328	ori	r0,r4,MSR_EE
329	xori	r0,r0,MSR_EE
330	mtmsrd	r0,1
331
332	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
333	 * (including parity). On current CPUs they must be 0'd,
334	 * and finally or in RW bit
335	 */
336	rlwinm	r3,r3,8,0,15
337	ori	r3,r3,0x8000
338
339	/* do the actual scom read */
340	sync
341	mtspr	SPRN_SCOMC,r3
342	isync
343	mfspr	r3,SPRN_SCOMD
344	isync
345	mfspr	r0,SPRN_SCOMC
346	isync
347
348	/* XXX:	fixup result on some buggy 970's (ouch ! we lost a bit, bah
349	 * that's the best we can do). Not implemented yet as we don't use
350	 * the scom on any of the bogus CPUs yet, but may have to be done
351	 * ultimately
352	 */
353
354	/* restore interrupts */
355	mtmsrd	r4,1
356	blr
357
358
359_GLOBAL(scom970_write)
360	/* interrupts off */
361	mfmsr	r5
362	ori	r0,r5,MSR_EE
363	xori	r0,r0,MSR_EE
364	mtmsrd	r0,1
365
366	/* rotate 24 bits SCOM address 8 bits left and mask out it's low 8 bits
367	 * (including parity). On current CPUs they must be 0'd.
368	 */
369
370	rlwinm	r3,r3,8,0,15
371
372	sync
373	mtspr	SPRN_SCOMD,r4      /* write data */
374	isync
375	mtspr	SPRN_SCOMC,r3      /* write command */
376	isync
377	mfspr	3,SPRN_SCOMC
378	isync
379
380	/* restore interrupts */
381	mtmsrd	r5,1
382	blr
383#endif /* CONFIG_CPU_FREQ_PMAC64 */
384
385
386/*
387 * Create a kernel thread
388 *   kernel_thread(fn, arg, flags)
389 */
390_GLOBAL(kernel_thread)
391	std	r29,-24(r1)
392	std	r30,-16(r1)
393	stdu	r1,-STACK_FRAME_OVERHEAD(r1)
394	mr	r29,r3
395	mr	r30,r4
396	ori	r3,r5,CLONE_VM	/* flags */
397	oris	r3,r3,(CLONE_UNTRACED>>16)
398	li	r4,0		/* new sp (unused) */
399	li	r0,__NR_clone
400	sc
401	cmpdi	0,r3,0		/* parent or child? */
402	bne	1f		/* return if parent */
403	li	r0,0
404	stdu	r0,-STACK_FRAME_OVERHEAD(r1)
405	ld	r2,8(r29)
406	ld	r29,0(r29)
407	mtlr	r29              /* fn addr in lr */
408	mr	r3,r30	        /* load arg and call fn */
409	blrl
410	li	r0,__NR_exit	/* exit after child exits */
411        li	r3,0
412	sc
4131:	addi	r1,r1,STACK_FRAME_OVERHEAD
414	ld	r29,-24(r1)
415	ld	r30,-16(r1)
416	blr
417
418/*
419 * disable_kernel_fp()
420 * Disable the FPU.
421 */
422_GLOBAL(disable_kernel_fp)
423	mfmsr	r3
424	rldicl	r0,r3,(63-MSR_FP_LG),1
425	rldicl	r3,r0,(MSR_FP_LG+1),0
426	mtmsrd	r3			/* disable use of fpu now */
427	isync
428	blr
429
430#ifdef CONFIG_ALTIVEC
431
432#if 0 /* this has no callers for now */
433/*
434 * disable_kernel_altivec()
435 * Disable the VMX.
436 */
437_GLOBAL(disable_kernel_altivec)
438	mfmsr	r3
439	rldicl	r0,r3,(63-MSR_VEC_LG),1
440	rldicl	r3,r0,(MSR_VEC_LG+1),0
441	mtmsrd	r3			/* disable use of VMX now */
442	isync
443	blr
444#endif /* 0 */
445
446/*
447 * giveup_altivec(tsk)
448 * Disable VMX for the task given as the argument,
449 * and save the vector registers in its thread_struct.
450 * Enables the VMX for use in the kernel on return.
451 */
452_GLOBAL(giveup_altivec)
453	mfmsr	r5
454	oris	r5,r5,MSR_VEC@h
455	mtmsrd	r5			/* enable use of VMX now */
456	isync
457	cmpdi	0,r3,0
458	beqlr-				/* if no previous owner, done */
459	addi	r3,r3,THREAD		/* want THREAD of task */
460	ld	r5,PT_REGS(r3)
461	cmpdi	0,r5,0
462	SAVE_32VRS(0,r4,r3)
463	mfvscr	vr0
464	li	r4,THREAD_VSCR
465	stvx	vr0,r4,r3
466	beq	1f
467	ld	r4,_MSR-STACK_FRAME_OVERHEAD(r5)
468	lis	r3,MSR_VEC@h
469	andc	r4,r4,r3		/* disable FP for previous task */
470	std	r4,_MSR-STACK_FRAME_OVERHEAD(r5)
4711:
472#ifndef CONFIG_SMP
473	li	r5,0
474	ld	r4,last_task_used_altivec@got(r2)
475	std	r5,0(r4)
476#endif /* CONFIG_SMP */
477	blr
478
479#endif /* CONFIG_ALTIVEC */
480
481_GLOBAL(kernel_execve)
482	li	r0,__NR_execve
483	sc
484	bnslr
485	neg	r3,r3
486	blr
487
488/* kexec_wait(phys_cpu)
489 *
490 * wait for the flag to change, indicating this kernel is going away but
491 * the slave code for the next one is at addresses 0 to 100.
492 *
493 * This is used by all slaves.
494 *
495 * Physical (hardware) cpu id should be in r3.
496 */
497_GLOBAL(kexec_wait)
498	bl	1f
4991:	mflr	r5
500	addi	r5,r5,kexec_flag-1b
501
50299:	HMT_LOW
503#ifdef CONFIG_KEXEC		/* use no memory without kexec */
504	lwz	r4,0(r5)
505	cmpwi	0,r4,0
506	bnea	0x60
507#endif
508	b	99b
509
510/* this can be in text because we won't change it until we are
511 * running in real anyways
512 */
513kexec_flag:
514	.long	0
515
516
517#ifdef CONFIG_KEXEC
518
519/* kexec_smp_wait(void)
520 *
521 * call with interrupts off
522 * note: this is a terminal routine, it does not save lr
523 *
524 * get phys id from paca
525 * set paca id to -1 to say we got here
526 * switch to real mode
527 * join other cpus in kexec_wait(phys_id)
528 */
529_GLOBAL(kexec_smp_wait)
530	lhz	r3,PACAHWCPUID(r13)
531	li	r4,-1
532	sth	r4,PACAHWCPUID(r13)	/* let others know we left */
533	bl	real_mode
534	b	.kexec_wait
535
536/*
537 * switch to real mode (turn mmu off)
538 * we use the early kernel trick that the hardware ignores bits
539 * 0 and 1 (big endian) of the effective address in real mode
540 *
541 * don't overwrite r3 here, it is live for kexec_wait above.
542 */
543real_mode:	/* assume normal blr return */
5441:	li	r9,MSR_RI
545	li	r10,MSR_DR|MSR_IR
546	mflr	r11		/* return address to SRR0 */
547	mfmsr	r12
548	andc	r9,r12,r9
549	andc	r10,r12,r10
550
551	mtmsrd	r9,1
552	mtspr	SPRN_SRR1,r10
553	mtspr	SPRN_SRR0,r11
554	rfid
555
556
557/*
558 * kexec_sequence(newstack, start, image, control, clear_all())
559 *
560 * does the grungy work with stack switching and real mode switches
561 * also does simple calls to other code
562 */
563
564_GLOBAL(kexec_sequence)
565	mflr	r0
566	std	r0,16(r1)
567
568	/* switch stacks to newstack -- &kexec_stack.stack */
569	stdu	r1,THREAD_SIZE-112(r3)
570	mr	r1,r3
571
572	li	r0,0
573	std	r0,16(r1)
574
575	/* save regs for local vars on new stack.
576	 * yes, we won't go back, but ...
577	 */
578	std	r31,-8(r1)
579	std	r30,-16(r1)
580	std	r29,-24(r1)
581	std	r28,-32(r1)
582	std	r27,-40(r1)
583	std	r26,-48(r1)
584	std	r25,-56(r1)
585
586	stdu	r1,-112-64(r1)
587
588	/* save args into preserved regs */
589	mr	r31,r3			/* newstack (both) */
590	mr	r30,r4			/* start (real) */
591	mr	r29,r5			/* image (virt) */
592	mr	r28,r6			/* control, unused */
593	mr	r27,r7			/* clear_all() fn desc */
594	mr	r26,r8			/* spare */
595	lhz	r25,PACAHWCPUID(r13)	/* get our phys cpu from paca */
596
597	/* disable interrupts, we are overwriting kernel data next */
598	mfmsr	r3
599	rlwinm	r3,r3,0,17,15
600	mtmsrd	r3,1
601
602	/* copy dest pages, flush whole dest image */
603	mr	r3,r29
604	bl	.kexec_copy_flush	/* (image) */
605
606	/* turn off mmu */
607	bl	real_mode
608
609	/* clear out hardware hash page table and tlb */
610	ld	r5,0(r27)		/* deref function descriptor */
611	mtctr	r5
612	bctrl				/* ppc_md.hpte_clear_all(void); */
613
614/*
615 *   kexec image calling is:
616 *      the first 0x100 bytes of the entry point are copied to 0
617 *
618 *      all slaves branch to slave = 0x60 (absolute)
619 *              slave(phys_cpu_id);
620 *
621 *      master goes to start = entry point
622 *              start(phys_cpu_id, start, 0);
623 *
624 *
625 *   a wrapper is needed to call existing kernels, here is an approximate
626 *   description of one method:
627 *
628 * v2: (2.6.10)
629 *   start will be near the boot_block (maybe 0x100 bytes before it?)
630 *   it will have a 0x60, which will b to boot_block, where it will wait
631 *   and 0 will store phys into struct boot-block and load r3 from there,
632 *   copy kernel 0-0x100 and tell slaves to back down to 0x60 again
633 *
634 * v1: (2.6.9)
635 *    boot block will have all cpus scanning device tree to see if they
636 *    are the boot cpu ?????
637 *    other device tree differences (prop sizes, va vs pa, etc)...
638 */
639
640	/* copy  0x100 bytes starting at start to 0 */
641	li	r3,0
642	mr	r4,r30
643	li	r5,0x100
644	li	r6,0
645	bl	.copy_and_flush	/* (dest, src, copy limit, start offset) */
6461:	/* assume normal blr return */
647
648	/* release other cpus to the new kernel secondary start at 0x60 */
649	mflr	r5
650	li	r6,1
651	stw	r6,kexec_flag-1b(5)
652	mr	r3,r25	# my phys cpu
653	mr	r4,r30	# start, aka phys mem offset
654	mtlr	4
655	li	r5,0
656	blr	/* image->start(physid, image->start, 0); */
657#endif /* CONFIG_KEXEC */
658