xref: /freebsd/sys/arm64/arm64/locore.S (revision ef0cb5db0af0d5d5b75b74f8e534fe601b7176d7)
1/*-
2 * Copyright (c) 2012-2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29#include "assym.s"
30#include <sys/syscall.h>
31#include <machine/asm.h>
32#include <machine/armreg.h>
33#include <machine/hypervisor.h>
34#include <machine/param.h>
35#include <machine/pte.h>
36
37#define	VIRT_BITS	39
38
39	.globl	kernbase
40	.set	kernbase, KERNBASE
41
42#define	DEVICE_MEM	0
43#define	NORMAL_UNCACHED	1
44#define	NORMAL_MEM	2
45
46/*
47 * We assume:
48 *  MMU      on with an identity map, or off
49 *  D-Cache: off
50 *  I-Cache: on or off
51 *  We are loaded at a 2MiB aligned address
52 */
53
54#define	INIT_STACK_SIZE	(PAGE_SIZE * 4)
55
56	.text
57	.globl _start
58_start:
59	/* Drop to EL1 */
60	bl	drop_to_el1
61
62	/*
63	 * Disable the MMU. We may have entered the kernel with it on and
64	 * will need to update the tables later. If this has been set up
65	 * with anything other than a VA == PA map then this will fail,
66	 * but in this case the code to find where we are running from
67	 * would have also failed.
68	 */
69	dsb	sy
70	mrs	x2, sctlr_el1
71	bic	x2, x2, SCTLR_M
72	msr	sctlr_el1, x2
73	isb
74
75
76	/* Get the virt -> phys offset */
77	bl	get_virt_delta
78
79	/*
80	 * At this point:
81	 * x29 = PA - VA
82	 * x28 = Our physical load address
83	 */
84
85	/* Create the page tables */
86	bl	create_pagetables
87
88	/*
89	 * At this point:
90	 * x27 = TTBR0 table
91	 * x26 = TTBR1 table
92	 */
93
94	/* Enable the mmu */
95	bl	start_mmu
96
97	/* Jump to the virtual address space */
98	ldr	x15, .Lvirtdone
99	br	x15
100
101virtdone:
102	/* Set up the stack */
103	adr	x25, initstack_end
104	mov	sp, x25
105	sub	sp, sp, #PCB_SIZE
106
107	/* Zero the BSS */
108	ldr	x15, .Lbss
109	ldr	x14, .Lend
1101:
111	str	xzr, [x15], #8
112	cmp	x15, x14
113	b.lo	1b
114
115	/* Backup the module pointer */
116	mov	x1, x0
117
118	/* Make the page table base a virtual address */
119	sub	x26, x26, x29
120
121	sub	sp, sp, #(64 * 4)
122	mov	x0, sp
123
124	/* Degate the delda so it is VA -> PA */
125	neg	x29, x29
126
127	str	x1,  [x0]	/* modulep */
128	str	x26, [x0, 8]	/* kern_l1pt */
129	str	x29, [x0, 16]	/* kern_delta */
130	str	x25, [x0, 24]	/* kern_stack */
131
132	/* trace back starts here */
133	mov	fp, #0
134	/* Branch to C code */
135	bl	initarm
136	bl	mi_startup
137
138	/* We should not get here */
139	brk	0
140
141	.align 3
142.Lvirtdone:
143	.quad	virtdone
144.Lbss:
145	.quad	__bss_start
146.Lend:
147	.quad	_end
148
149/*
150 * If we are started in EL2, configure the required hypervisor
151 * registers and drop to EL1.
152 */
153drop_to_el1:
154	mrs	x1, CurrentEL
155	lsr	x1, x1, #2
156	cmp	x1, #0x2
157	b.eq	1f
158	ret
1591:
160	/* Configure the Hypervisor */
161	mov	x2, #(HCR_RW)
162	msr	hcr_el2, x2
163
164	/* Load the Virtualization Process ID Register */
165	mrs	x2, midr_el1
166	msr	vpidr_el2, x2
167
168	/* Load the Virtualization Multiprocess ID Register */
169	mrs	x2, mpidr_el1
170	msr	vmpidr_el2, x2
171
172	/* Set the bits that need to be 1 in sctlr_el1 */
173	ldr	x2, .Lsctlr_res1
174	msr	sctlr_el1, x2
175
176	/* Don't trap to EL2 for exceptions */
177	mov	x2, #CPTR_RES1
178	msr	cptr_el2, x2
179
180	/* Don't trap to EL2 for CP15 traps */
181	msr	hstr_el2, xzr
182
183	/* Hypervisor trap functions */
184	adr	x2, hyp_vectors
185	msr	vbar_el2, x2
186
187	mov	x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h)
188	msr	spsr_el2, x2
189
190	/* Configure GICv3 CPU interface */
191	mrs	x2, id_aa64pfr0_el1
192	/* Extract GIC bits from the register */
193	ubfx	x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS
194	/* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */
195	cmp	x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT)
196	b.ne	2f
197
198	mrs	x2, icc_sre_el2
199	orr	x2, x2, #ICC_SRE_EL2_EN	/* Enable access from insecure EL1 */
200	msr	icc_sre_el2, x2
201	isb
2022:
203
204	/* Set the address to return to our return address */
205	msr	elr_el2, x30
206
207	eret
208
209	.align 3
210.Lsctlr_res1:
211	.quad SCTLR_RES1
212
213#define	VECT_EMPTY	\
214	.align 7;	\
215	1:	b	1b
216
217	.align 11
218hyp_vectors:
219	VECT_EMPTY	/* Synchronous EL2t */
220	VECT_EMPTY	/* IRQ EL2t */
221	VECT_EMPTY	/* FIQ EL2t */
222	VECT_EMPTY	/* Error EL2t */
223
224	VECT_EMPTY	/* Synchronous EL2h */
225	VECT_EMPTY	/* IRQ EL2h */
226	VECT_EMPTY	/* FIQ EL2h */
227	VECT_EMPTY	/* Error EL2h */
228
229	VECT_EMPTY	/* Synchronous 64-bit EL1 */
230	VECT_EMPTY	/* IRQ 64-bit EL1 */
231	VECT_EMPTY	/* FIQ 64-bit EL1 */
232	VECT_EMPTY	/* Error 64-bit EL1 */
233
234	VECT_EMPTY	/* Synchronous 32-bit EL1 */
235	VECT_EMPTY	/* IRQ 32-bit EL1 */
236	VECT_EMPTY	/* FIQ 32-bit EL1 */
237	VECT_EMPTY	/* Error 32-bit EL1 */
238
239/*
240 * Get the delta between the physical address we were loaded to and the
241 * virtual address we expect to run from. This is used when building the
242 * initial page table.
243 */
244get_virt_delta:
245	/* Load the physical address of virt_map */
246	adr	x29, virt_map
247	/* Load the virtual address of virt_map stored in virt_map */
248	ldr	x28, [x29]
249	/* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */
250	sub	x29, x29, x28
251	/* Find the load address for the kernel */
252	mov	x28, #(KERNBASE)
253	add	x28, x28, x29
254	ret
255
256	.align 3
257virt_map:
258	.quad	virt_map
259
260/*
261 * This builds the page tables containing the identity map, and the kernel
262 * virtual map.
263 *
264 * It relys on:
265 *  We were loaded to an address that is on a 2MiB boundary
266 *  All the memory must not cross a 1GiB boundaty
267 *  x28 contains the physical address we were loaded from
268 *
269 * TODO: This is out of date.
270 *  There are at least 5 pages before that address for the page tables
271 *   The pages used are:
272 *    - The identity (PA = VA) table (TTBR0)
273 *    - The Kernel L1 table          (TTBR1)(not yet)
274 *    -  The PA != VA L2 table to jump into (not yet)
275 *    -  The FDT L2 table                   (not yet)
276 */
277create_pagetables:
278	/* Save the Link register */
279	mov	x5, x30
280
281	/* Clean the page table */
282	adr	x6, pagetable
283	mov	x26, x6
284	adr	x27, pagetable_end
2851:
286	stp	xzr, xzr, [x6], #16
287	stp	xzr, xzr, [x6], #16
288	stp	xzr, xzr, [x6], #16
289	stp	xzr, xzr, [x6], #16
290	cmp	x6, x27
291	b.lo	1b
292
293	/*
294	 * Build the TTBR1 maps.
295	 */
296
297	/* Find the size of the kernel */
298	mov	x6, #(KERNBASE)
299	ldr	x7, .Lend
300	/* Find the end - begin */
301	sub	x8, x7, x6
302	/* Get the number of l2 pages to allocate, rounded down */
303	lsr	x10, x8, #(L2_SHIFT)
304	/* Add 4 MiB for any rounding above and the module data */
305	add	x10, x10, #2
306
307	/* Create the kernel space L2 table */
308	mov	x6, x26
309	mov	x7, #NORMAL_MEM
310	mov	x8, #(KERNBASE & L2_BLOCK_MASK)
311	mov	x9, x28
312	bl	build_block_pagetable
313
314	/* Move to the l1 table */
315	add	x26, x26, #PAGE_SIZE
316
317	/* Link the l1 -> l2 table */
318	mov	x9, x6
319	mov	x6, x26
320	bl	link_l1_pagetable
321
322
323	/*
324	 * Build the TTBR0 maps.
325	 */
326	add	x27, x26, #PAGE_SIZE
327
328#if defined(SOCDEV_PA) && defined(SOCDEV_VA)
329	/* Create a table for the UART */
330	mov	x6, x27		/* The initial page table */
331	mov	x7, #DEVICE_MEM
332	mov	x8, #(SOCDEV_VA)	/* VA start */
333	mov	x9, #(SOCDEV_PA)	/* PA start */
334	bl	build_section_pagetable
335#endif
336
337	/* Create the VA = PA map */
338	mov	x6, x27		/* The initial page table */
339	mov	x7, #NORMAL_UNCACHED /* Uncached as it's only needed early on */
340	mov	x9, x27
341	mov	x8, x9		/* VA start (== PA start) */
342	bl	build_section_pagetable
343
344	/* Restore the Link register */
345	mov	x30, x5
346	ret
347
348/*
349 * Builds a 1 GiB page table entry
350 *  x6 = L1 table
351 *  x7 = Type (0 = Device, 1 = Normal)
352 *  x8 = VA start
353 *  x9 = PA start (trashed)
354 *  x11, x12 and x13 are trashed
355 */
356build_section_pagetable:
357	/*
358	 * Build the L1 table entry.
359	 */
360	/* Find the table index */
361	lsr	x11, x8, #L1_SHIFT
362	and	x11, x11, #Ln_ADDR_MASK
363
364	/* Build the L1 block entry */
365	lsl	x12, x7, #2
366	orr	x12, x12, #L1_BLOCK
367	orr	x12, x12, #(ATTR_AF)
368
369	/* Only use the output address bits */
370	lsr	x9, x9, #L1_SHIFT
371	orr	x12, x12, x9, lsl #L1_SHIFT
372
373	/* Store the entry */
374	str	x12, [x6, x11, lsl #3]
375
376	ret
377
378/*
379 * Builds an L1 -> L2 table descriptor
380 *
381 * This is a link for a 1GiB block of memory with up to 2MiB regions mapped
382 * within it by build_block_pagetable.
383 *
384 *  x6  = L1 table
385 *  x8  = Virtual Address
386 *  x9  = L2 PA (trashed)
387 *  x11, x12 and x13 are trashed
388 */
389link_l1_pagetable:
390	/*
391	 * Link an L1 -> L2 table entry.
392	 */
393	/* Find the table index */
394	lsr	x11, x8, #L1_SHIFT
395	and	x11, x11, #Ln_ADDR_MASK
396
397	/* Build the L1 block entry */
398	mov	x12, #L1_TABLE
399
400	/* Only use the output address bits */
401	lsr	x9, x9, #12
402	orr	x12, x12, x9, lsl #12
403
404	/* Store the entry */
405	str	x12, [x6, x11, lsl #3]
406
407	ret
408
409/*
410 * Builds count 2 MiB page table entry
411 *  x6  = L2 table
412 *  x7  = Type (0 = Device, 1 = Normal)
413 *  x8  = VA start
414 *  x9  = PA start (trashed)
415 *  x10 = Entry count (TODO)
416 *  x11, x12 and x13 are trashed
417 */
418build_block_pagetable:
419	/*
420	 * Build the L2 table entry.
421	 */
422	/* Find the table index */
423	lsr	x11, x8, #L2_SHIFT
424	and	x11, x11, #Ln_ADDR_MASK
425
426	/* Build the L2 block entry */
427	lsl	x12, x7, #2
428	orr	x12, x12, #L2_BLOCK
429	orr	x12, x12, #(ATTR_AF)
430
431	/* Only use the output address bits */
432	lsr	x9, x9, #L2_SHIFT
433
434	/* Set the physical address for this virtual address */
4351:	orr	x12, x12, x9, lsl #L2_SHIFT
436
437	/* Store the entry */
438	str	x12, [x6, x11, lsl #3]
439
440	/* Clear the address bits */
441	and	x12, x12, #ATTR_MASK_L
442
443	sub	x10, x10, #1
444	add	x11, x11, #1
445	add	x9, x9, #1
446	cbnz	x10, 1b
447
4482:	ret
449
450start_mmu:
451	dsb	sy
452
453	/* Load the exception vectors */
454	ldr	x2, =exception_vectors
455	msr	vbar_el1, x2
456
457	/* Load ttbr0 and ttbr1 */
458	msr	ttbr0_el1, x27
459	msr	ttbr1_el1, x26
460	isb
461
462	/* Clear the Monitor Debug System control register */
463	msr	mdscr_el1, xzr
464
465	/* Invalidate the TLB */
466	tlbi	vmalle1is
467
468	ldr	x2, mair
469	msr	mair_el1, x2
470
471	/* Setup TCR according to PARange bits from ID_AA64MMFR0_EL1 */
472	ldr	x2, tcr
473	mrs	x3, id_aa64mmfr0_el1
474	bfi	x2, x3, #32, #3
475	msr	tcr_el1, x2
476
477	/* Setup SCTLR */
478	ldr	x2, sctlr_set
479	ldr	x3, sctlr_clear
480	mrs	x1, sctlr_el1
481	bic	x1, x1, x3	/* Clear the required bits */
482	orr	x1, x1, x2	/* Set the required bits */
483	msr	sctlr_el1, x1
484	isb
485
486	ret
487
488	.align 3
489mair:
490		/* Device            Normal, no cache     Normal, write-back */
491	.quad	MAIR_ATTR(0x00, 0) | MAIR_ATTR(0x44, 1) | MAIR_ATTR(0xff, 2)
492tcr:
493	.quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K)
494sctlr_set:
495	/* Bits to set */
496	.quad (SCTLR_UCI | SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \
497	    SCTLR_I | SCTLR_SED | SCTLR_C | SCTLR_M)
498sctlr_clear:
499	/* Bits to clear */
500	.quad (SCTLR_EE | SCTLR_EOE | SCTLR_WXN | SCTLR_UMA | SCTLR_ITD | \
501	    SCTLR_THEE | SCTLR_CP15BEN | SCTLR_SA0 | SCTLR_SA | SCTLR_A)
502
503	.globl abort
504abort:
505	b abort
506
507	//.section .init_pagetable
508	.align 12 /* 4KiB aligned */
509	/*
510	 * 3 initial tables (in the following order):
511	 *           L2 for kernel (High addresses)
512	 *           L1 for kernel
513	 *           L1 for user   (Low addresses)
514	 */
515pagetable:
516	.space	PAGE_SIZE
517pagetable_l1_ttbr1:
518	.space	PAGE_SIZE
519pagetable_l1_ttbr0:
520	.space	PAGE_SIZE
521pagetable_end:
522
523el2_pagetable:
524	.space	PAGE_SIZE
525
526	.globl init_pt_va
527init_pt_va:
528	.quad pagetable		/* XXX: Keep page tables VA */
529
530	.align	4
531initstack:
532	.space	(PAGE_SIZE * KSTACK_PAGES)
533initstack_end:
534
535
536ENTRY(sigcode)
537	mov	x0, sp
538	add	x0, x0, #SF_UC
539
5401:
541	mov	x8, #SYS_sigreturn
542	svc	0
543
544	/* sigreturn failed, exit */
545	mov	x8, #SYS_exit
546	svc	0
547
548	b	1b
549END(sigcode)
550	/* This may be copied to the stack, keep it 16-byte aligned */
551	.align	3
552esigcode:
553
554	.data
555	.align	3
556	.global	szsigcode
557szsigcode:
558	.quad	esigcode - sigcode
559