xref: /freebsd/sys/arm64/arm64/locore.S (revision c7046f76c2c027b00c0e6ba57cfd28f1a78f5e23)
1/*-
2 * Copyright (c) 2012-2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29#include "assym.inc"
30#include "opt_kstack_pages.h"
31#include <sys/syscall.h>
32#include <machine/asm.h>
33#include <machine/armreg.h>
34#include <machine/hypervisor.h>
35#include <machine/param.h>
36#include <machine/pte.h>
37#include <machine/vm.h>
38#include <machine/vmparam.h>
39
40#define	VIRT_BITS	48
41
42#if PAGE_SIZE == PAGE_SIZE_16K
43/*
44 * The number of level 3 tables to create. 32 will allow for 1G of address
45 * space, the same as a single level 2 page with 4k pages.
46 */
47#define	L3_PAGE_COUNT	32
48#endif
49
50	.globl	kernbase
51	.set	kernbase, KERNBASE
52
53/*
54 * We assume:
55 *  MMU      on with an identity map, or off
56 *  D-Cache: off
57 *  I-Cache: on or off
58 *  We are loaded at a 2MiB aligned address
59 */
60
61ENTRY(_start)
62	/* Drop to EL1 */
63	bl	drop_to_el1
64
65	/*
66	 * Disable the MMU. We may have entered the kernel with it on and
67	 * will need to update the tables later. If this has been set up
68	 * with anything other than a VA == PA map then this will fail,
69	 * but in this case the code to find where we are running from
70	 * would have also failed.
71	 */
72	dsb	sy
73	mrs	x2, sctlr_el1
74	bic	x2, x2, SCTLR_M
75	msr	sctlr_el1, x2
76	isb
77
78	/* Set the context id */
79	msr	contextidr_el1, xzr
80
81	/* Get the virt -> phys offset */
82	bl	get_virt_delta
83
84	/*
85	 * At this point:
86	 * x29 = PA - VA
87	 * x28 = Our physical load address
88	 */
89
90	/* Create the page tables */
91	bl	create_pagetables
92
93	/*
94	 * At this point:
95	 * x27 = TTBR0 table
96	 * x26 = Kernel L1 table
97	 * x24 = TTBR1 table
98	 */
99
100	/* Enable the mmu */
101	bl	start_mmu
102
103	/* Load the new ttbr0 pagetable */
104	adrp	x27, pagetable_l0_ttbr0
105	add	x27, x27, :lo12:pagetable_l0_ttbr0
106
107	/* Jump to the virtual address space */
108	ldr	x15, .Lvirtdone
109	br	x15
110
111virtdone:
112	/* Set up the stack */
113	adrp	x25, initstack_end
114	add	x25, x25, :lo12:initstack_end
115	mov	sp, x25
116	sub	sp, sp, #PCB_SIZE
117
118	/* Zero the BSS */
119	ldr	x15, .Lbss
120	ldr	x14, .Lend
1211:
122	str	xzr, [x15], #8
123	cmp	x15, x14
124	b.lo	1b
125
126#if defined(PERTHREAD_SSP)
127	/* Set sp_el0 to the boot canary for early per-thread SSP to work */
128	adrp	x15, boot_canary
129	add	x15, x15, :lo12:boot_canary
130	msr	sp_el0, x15
131#endif
132
133	/* Backup the module pointer */
134	mov	x1, x0
135
136	/* Make the page table base a virtual address */
137	sub	x26, x26, x29
138	sub	x24, x24, x29
139
140	sub	sp, sp, #BOOTPARAMS_SIZE
141	mov	x0, sp
142
143	/* Degate the delda so it is VA -> PA */
144	neg	x29, x29
145
146	str	x1,  [x0, #BP_MODULEP]
147	str	x26, [x0, #BP_KERN_L1PT]
148	str	x29, [x0, #BP_KERN_DELTA]
149	adrp	x25, initstack
150	add	x25, x25, :lo12:initstack
151	str	x25, [x0, #BP_KERN_STACK]
152	str	x24, [x0, #BP_KERN_L0PT]
153	str	x27, [x0, #BP_KERN_TTBR0]
154	str	x23, [x0, #BP_BOOT_EL]
155
156	/* trace back starts here */
157	mov	fp, #0
158	/* Branch to C code */
159	bl	initarm
160	/* We are done with the boot params */
161	add	sp, sp, #BOOTPARAMS_SIZE
162
163	/*
164	 * Enable pointer authentication in the kernel. We set the keys for
165	 * thread0 in initarm so have to wait until it returns to enable it.
166	 * If we were to enable it in initarm then any authentication when
167	 * returning would fail as it was called with pointer authentication
168	 * disabled.
169	 */
170	bl	ptrauth_start
171
172	bl	mi_startup
173
174	/* We should not get here */
175	brk	0
176
177	.align 3
178.Lvirtdone:
179	.quad	virtdone
180.Lbss:
181	.quad	__bss_start
182.Lend:
183	.quad	__bss_end
184END(_start)
185
186#ifdef SMP
187/*
188 * mpentry(unsigned long)
189 *
190 * Called by a core when it is being brought online.
191 * The data in x0 is passed straight to init_secondary.
192 */
193ENTRY(mpentry)
194	/* Disable interrupts */
195	msr	daifset, #DAIF_INTR
196
197	/* Drop to EL1 */
198	bl	drop_to_el1
199
200	/* Set the context id */
201	msr	contextidr_el1, xzr
202
203	/* Load the kernel page table */
204	adrp	x24, pagetable_l0_ttbr1
205	add	x24, x24, :lo12:pagetable_l0_ttbr1
206	/* Load the identity page table */
207	adrp	x27, pagetable_l0_ttbr0_boostrap
208	add	x27, x27, :lo12:pagetable_l0_ttbr0_boostrap
209
210	/* Enable the mmu */
211	bl	start_mmu
212
213	/* Load the new ttbr0 pagetable */
214	adrp	x27, pagetable_l0_ttbr0
215	add	x27, x27, :lo12:pagetable_l0_ttbr0
216
217	/* Jump to the virtual address space */
218	ldr	x15, =mp_virtdone
219	br	x15
220
221mp_virtdone:
222	/* Start using the AP boot stack */
223	ldr	x4, =bootstack
224	ldr	x4, [x4]
225	mov	sp, x4
226
227#if defined(PERTHREAD_SSP)
228	/* Set sp_el0 to the boot canary for early per-thread SSP to work */
229	adrp	x15, boot_canary
230	add	x15, x15, :lo12:boot_canary
231	msr	sp_el0, x15
232#endif
233
234	/* Load the kernel ttbr0 pagetable */
235	msr	ttbr0_el1, x27
236	isb
237
238	/* Invalidate the TLB */
239	tlbi	vmalle1
240	dsb	sy
241	isb
242
243	b	init_secondary
244END(mpentry)
245#endif
246
247/*
248 * If we are started in EL2, configure the required hypervisor
249 * registers and drop to EL1.
250 */
251LENTRY(drop_to_el1)
252	mrs	x23, CurrentEL
253	lsr	x23, x23, #2
254	cmp	x23, #0x2
255	b.eq	1f
256	ret
2571:
258	/*
259	 * Disable the MMU. If the HCR_EL2.E2H field is set we will clear it
260	 * which may break address translation.
261	 */
262	dsb	sy
263	mrs	x2, sctlr_el2
264	bic	x2, x2, SCTLR_M
265	msr	sctlr_el2, x2
266	isb
267
268	/* Configure the Hypervisor */
269	ldr	x2, =(HCR_RW | HCR_APK | HCR_API)
270	msr	hcr_el2, x2
271
272	/* Load the Virtualization Process ID Register */
273	mrs	x2, midr_el1
274	msr	vpidr_el2, x2
275
276	/* Load the Virtualization Multiprocess ID Register */
277	mrs	x2, mpidr_el1
278	msr	vmpidr_el2, x2
279
280	/* Set the bits that need to be 1 in sctlr_el1 */
281	ldr	x2, .Lsctlr_res1
282	msr	sctlr_el1, x2
283
284	/* Don't trap to EL2 for exceptions */
285	mov	x2, #CPTR_RES1
286	msr	cptr_el2, x2
287
288	/* Don't trap to EL2 for CP15 traps */
289	msr	hstr_el2, xzr
290
291	/* Enable access to the physical timers at EL1 */
292	mrs	x2, cnthctl_el2
293	orr	x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN)
294	msr	cnthctl_el2, x2
295
296	/* Set the counter offset to a known value */
297	msr	cntvoff_el2, xzr
298
299	/* Hypervisor trap functions */
300	adrp	x2, hyp_vectors
301	add	x2, x2, :lo12:hyp_vectors
302	msr	vbar_el2, x2
303
304	mov	x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h)
305	msr	spsr_el2, x2
306
307	/* Configure GICv3 CPU interface */
308	mrs	x2, id_aa64pfr0_el1
309	/* Extract GIC bits from the register */
310	ubfx	x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS
311	/* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */
312	cmp	x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT)
313	b.ne	2f
314
315	mrs	x2, icc_sre_el2
316	orr	x2, x2, #ICC_SRE_EL2_EN	/* Enable access from insecure EL1 */
317	orr	x2, x2, #ICC_SRE_EL2_SRE	/* Enable system registers */
318	msr	icc_sre_el2, x2
3192:
320
321	/* Set the address to return to our return address */
322	msr	elr_el2, x30
323	isb
324
325	eret
326
327	.align 3
328.Lsctlr_res1:
329	.quad SCTLR_RES1
330LEND(drop_to_el1)
331
332#define	VECT_EMPTY	\
333	.align 7;	\
334	1:	b	1b
335
336	.align 11
337hyp_vectors:
338	VECT_EMPTY	/* Synchronous EL2t */
339	VECT_EMPTY	/* IRQ EL2t */
340	VECT_EMPTY	/* FIQ EL2t */
341	VECT_EMPTY	/* Error EL2t */
342
343	VECT_EMPTY	/* Synchronous EL2h */
344	VECT_EMPTY	/* IRQ EL2h */
345	VECT_EMPTY	/* FIQ EL2h */
346	VECT_EMPTY	/* Error EL2h */
347
348	VECT_EMPTY	/* Synchronous 64-bit EL1 */
349	VECT_EMPTY	/* IRQ 64-bit EL1 */
350	VECT_EMPTY	/* FIQ 64-bit EL1 */
351	VECT_EMPTY	/* Error 64-bit EL1 */
352
353	VECT_EMPTY	/* Synchronous 32-bit EL1 */
354	VECT_EMPTY	/* IRQ 32-bit EL1 */
355	VECT_EMPTY	/* FIQ 32-bit EL1 */
356	VECT_EMPTY	/* Error 32-bit EL1 */
357
358/*
359 * Get the delta between the physical address we were loaded to and the
360 * virtual address we expect to run from. This is used when building the
361 * initial page table.
362 */
363LENTRY(get_virt_delta)
364	/* Load the physical address of virt_map */
365	adrp	x29, virt_map
366	add	x29, x29, :lo12:virt_map
367	/* Load the virtual address of virt_map stored in virt_map */
368	ldr	x28, [x29]
369	/* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */
370	sub	x29, x29, x28
371	/* Find the load address for the kernel */
372	mov	x28, #(KERNBASE)
373	add	x28, x28, x29
374	ret
375
376	.align 3
377virt_map:
378	.quad	virt_map
379LEND(get_virt_delta)
380
381/*
382 * This builds the page tables containing the identity map, and the kernel
383 * virtual map.
384 *
385 * It relys on:
386 *  We were loaded to an address that is on a 2MiB boundary
387 *  All the memory must not cross a 1GiB boundaty
388 *  x28 contains the physical address we were loaded from
389 *
390 * TODO: This is out of date.
391 *  There are at least 5 pages before that address for the page tables
392 *   The pages used are:
393 *    - The Kernel L2 table
394 *    - The Kernel L1 table
395 *    - The Kernel L0 table             (TTBR1)
396 *    - The identity (PA = VA) L1 table
397 *    - The identity (PA = VA) L0 table (TTBR0)
398 */
399LENTRY(create_pagetables)
400	/* Save the Link register */
401	mov	x5, x30
402
403	/* Clean the page table */
404	adrp	x6, pagetable
405	add	x6, x6, :lo12:pagetable
406	mov	x26, x6
407	adrp	x27, pagetable_end
408	add	x27, x27, :lo12:pagetable_end
4091:
410	stp	xzr, xzr, [x6], #16
411	stp	xzr, xzr, [x6], #16
412	stp	xzr, xzr, [x6], #16
413	stp	xzr, xzr, [x6], #16
414	cmp	x6, x27
415	b.lo	1b
416
417	/*
418	 * Build the TTBR1 maps.
419	 */
420
421	/* Find the size of the kernel */
422	mov	x6, #(KERNBASE)
423
424#if defined(LINUX_BOOT_ABI)
425	/* X19 is used as 'map FDT data' flag */
426	mov	x19, xzr
427
428	/* No modules or FDT pointer ? */
429	cbz	x0, booti_no_fdt
430
431	/*
432	 * Test if x0 points to modules descriptor(virtual address) or
433	 * to FDT (physical address)
434	 */
435	cmp	x0, x6		/* x6 is #(KERNBASE) */
436	b.lo	booti_fdt
437#endif
438
439	/* Booted with modules pointer */
440	/* Find modulep - begin */
441	sub	x8, x0, x6
442	/*
443	 * Add space for the module data. When PAGE_SIZE is 4k this will
444	 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is
445	 * larger it will be at least as large as we use smaller level 3
446	 * pages.
447	 */
448	ldr	x7, =((6 * 1024 * 1024) - 1)
449	add	x8, x8, x7
450	b	common
451
452#if defined(LINUX_BOOT_ABI)
453booti_fdt:
454	/* Booted by U-Boot booti with FDT data */
455	/* Set 'map FDT data' flag */
456	mov	x19, #1
457
458booti_no_fdt:
459	/* Booted by U-Boot booti without FTD data */
460	/* Find the end - begin */
461	ldr     x7, .Lend
462	sub     x8, x7, x6
463
464	/*
465	 * Add one 2MiB page for copy of FDT data (maximum FDT size),
466	 * one for metadata and round up
467	 */
468	ldr	x7, =(3 * L2_SIZE - 1)
469	add	x8, x8, x7
470#endif
471
472common:
473#if PAGE_SIZE != PAGE_SIZE_4K
474	/*
475	 * Create L3 pages. The kernel will be loaded at a 2M aligned
476	 * address, however L2 blocks are too large when the page size is
477	 * not 4k to map the kernel with such an aligned address. However,
478	 * when the page size is larger than 4k, L2 blocks are too large to
479	 * map the kernel with such an alignment.
480	 */
481
482	/* Get the number of l3 pages to allocate, rounded down */
483	lsr	x10, x8, #(L3_SHIFT)
484
485	/* Create the kernel space L2 table */
486	mov	x6, x26
487	mov	x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
488	mov	x8, #(KERNBASE)
489	mov	x9, x28
490	bl	build_l3_page_pagetable
491
492	/* Move to the l2 table */
493	ldr	x9, =(PAGE_SIZE * L3_PAGE_COUNT)
494	add	x26, x26, x9
495
496	/* Link the l2 -> l3 table */
497	mov	x9, x6
498	mov	x6, x26
499	bl	link_l2_pagetable
500#else
501	/* Get the number of l2 pages to allocate, rounded down */
502	lsr	x10, x8, #(L2_SHIFT)
503
504	/* Create the kernel space L2 table */
505	mov	x6, x26
506	mov	x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
507	mov	x8, #(KERNBASE)
508	mov	x9, x28
509	bl	build_l2_block_pagetable
510#endif
511
512	/* Move to the l1 table */
513	add	x26, x26, #PAGE_SIZE
514
515	/* Link the l1 -> l2 table */
516	mov	x9, x6
517	mov	x6, x26
518	bl	link_l1_pagetable
519
520	/* Move to the l0 table */
521	add	x24, x26, #PAGE_SIZE
522
523	/* Link the l0 -> l1 table */
524	mov	x9, x6
525	mov	x6, x24
526	mov	x10, #1
527	bl	link_l0_pagetable
528
529	/*
530	 * Build the TTBR0 maps.  As TTBR0 maps, they must specify ATTR_S1_nG.
531	 * They are only needed early on, so the VA = PA map is uncached.
532	 */
533	add	x27, x24, #PAGE_SIZE
534
535	mov	x6, x27		/* The initial page table */
536
537	/* Create the VA = PA map */
538	mov	x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
539	adrp	x16, _start
540	and	x16, x16, #(~L2_OFFSET)
541	mov	x9, x16		/* PA start */
542	mov	x8, x16		/* VA start (== PA start) */
543	mov	x10, #1
544	bl	build_l2_block_pagetable
545
546#if defined(SOCDEV_PA)
547	/* Create a table for the UART */
548	mov	x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE))
549	ldr	x9, =(L2_SIZE)
550	add	x16, x16, x9	/* VA start */
551	mov	x8, x16
552
553	/* Store the socdev virtual address */
554	add	x17, x8, #(SOCDEV_PA & L2_OFFSET)
555	adrp	x9, socdev_va
556	str	x17, [x9, :lo12:socdev_va]
557
558	mov	x9, #(SOCDEV_PA & ~L2_OFFSET)	/* PA start */
559	mov	x10, #1
560	bl	build_l2_block_pagetable
561#endif
562
563#if defined(LINUX_BOOT_ABI)
564	/* Map FDT data ? */
565	cbz	x19, 1f
566
567	/* Create the mapping for FDT data (2 MiB max) */
568	mov	x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
569	ldr	x9, =(L2_SIZE)
570	add	x16, x16, x9	/* VA start */
571	mov	x8, x16
572	mov	x9, x0			/* PA start */
573	/* Update the module pointer to point at the allocated memory */
574	and	x0, x0, #(L2_OFFSET)	/* Keep the lower bits */
575	add	x0, x0, x8		/* Add the aligned virtual address */
576
577	mov	x10, #1
578	bl	build_l2_block_pagetable
579
5801:
581#endif
582
583	/* Move to the l1 table */
584	add	x27, x27, #PAGE_SIZE
585
586	/* Link the l1 -> l2 table */
587	mov	x9, x6
588	mov	x6, x27
589	bl	link_l1_pagetable
590
591	/* Move to the l0 table */
592	add	x27, x27, #PAGE_SIZE
593
594	/* Link the l0 -> l1 table */
595	mov	x9, x6
596	mov	x6, x27
597	mov	x10, #1
598	bl	link_l0_pagetable
599
600	/* Restore the Link register */
601	mov	x30, x5
602	ret
603LEND(create_pagetables)
604
605/*
606 * Builds an L0 -> L1 table descriptor
607 *
608 *  x6  = L0 table
609 *  x8  = Virtual Address
610 *  x9  = L1 PA (trashed)
611 *  x10 = Entry count (trashed)
612 *  x11, x12 and x13 are trashed
613 */
614LENTRY(link_l0_pagetable)
615	/*
616	 * Link an L0 -> L1 table entry.
617	 */
618	/* Find the table index */
619	lsr	x11, x8, #L0_SHIFT
620	and	x11, x11, #L0_ADDR_MASK
621
622	/* Build the L0 block entry */
623	mov	x12, #L0_TABLE
624	orr	x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0)
625
626	/* Only use the output address bits */
627	lsr	x9, x9, #PAGE_SHIFT
6281:	orr	x13, x12, x9, lsl #PAGE_SHIFT
629
630	/* Store the entry */
631	str	x13, [x6, x11, lsl #3]
632
633	sub	x10, x10, #1
634	add	x11, x11, #1
635	add	x9, x9, #1
636	cbnz	x10, 1b
637
638	ret
639LEND(link_l0_pagetable)
640
641/*
642 * Builds an L1 -> L2 table descriptor
643 *
644 *  x6  = L1 table
645 *  x8  = Virtual Address
646 *  x9  = L2 PA (trashed)
647 *  x11, x12 and x13 are trashed
648 */
649LENTRY(link_l1_pagetable)
650	/*
651	 * Link an L1 -> L2 table entry.
652	 */
653	/* Find the table index */
654	lsr	x11, x8, #L1_SHIFT
655	and	x11, x11, #Ln_ADDR_MASK
656
657	/* Build the L1 block entry */
658	mov	x12, #L1_TABLE
659
660	/* Only use the output address bits */
661	lsr	x9, x9, #PAGE_SHIFT
662	orr	x13, x12, x9, lsl #PAGE_SHIFT
663
664	/* Store the entry */
665	str	x13, [x6, x11, lsl #3]
666
667	ret
668LEND(link_l1_pagetable)
669
670/*
671 * Builds count 2 MiB page table entry
672 *  x6  = L2 table
673 *  x7  = Block attributes
674 *  x8  = VA start
675 *  x9  = PA start (trashed)
676 *  x10 = Entry count (trashed)
677 *  x11, x12 and x13 are trashed
678 */
679LENTRY(build_l2_block_pagetable)
680	/*
681	 * Build the L2 table entry.
682	 */
683	/* Find the table index */
684	lsr	x11, x8, #L2_SHIFT
685	and	x11, x11, #Ln_ADDR_MASK
686
687	/* Build the L2 block entry */
688	orr	x12, x7, #L2_BLOCK
689	orr	x12, x12, #(ATTR_DEFAULT)
690	orr	x12, x12, #(ATTR_S1_UXN)
691
692	/* Only use the output address bits */
693	lsr	x9, x9, #L2_SHIFT
694
695	/* Set the physical address for this virtual address */
6961:	orr	x13, x12, x9, lsl #L2_SHIFT
697
698	/* Store the entry */
699	str	x13, [x6, x11, lsl #3]
700
701	sub	x10, x10, #1
702	add	x11, x11, #1
703	add	x9, x9, #1
704	cbnz	x10, 1b
705
706	ret
707LEND(build_l2_block_pagetable)
708
709#if PAGE_SIZE != PAGE_SIZE_4K
710/*
711 * Builds an L2 -> L3 table descriptor
712 *
713 *  x6  = L2 table
714 *  x8  = Virtual Address
715 *  x9  = L3 PA (trashed)
716 *  x11, x12 and x13 are trashed
717 */
718LENTRY(link_l2_pagetable)
719	/*
720	 * Link an L2 -> L3 table entry.
721	 */
722	/* Find the table index */
723	lsr	x11, x8, #L2_SHIFT
724	and	x11, x11, #Ln_ADDR_MASK
725
726	/* Build the L1 block entry */
727	mov	x12, #L2_TABLE
728
729	/* Only use the output address bits */
730	lsr	x9, x9, #PAGE_SHIFT
731	orr	x13, x12, x9, lsl #PAGE_SHIFT
732
733	/* Store the entry */
734	str	x13, [x6, x11, lsl #3]
735
736	ret
737LEND(link_l2_pagetable)
738
739/*
740 * Builds count level 3 page table entries
741 *  x6  = L3 table
742 *  x7  = Block attributes
743 *  x8  = VA start
744 *  x9  = PA start (trashed)
745 *  x10 = Entry count (trashed)
746 *  x11, x12 and x13 are trashed
747 */
748LENTRY(build_l3_page_pagetable)
749	/*
750	 * Build the L3 table entry.
751	 */
752	/* Find the table index */
753	lsr	x11, x8, #L3_SHIFT
754	and	x11, x11, #Ln_ADDR_MASK
755
756	/* Build the L3 page entry */
757	orr	x12, x7, #L3_PAGE
758	orr	x12, x12, #(ATTR_DEFAULT)
759	orr	x12, x12, #(ATTR_S1_UXN)
760
761	/* Only use the output address bits */
762	lsr	x9, x9, #L3_SHIFT
763
764	/* Set the physical address for this virtual address */
7651:	orr	x13, x12, x9, lsl #L3_SHIFT
766
767	/* Store the entry */
768	str	x13, [x6, x11, lsl #3]
769
770	sub	x10, x10, #1
771	add	x11, x11, #1
772	add	x9, x9, #1
773	cbnz	x10, 1b
774
775	ret
776LEND(build_l3_page_pagetable)
777#endif
778
779LENTRY(start_mmu)
780	dsb	sy
781
782	/* Load the exception vectors */
783	ldr	x2, =exception_vectors
784	msr	vbar_el1, x2
785
786	/* Load ttbr0 and ttbr1 */
787	msr	ttbr0_el1, x27
788	msr	ttbr1_el1, x24
789	isb
790
791	/* Clear the Monitor Debug System control register */
792	msr	mdscr_el1, xzr
793
794	/* Invalidate the TLB */
795	tlbi	vmalle1is
796	dsb	ish
797	isb
798
799	ldr	x2, mair
800	msr	mair_el1, x2
801
802	/*
803	 * Setup TCR according to the PARange and ASIDBits fields
804	 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the
805	 * ID_AA64MMFR1_EL1.  More precisely, set TCR_EL1.AS
806	 * to 1 only if the ASIDBits field equals 0b0010.
807	 */
808	ldr	x2, tcr
809	mrs	x3, id_aa64mmfr0_el1
810
811	/* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */
812	bfi	x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH)
813	and	x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK)
814
815	/* Check if the HW supports 16 bit ASIDS */
816	cmp	x3, #(ID_AA64MMFR0_ASIDBits_16)
817	/* If so x3 == 1, else x3 == 0 */
818	cset	x3, eq
819	/* Set TCR.AS with x3 */
820	bfi	x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH)
821
822	/*
823	 * Check if the HW supports access flag and dirty state updates,
824	 * and set TCR_EL1.HA and TCR_EL1.HD accordingly.
825	 */
826	mrs	x3, id_aa64mmfr1_el1
827	and	x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK)
828	cmp	x3, #1
829	b.ne	1f
830	orr 	x2, x2, #(TCR_HA)
831	b	2f
8321:
833	cmp	x3, #2
834	b.ne	2f
835	orr 	x2, x2, #(TCR_HA | TCR_HD)
8362:
837	msr	tcr_el1, x2
838
839	/*
840	 * Setup SCTLR.
841	 */
842	ldr	x2, sctlr_set
843	ldr	x3, sctlr_clear
844	mrs	x1, sctlr_el1
845	bic	x1, x1, x3	/* Clear the required bits */
846	orr	x1, x1, x2	/* Set the required bits */
847	msr	sctlr_el1, x1
848	isb
849
850	ret
851
852	.align 3
853mair:
854	.quad	MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \
855		MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE)   |	\
856		MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK)    |	\
857		MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) |	\
858		MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE)
859tcr:
860#if PAGE_SIZE == PAGE_SIZE_4K
861#define	TCR_TG	(TCR_TG1_4K | TCR_TG0_4K)
862#elif PAGE_SIZE == PAGE_SIZE_16K
863#define	TCR_TG	(TCR_TG1_16K | TCR_TG0_16K)
864#else
865#error Unsupported page size
866#endif
867
868	.quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \
869	    TCR_CACHE_ATTRS | TCR_SMP_ATTRS)
870sctlr_set:
871	/* Bits to set */
872	.quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \
873	    SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \
874	    SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \
875	    SCTLR_M | SCTLR_CP15BEN)
876sctlr_clear:
877	/* Bits to clear */
878	.quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \
879	    SCTLR_ITD | SCTLR_A)
880LEND(start_mmu)
881
882ENTRY(abort)
883	b abort
884END(abort)
885
886	.section .init_pagetable, "aw", %nobits
887	.align PAGE_SHIFT
888	/*
889	 * 6 initial tables (in the following order):
890	 *           L2 for kernel (High addresses)
891	 *           L1 for kernel
892	 *           L0 for kernel
893	 *           L1 bootstrap for user   (Low addresses)
894	 *           L0 bootstrap for user
895	 *           L0 for user
896	 */
897	.globl pagetable_l0_ttbr1
898pagetable:
899#if PAGE_SIZE != PAGE_SIZE_4K
900	.space	(PAGE_SIZE * L3_PAGE_COUNT)
901pagetable_l2_ttbr1:
902#endif
903	.space	PAGE_SIZE
904pagetable_l1_ttbr1:
905	.space	PAGE_SIZE
906pagetable_l0_ttbr1:
907	.space	PAGE_SIZE
908pagetable_l2_ttbr0_bootstrap:
909	.space	PAGE_SIZE
910pagetable_l1_ttbr0_bootstrap:
911	.space	PAGE_SIZE
912pagetable_l0_ttbr0_boostrap:
913	.space	PAGE_SIZE
914pagetable_l0_ttbr0:
915	.space	PAGE_SIZE
916
917pagetable_end:
918
919el2_pagetable:
920	.space	PAGE_SIZE
921
922	.align	4
923initstack:
924	.space	(PAGE_SIZE * KSTACK_PAGES)
925initstack_end:
926
927
928.text
929EENTRY(aarch32_sigcode)
930	.word 0xe1a0000d	// mov r0, sp
931	.word 0xe2800040	// add r0, r0, #SIGF_UC
932	.word 0xe59f700c	// ldr r7, [pc, #12]
933	.word 0xef000000	// swi #0
934	.word 0xe59f7008	// ldr r7, [pc, #8]
935	.word 0xef000000	// swi #0
936	.word 0xeafffffa	// b . - 16
937EEND(aarch32_sigcode)
938	.word SYS_sigreturn
939	.word SYS_exit
940	.align	3
941aarch32_esigcode:
942	.data
943	.global sz_aarch32_sigcode
944sz_aarch32_sigcode:
945	.quad aarch32_esigcode - aarch32_sigcode
946