xref: /freebsd/sys/arm64/arm64/locore.S (revision d485c77f203fb0f4cdc08dea5ff81631b51d8809)
1/*-
2 * Copyright (c) 2012-2014 Andrew Turner
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 *
26 * $FreeBSD$
27 */
28
29#include "assym.inc"
30#include "opt_kstack_pages.h"
31#include <sys/syscall.h>
32#include <machine/asm.h>
33#include <machine/armreg.h>
34#include <machine/hypervisor.h>
35#include <machine/param.h>
36#include <machine/pte.h>
37#include <machine/vm.h>
38#include <machine/vmparam.h>
39
40#define	VIRT_BITS	48
41#define	DMAP_TABLES	((DMAP_MAX_ADDRESS - DMAP_MIN_ADDRESS) >> L0_SHIFT)
42
43	.globl	kernbase
44	.set	kernbase, KERNBASE
45
46/*
47 * We assume:
48 *  MMU      on with an identity map, or off
49 *  D-Cache: off
50 *  I-Cache: on or off
51 *  We are loaded at a 2MiB aligned address
52 */
53
54ENTRY(_start)
55	/* Drop to EL1 */
56	bl	drop_to_el1
57
58	/*
59	 * Disable the MMU. We may have entered the kernel with it on and
60	 * will need to update the tables later. If this has been set up
61	 * with anything other than a VA == PA map then this will fail,
62	 * but in this case the code to find where we are running from
63	 * would have also failed.
64	 */
65	dsb	sy
66	mrs	x2, sctlr_el1
67	bic	x2, x2, SCTLR_M
68	msr	sctlr_el1, x2
69	isb
70
71	/* Set the context id */
72	msr	contextidr_el1, xzr
73
74	/* Get the virt -> phys offset */
75	bl	get_virt_delta
76
77	/*
78	 * At this point:
79	 * x29 = PA - VA
80	 * x28 = Our physical load address
81	 */
82
83	/* Create the page tables */
84	bl	create_pagetables
85
86	/*
87	 * At this point:
88	 * x27 = TTBR0 table
89	 * x26 = Kernel L1 table
90	 * x24 = TTBR1 table
91	 */
92
93	/* Enable the mmu */
94	bl	start_mmu
95
96	/* Load the new ttbr0 pagetable */
97	adrp	x27, pagetable_l0_ttbr0
98	add	x27, x27, :lo12:pagetable_l0_ttbr0
99
100	/* Jump to the virtual address space */
101	ldr	x15, .Lvirtdone
102	br	x15
103
104virtdone:
105	/* Set up the stack */
106	adrp	x25, initstack_end
107	add	x25, x25, :lo12:initstack_end
108	mov	sp, x25
109	sub	sp, sp, #PCB_SIZE
110
111	/* Zero the BSS */
112	ldr	x15, .Lbss
113	ldr	x14, .Lend
1141:
115	str	xzr, [x15], #8
116	cmp	x15, x14
117	b.lo	1b
118
119	/* Backup the module pointer */
120	mov	x1, x0
121
122	/* Make the page table base a virtual address */
123	sub	x26, x26, x29
124	sub	x24, x24, x29
125
126	sub	sp, sp, #BOOTPARAMS_SIZE
127	mov	x0, sp
128
129	/* Degate the delda so it is VA -> PA */
130	neg	x29, x29
131
132	str	x1,  [x0, #BP_MODULEP]
133	str	x26, [x0, #BP_KERN_L1PT]
134	str	x29, [x0, #BP_KERN_DELTA]
135	adrp	x25, initstack
136	add	x25, x25, :lo12:initstack
137	str	x25, [x0, #BP_KERN_STACK]
138	str	x24, [x0, #BP_KERN_L0PT]
139	str	x27, [x0, #BP_KERN_TTBR0]
140	str	x23, [x0, #BP_BOOT_EL]
141
142	/* trace back starts here */
143	mov	fp, #0
144	/* Branch to C code */
145	bl	initarm
146	/* We are done with the boot params */
147	add	sp, sp, #BOOTPARAMS_SIZE
148	bl	mi_startup
149
150	/* We should not get here */
151	brk	0
152
153	.align 3
154.Lvirtdone:
155	.quad	virtdone
156.Lbss:
157	.quad	__bss_start
158.Lend:
159	.quad	__bss_end
160END(_start)
161
162#ifdef SMP
163/*
164 * mpentry(unsigned long)
165 *
166 * Called by a core when it is being brought online.
167 * The data in x0 is passed straight to init_secondary.
168 */
169ENTRY(mpentry)
170	/* Disable interrupts */
171	msr	daifset, #2
172
173	/* Drop to EL1 */
174	bl	drop_to_el1
175
176	/* Set the context id */
177	msr	contextidr_el1, xzr
178
179	/* Load the kernel page table */
180	adrp	x24, pagetable_l0_ttbr1
181	add	x24, x24, :lo12:pagetable_l0_ttbr1
182	/* Load the identity page table */
183	adrp	x27, pagetable_l0_ttbr0_boostrap
184	add	x27, x27, :lo12:pagetable_l0_ttbr0_boostrap
185
186	/* Enable the mmu */
187	bl	start_mmu
188
189	/* Load the new ttbr0 pagetable */
190	adrp	x27, pagetable_l0_ttbr0
191	add	x27, x27, :lo12:pagetable_l0_ttbr0
192
193	/* Jump to the virtual address space */
194	ldr	x15, =mp_virtdone
195	br	x15
196
197mp_virtdone:
198	/* Start using the AP boot stack */
199	ldr	x4, =bootstack
200	ldr	x4, [x4]
201	mov	sp, x4
202
203	/* Load the kernel ttbr0 pagetable */
204	msr	ttbr0_el1, x27
205	isb
206
207	/* Invalidate the TLB */
208	tlbi	vmalle1
209	dsb	sy
210	isb
211
212	b	init_secondary
213END(mpentry)
214#endif
215
216/*
217 * If we are started in EL2, configure the required hypervisor
218 * registers and drop to EL1.
219 */
220LENTRY(drop_to_el1)
221	mrs	x23, CurrentEL
222	lsr	x23, x23, #2
223	cmp	x23, #0x2
224	b.eq	1f
225	ret
2261:
227	/* Configure the Hypervisor */
228	mov	x2, #(HCR_RW)
229	msr	hcr_el2, x2
230
231	/* Load the Virtualization Process ID Register */
232	mrs	x2, midr_el1
233	msr	vpidr_el2, x2
234
235	/* Load the Virtualization Multiprocess ID Register */
236	mrs	x2, mpidr_el1
237	msr	vmpidr_el2, x2
238
239	/* Set the bits that need to be 1 in sctlr_el1 */
240	ldr	x2, .Lsctlr_res1
241	msr	sctlr_el1, x2
242
243	/* Don't trap to EL2 for exceptions */
244	mov	x2, #CPTR_RES1
245	msr	cptr_el2, x2
246
247	/* Don't trap to EL2 for CP15 traps */
248	msr	hstr_el2, xzr
249
250	/* Enable access to the physical timers at EL1 */
251	mrs	x2, cnthctl_el2
252	orr	x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN)
253	msr	cnthctl_el2, x2
254
255	/* Set the counter offset to a known value */
256	msr	cntvoff_el2, xzr
257
258	/* Hypervisor trap functions */
259	adrp	x2, hyp_vectors
260	add	x2, x2, :lo12:hyp_vectors
261	msr	vbar_el2, x2
262
263	mov	x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h)
264	msr	spsr_el2, x2
265
266	/* Configure GICv3 CPU interface */
267	mrs	x2, id_aa64pfr0_el1
268	/* Extract GIC bits from the register */
269	ubfx	x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS
270	/* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */
271	cmp	x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT)
272	b.ne	2f
273
274	mrs	x2, icc_sre_el2
275	orr	x2, x2, #ICC_SRE_EL2_EN	/* Enable access from insecure EL1 */
276	orr	x2, x2, #ICC_SRE_EL2_SRE	/* Enable system registers */
277	msr	icc_sre_el2, x2
2782:
279
280	/* Set the address to return to our return address */
281	msr	elr_el2, x30
282	isb
283
284	eret
285
286	.align 3
287.Lsctlr_res1:
288	.quad SCTLR_RES1
289LEND(drop_to_el1)
290
291#define	VECT_EMPTY	\
292	.align 7;	\
293	1:	b	1b
294
295	.align 11
296hyp_vectors:
297	VECT_EMPTY	/* Synchronous EL2t */
298	VECT_EMPTY	/* IRQ EL2t */
299	VECT_EMPTY	/* FIQ EL2t */
300	VECT_EMPTY	/* Error EL2t */
301
302	VECT_EMPTY	/* Synchronous EL2h */
303	VECT_EMPTY	/* IRQ EL2h */
304	VECT_EMPTY	/* FIQ EL2h */
305	VECT_EMPTY	/* Error EL2h */
306
307	VECT_EMPTY	/* Synchronous 64-bit EL1 */
308	VECT_EMPTY	/* IRQ 64-bit EL1 */
309	VECT_EMPTY	/* FIQ 64-bit EL1 */
310	VECT_EMPTY	/* Error 64-bit EL1 */
311
312	VECT_EMPTY	/* Synchronous 32-bit EL1 */
313	VECT_EMPTY	/* IRQ 32-bit EL1 */
314	VECT_EMPTY	/* FIQ 32-bit EL1 */
315	VECT_EMPTY	/* Error 32-bit EL1 */
316
317/*
318 * Get the delta between the physical address we were loaded to and the
319 * virtual address we expect to run from. This is used when building the
320 * initial page table.
321 */
322LENTRY(get_virt_delta)
323	/* Load the physical address of virt_map */
324	adrp	x29, virt_map
325	add	x29, x29, :lo12:virt_map
326	/* Load the virtual address of virt_map stored in virt_map */
327	ldr	x28, [x29]
328	/* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */
329	sub	x29, x29, x28
330	/* Find the load address for the kernel */
331	mov	x28, #(KERNBASE)
332	add	x28, x28, x29
333	ret
334
335	.align 3
336virt_map:
337	.quad	virt_map
338LEND(get_virt_delta)
339
340/*
341 * This builds the page tables containing the identity map, and the kernel
342 * virtual map.
343 *
344 * It relys on:
345 *  We were loaded to an address that is on a 2MiB boundary
346 *  All the memory must not cross a 1GiB boundaty
347 *  x28 contains the physical address we were loaded from
348 *
349 * TODO: This is out of date.
350 *  There are at least 5 pages before that address for the page tables
351 *   The pages used are:
352 *    - The Kernel L2 table
353 *    - The Kernel L1 table
354 *    - The Kernel L0 table             (TTBR1)
355 *    - The identity (PA = VA) L1 table
356 *    - The identity (PA = VA) L0 table (TTBR0)
357 *    - The DMAP L1 tables
358 */
359LENTRY(create_pagetables)
360	/* Save the Link register */
361	mov	x5, x30
362
363	/* Clean the page table */
364	adrp	x6, pagetable
365	add	x6, x6, :lo12:pagetable
366	mov	x26, x6
367	adrp	x27, pagetable_end
368	add	x27, x27, :lo12:pagetable_end
3691:
370	stp	xzr, xzr, [x6], #16
371	stp	xzr, xzr, [x6], #16
372	stp	xzr, xzr, [x6], #16
373	stp	xzr, xzr, [x6], #16
374	cmp	x6, x27
375	b.lo	1b
376
377	/*
378	 * Build the TTBR1 maps.
379	 */
380
381	/* Find the size of the kernel */
382	mov	x6, #(KERNBASE)
383
384#if defined(LINUX_BOOT_ABI)
385	/* X19 is used as 'map FDT data' flag */
386	mov	x19, xzr
387
388	/* No modules or FDT pointer ? */
389	cbz	x0, booti_no_fdt
390
391	/*
392	 * Test if x0 points to modules descriptor(virtual address) or
393	 * to FDT (physical address)
394	 */
395	cmp	x0, x6		/* x6 is #(KERNBASE) */
396	b.lo	booti_fdt
397#endif
398
399	/* Booted with modules pointer */
400	/* Find modulep - begin */
401	sub	x8, x0, x6
402	/* Add two 2MiB pages for the module data and round up */
403	ldr	x7, =(3 * L2_SIZE - 1)
404	add	x8, x8, x7
405	b	common
406
407#if defined(LINUX_BOOT_ABI)
408booti_fdt:
409	/* Booted by U-Boot booti with FDT data */
410	/* Set 'map FDT data' flag */
411	mov	x19, #1
412
413booti_no_fdt:
414	/* Booted by U-Boot booti without FTD data */
415	/* Find the end - begin */
416	ldr     x7, .Lend
417	sub     x8, x7, x6
418
419	/*
420	 * Add one 2MiB page for copy of FDT data (maximum FDT size),
421	 * one for metadata and round up
422	 */
423	ldr	x7, =(3 * L2_SIZE - 1)
424	add	x8, x8, x7
425#endif
426
427common:
428	/* Get the number of l2 pages to allocate, rounded down */
429	lsr	x10, x8, #(L2_SHIFT)
430
431	/* Create the kernel space L2 table */
432	mov	x6, x26
433	mov	x7, #VM_MEMATTR_WRITE_BACK
434	mov	x8, #(KERNBASE & L2_BLOCK_MASK)
435	mov	x9, x28
436	bl	build_l2_block_pagetable
437
438	/* Move to the l1 table */
439	add	x26, x26, #PAGE_SIZE
440
441	/* Link the l1 -> l2 table */
442	mov	x9, x6
443	mov	x6, x26
444	bl	link_l1_pagetable
445
446	/* Move to the l0 table */
447	add	x24, x26, #PAGE_SIZE
448
449	/* Link the l0 -> l1 table */
450	mov	x9, x6
451	mov	x6, x24
452	mov	x10, #1
453	bl	link_l0_pagetable
454
455	/* Link the DMAP tables */
456	ldr	x8, =DMAP_MIN_ADDRESS
457	adrp	x9, pagetable_dmap
458	add	x9, x9, :lo12:pagetable_dmap
459	mov	x10, #DMAP_TABLES
460	bl	link_l0_pagetable
461
462	/*
463	 * Build the TTBR0 maps.  As TTBR0 maps, they must specify ATTR_S1_nG.
464	 * They are only needed early on, so the VA = PA map is uncached.
465	 */
466	add	x27, x24, #PAGE_SIZE
467
468	mov	x6, x27		/* The initial page table */
469#if defined(SOCDEV_PA) && defined(SOCDEV_VA)
470	/* Create a table for the UART */
471	mov	x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE))
472	mov	x8, #(SOCDEV_VA)	/* VA start */
473	mov	x9, #(SOCDEV_PA)	/* PA start */
474	mov	x10, #1
475	bl	build_l1_block_pagetable
476#endif
477
478#if defined(LINUX_BOOT_ABI)
479	/* Map FDT data ? */
480	cbz	x19, 1f
481
482	/* Create the identity mapping for FDT data (2 MiB max) */
483	mov	x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
484	mov	x9, x0
485	mov	x8, x0		/* VA start (== PA start) */
486	mov	x10, #1
487	bl	build_l1_block_pagetable
488
4891:
490#endif
491
492	/* Create the VA = PA map */
493	mov	x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK))
494	mov	x9, x28
495	mov	x8, x9		/* VA start (== PA start) */
496	mov	x10, #1
497	bl	build_l1_block_pagetable
498
499	/* Move to the l0 table */
500	add	x27, x27, #PAGE_SIZE
501
502	/* Link the l0 -> l1 table */
503	mov	x9, x6
504	mov	x6, x27
505	mov	x10, #1
506	bl	link_l0_pagetable
507
508	/* Restore the Link register */
509	mov	x30, x5
510	ret
511LEND(create_pagetables)
512
513/*
514 * Builds an L0 -> L1 table descriptor
515 *
516 * This is a link for a 512GiB block of memory with up to 1GiB regions mapped
517 * within it by build_l1_block_pagetable.
518 *
519 *  x6  = L0 table
520 *  x8  = Virtual Address
521 *  x9  = L1 PA (trashed)
522 *  x10 = Entry count
523 *  x11, x12 and x13 are trashed
524 */
525LENTRY(link_l0_pagetable)
526	/*
527	 * Link an L0 -> L1 table entry.
528	 */
529	/* Find the table index */
530	lsr	x11, x8, #L0_SHIFT
531	and	x11, x11, #L0_ADDR_MASK
532
533	/* Build the L0 block entry */
534	mov	x12, #L0_TABLE
535
536	/* Only use the output address bits */
537	lsr	x9, x9, #PAGE_SHIFT
5381:	orr	x13, x12, x9, lsl #PAGE_SHIFT
539
540	/* Store the entry */
541	str	x13, [x6, x11, lsl #3]
542
543	sub	x10, x10, #1
544	add	x11, x11, #1
545	add	x9, x9, #1
546	cbnz	x10, 1b
547
548	ret
549LEND(link_l0_pagetable)
550
551/*
552 * Builds an L1 -> L2 table descriptor
553 *
554 * This is a link for a 1GiB block of memory with up to 2MiB regions mapped
555 * within it by build_l2_block_pagetable.
556 *
557 *  x6  = L1 table
558 *  x8  = Virtual Address
559 *  x9  = L2 PA (trashed)
560 *  x11, x12 and x13 are trashed
561 */
562LENTRY(link_l1_pagetable)
563	/*
564	 * Link an L1 -> L2 table entry.
565	 */
566	/* Find the table index */
567	lsr	x11, x8, #L1_SHIFT
568	and	x11, x11, #Ln_ADDR_MASK
569
570	/* Build the L1 block entry */
571	mov	x12, #L1_TABLE
572
573	/* Only use the output address bits */
574	lsr	x9, x9, #PAGE_SHIFT
575	orr	x13, x12, x9, lsl #PAGE_SHIFT
576
577	/* Store the entry */
578	str	x13, [x6, x11, lsl #3]
579
580	ret
581LEND(link_l1_pagetable)
582
583/*
584 * Builds count 1 GiB page table entry
585 *  x6  = L1 table
586 *  x7  = Variable lower block attributes
587 *  x8  = VA start
588 *  x9  = PA start (trashed)
589 *  x10 = Entry count
590 *  x11, x12 and x13 are trashed
591 */
592LENTRY(build_l1_block_pagetable)
593	/*
594	 * Build the L1 table entry.
595	 */
596	/* Find the table index */
597	lsr	x11, x8, #L1_SHIFT
598	and	x11, x11, #Ln_ADDR_MASK
599
600	/* Build the L1 block entry */
601	orr	x12, x7, #L1_BLOCK
602	orr	x12, x12, #(ATTR_DEFAULT)
603
604	/* Only use the output address bits */
605	lsr	x9, x9, #L1_SHIFT
606
607	/* Set the physical address for this virtual address */
6081:	orr	x13, x12, x9, lsl #L1_SHIFT
609
610	/* Store the entry */
611	str	x13, [x6, x11, lsl #3]
612
613	sub	x10, x10, #1
614	add	x11, x11, #1
615	add	x9, x9, #1
616	cbnz	x10, 1b
617
618	ret
619LEND(build_l1_block_pagetable)
620
621/*
622 * Builds count 2 MiB page table entry
623 *  x6  = L2 table
624 *  x7  = Type (0 = Device, 1 = Normal)
625 *  x8  = VA start
626 *  x9  = PA start (trashed)
627 *  x10 = Entry count
628 *  x11, x12 and x13 are trashed
629 */
630LENTRY(build_l2_block_pagetable)
631	/*
632	 * Build the L2 table entry.
633	 */
634	/* Find the table index */
635	lsr	x11, x8, #L2_SHIFT
636	and	x11, x11, #Ln_ADDR_MASK
637
638	/* Build the L2 block entry */
639	lsl	x12, x7, #2
640	orr	x12, x12, #L2_BLOCK
641	orr	x12, x12, #(ATTR_DEFAULT)
642	orr	x12, x12, #(ATTR_S1_UXN)
643
644	/* Only use the output address bits */
645	lsr	x9, x9, #L2_SHIFT
646
647	/* Set the physical address for this virtual address */
6481:	orr	x13, x12, x9, lsl #L2_SHIFT
649
650	/* Store the entry */
651	str	x13, [x6, x11, lsl #3]
652
653	sub	x10, x10, #1
654	add	x11, x11, #1
655	add	x9, x9, #1
656	cbnz	x10, 1b
657
658	ret
659LEND(build_l2_block_pagetable)
660
661LENTRY(start_mmu)
662	dsb	sy
663
664	/* Load the exception vectors */
665	ldr	x2, =exception_vectors
666	msr	vbar_el1, x2
667
668	/* Load ttbr0 and ttbr1 */
669	msr	ttbr0_el1, x27
670	msr	ttbr1_el1, x24
671	isb
672
673	/* Clear the Monitor Debug System control register */
674	msr	mdscr_el1, xzr
675
676	/* Invalidate the TLB */
677	tlbi	vmalle1is
678	dsb	ish
679	isb
680
681	ldr	x2, mair
682	msr	mair_el1, x2
683
684	/*
685	 * Setup TCR according to the PARange and ASIDBits fields
686	 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the
687	 * ID_AA64MMFR1_EL1.  More precisely, set TCR_EL1.AS
688	 * to 1 only if the ASIDBits field equals 0b0010.
689	 */
690	ldr	x2, tcr
691	mrs	x3, id_aa64mmfr0_el1
692
693	/* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */
694	bfi	x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH)
695	and	x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK)
696
697	/* Check if the HW supports 16 bit ASIDS */
698	cmp	x3, #(ID_AA64MMFR0_ASIDBits_16)
699	/* If so x3 == 1, else x3 == 0 */
700	cset	x3, eq
701	/* Set TCR.AS with x3 */
702	bfi	x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH)
703
704	/*
705	 * Check if the HW supports access flag and dirty state updates,
706	 * and set TCR_EL1.HA and TCR_EL1.HD accordingly.
707	 */
708	mrs	x3, id_aa64mmfr1_el1
709	and	x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK)
710	cmp	x3, #1
711	b.ne	1f
712	orr 	x2, x2, #(TCR_HA)
713	b	2f
7141:
715	cmp	x3, #2
716	b.ne	2f
717	orr 	x2, x2, #(TCR_HA | TCR_HD)
7182:
719	msr	tcr_el1, x2
720
721	/*
722	 * Setup SCTLR.
723	 */
724	ldr	x2, sctlr_set
725	ldr	x3, sctlr_clear
726	mrs	x1, sctlr_el1
727	bic	x1, x1, x3	/* Clear the required bits */
728	orr	x1, x1, x2	/* Set the required bits */
729	msr	sctlr_el1, x1
730	isb
731
732	ret
733
734	.align 3
735mair:
736	.quad	MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \
737		MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE)   |	\
738		MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK)    |	\
739		MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) |	\
740		MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE)
741tcr:
742	.quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG1_4K | \
743	    TCR_CACHE_ATTRS | TCR_SMP_ATTRS)
744sctlr_set:
745	/* Bits to set */
746	.quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \
747	    SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \
748	    SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \
749	    SCTLR_M | SCTLR_CP15BEN)
750sctlr_clear:
751	/* Bits to clear */
752	.quad (SCTLR_EE | SCTLR_EOE | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \
753	    SCTLR_ITD | SCTLR_A)
754LEND(start_mmu)
755
756ENTRY(abort)
757	b abort
758END(abort)
759
760	.align 3
761init_pt_va:
762	.quad pagetable		/* XXX: Keep page tables VA */
763
764	.section .init_pagetable, "aw", %nobits
765	.align PAGE_SHIFT
766	/*
767	 * 6 initial tables (in the following order):
768	 *           L2 for kernel (High addresses)
769	 *           L1 for kernel
770	 *           L0 for kernel
771	 *           L1 bootstrap for user   (Low addresses)
772	 *           L0 bootstrap for user
773	 *           L0 for user
774	 */
775pagetable:
776	.space	PAGE_SIZE
777pagetable_l1_ttbr1:
778	.space	PAGE_SIZE
779pagetable_l0_ttbr1:
780	.space	PAGE_SIZE
781pagetable_l1_ttbr0_bootstrap:
782	.space	PAGE_SIZE
783pagetable_l0_ttbr0_boostrap:
784	.space	PAGE_SIZE
785pagetable_l0_ttbr0:
786	.space	PAGE_SIZE
787
788	.globl pagetable_dmap
789pagetable_dmap:
790	.space	PAGE_SIZE * DMAP_TABLES
791pagetable_end:
792
793el2_pagetable:
794	.space	PAGE_SIZE
795
796	.globl init_pt_va
797
798	.align	4
799initstack:
800	.space	(PAGE_SIZE * KSTACK_PAGES)
801initstack_end:
802
803
804ENTRY(sigcode)
805	mov	x0, sp
806	add	x0, x0, #SF_UC
807
8081:
809	mov	x8, #SYS_sigreturn
810	svc	0
811
812	/* sigreturn failed, exit */
813	mov	x8, #SYS_exit
814	svc	0
815
816	b	1b
817END(sigcode)
818	/* This may be copied to the stack, keep it 16-byte aligned */
819	.align	3
820esigcode:
821
822	.data
823	.align	3
824	.global	szsigcode
825szsigcode:
826	.quad	esigcode - sigcode
827
828ENTRY(aarch32_sigcode)
829	.word 0xe1a0000d	// mov r0, sp
830	.word 0xe2800040	// add r0, r0, #SIGF_UC
831	.word 0xe59f700c	// ldr r7, [pc, #12]
832	.word 0xef000000	// swi #0
833	.word 0xe59f7008	// ldr r7, [pc, #8]
834	.word 0xef000000	// swi #0
835	.word 0xeafffffa	// b . - 16
836END(aarch32_sigcode)
837	.word SYS_sigreturn
838	.word SYS_exit
839	.align	3
840aarch32_esigcode:
841	.data
842	.global sz_aarch32_sigcode
843sz_aarch32_sigcode:
844	.quad aarch32_esigcode - aarch32_sigcode
845