1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6#include <asm/asm-offsets.h> 7#include <asm/asm.h> 8#include <linux/init.h> 9#include <linux/linkage.h> 10#include <asm/thread_info.h> 11#include <asm/page.h> 12#include <asm/pgtable.h> 13#include <asm/csr.h> 14#include <asm/cpu_ops_sbi.h> 15#include <asm/hwcap.h> 16#include <asm/image.h> 17#include "efi-header.S" 18 19#ifdef CONFIG_XIP_KERNEL 20.macro XIP_FIXUP_OFFSET reg 21 REG_L t0, _xip_fixup 22 add \reg, \reg, t0 23.endm 24.macro XIP_FIXUP_FLASH_OFFSET reg 25 la t1, __data_loc 26 li t0, XIP_OFFSET_MASK 27 and t1, t1, t0 28 li t1, XIP_OFFSET 29 sub t0, t0, t1 30 sub \reg, \reg, t0 31.endm 32_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET 33#else 34.macro XIP_FIXUP_OFFSET reg 35.endm 36.macro XIP_FIXUP_FLASH_OFFSET reg 37.endm 38#endif /* CONFIG_XIP_KERNEL */ 39 40__HEAD 41ENTRY(_start) 42 /* 43 * Image header expected by Linux boot-loaders. The image header data 44 * structure is described in asm/image.h. 45 * Do not modify it without modifying the structure and all bootloaders 46 * that expects this header format!! 47 */ 48#ifdef CONFIG_EFI 49 /* 50 * This instruction decodes to "MZ" ASCII required by UEFI. 51 */ 52 c.li s4,-13 53 j _start_kernel 54#else 55 /* jump to start kernel */ 56 j _start_kernel 57 /* reserved */ 58 .word 0 59#endif 60 .balign 8 61#ifdef CONFIG_RISCV_M_MODE 62 /* Image load offset (0MB) from start of RAM for M-mode */ 63 .dword 0 64#else 65#if __riscv_xlen == 64 66 /* Image load offset(2MB) from start of RAM */ 67 .dword 0x200000 68#else 69 /* Image load offset(4MB) from start of RAM */ 70 .dword 0x400000 71#endif 72#endif 73 /* Effective size of kernel image */ 74 .dword _end - _start 75 .dword __HEAD_FLAGS 76 .word RISCV_HEADER_VERSION 77 .word 0 78 .dword 0 79 .ascii RISCV_IMAGE_MAGIC 80 .balign 4 81 .ascii RISCV_IMAGE_MAGIC2 82#ifdef CONFIG_EFI 83 .word pe_head_start - _start 84pe_head_start: 85 86 __EFI_PE_HEADER 87#else 88 .word 0 89#endif 90 91.align 2 92#ifdef CONFIG_MMU 93 .global relocate_enable_mmu 94relocate_enable_mmu: 95 /* Relocate return address */ 96 la a1, kernel_map 97 XIP_FIXUP_OFFSET a1 98 REG_L a1, KERNEL_MAP_VIRT_ADDR(a1) 99 la a2, _start 100 sub a1, a1, a2 101 add ra, ra, a1 102 103 /* Point stvec to virtual address of intruction after satp write */ 104 la a2, 1f 105 add a2, a2, a1 106 csrw CSR_TVEC, a2 107 108 /* Compute satp for kernel page tables, but don't load it yet */ 109 srl a2, a0, PAGE_SHIFT 110 la a1, satp_mode 111 REG_L a1, 0(a1) 112 or a2, a2, a1 113 114 /* 115 * Load trampoline page directory, which will cause us to trap to 116 * stvec if VA != PA, or simply fall through if VA == PA. We need a 117 * full fence here because setup_vm() just wrote these PTEs and we need 118 * to ensure the new translations are in use. 119 */ 120 la a0, trampoline_pg_dir 121 XIP_FIXUP_OFFSET a0 122 srl a0, a0, PAGE_SHIFT 123 or a0, a0, a1 124 sfence.vma 125 csrw CSR_SATP, a0 126.align 2 1271: 128 /* Set trap vector to spin forever to help debug */ 129 la a0, .Lsecondary_park 130 csrw CSR_TVEC, a0 131 132 /* Reload the global pointer */ 133.option push 134.option norelax 135 la gp, __global_pointer$ 136.option pop 137 138 /* 139 * Switch to kernel page tables. A full fence is necessary in order to 140 * avoid using the trampoline translations, which are only correct for 141 * the first superpage. Fetching the fence is guaranteed to work 142 * because that first superpage is translated the same way. 143 */ 144 csrw CSR_SATP, a2 145 sfence.vma 146 147 ret 148#endif /* CONFIG_MMU */ 149#ifdef CONFIG_SMP 150 .global secondary_start_sbi 151secondary_start_sbi: 152 /* Mask all interrupts */ 153 csrw CSR_IE, zero 154 csrw CSR_IP, zero 155 156 /* Load the global pointer */ 157 .option push 158 .option norelax 159 la gp, __global_pointer$ 160 .option pop 161 162 /* 163 * Disable FPU to detect illegal usage of 164 * floating point in kernel space 165 */ 166 li t0, SR_FS 167 csrc CSR_STATUS, t0 168 169 /* Set trap vector to spin forever to help debug */ 170 la a3, .Lsecondary_park 171 csrw CSR_TVEC, a3 172 173 /* a0 contains the hartid & a1 contains boot data */ 174 li a2, SBI_HART_BOOT_TASK_PTR_OFFSET 175 XIP_FIXUP_OFFSET a2 176 add a2, a2, a1 177 REG_L tp, (a2) 178 li a3, SBI_HART_BOOT_STACK_PTR_OFFSET 179 XIP_FIXUP_OFFSET a3 180 add a3, a3, a1 181 REG_L sp, (a3) 182 183.Lsecondary_start_common: 184 185#ifdef CONFIG_MMU 186 /* Enable virtual memory and relocate to virtual address */ 187 la a0, swapper_pg_dir 188 XIP_FIXUP_OFFSET a0 189 call relocate_enable_mmu 190#endif 191 call setup_trap_vector 192 tail smp_callin 193#endif /* CONFIG_SMP */ 194 195.align 2 196setup_trap_vector: 197 /* Set trap vector to exception handler */ 198 la a0, handle_exception 199 csrw CSR_TVEC, a0 200 201 /* 202 * Set sup0 scratch register to 0, indicating to exception vector that 203 * we are presently executing in kernel. 204 */ 205 csrw CSR_SCRATCH, zero 206 ret 207 208.align 2 209.Lsecondary_park: 210 /* We lack SMP support or have too many harts, so park this hart */ 211 wfi 212 j .Lsecondary_park 213 214END(_start) 215 216ENTRY(_start_kernel) 217 /* Mask all interrupts */ 218 csrw CSR_IE, zero 219 csrw CSR_IP, zero 220 221#ifdef CONFIG_RISCV_M_MODE 222 /* flush the instruction cache */ 223 fence.i 224 225 /* Reset all registers except ra, a0, a1 */ 226 call reset_regs 227 228 /* 229 * Setup a PMP to permit access to all of memory. Some machines may 230 * not implement PMPs, so we set up a quick trap handler to just skip 231 * touching the PMPs on any trap. 232 */ 233 la a0, pmp_done 234 csrw CSR_TVEC, a0 235 236 li a0, -1 237 csrw CSR_PMPADDR0, a0 238 li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X) 239 csrw CSR_PMPCFG0, a0 240.align 2 241pmp_done: 242 243 /* 244 * The hartid in a0 is expected later on, and we have no firmware 245 * to hand it to us. 246 */ 247 csrr a0, CSR_MHARTID 248#endif /* CONFIG_RISCV_M_MODE */ 249 250 /* Load the global pointer */ 251.option push 252.option norelax 253 la gp, __global_pointer$ 254.option pop 255 256 /* 257 * Disable FPU to detect illegal usage of 258 * floating point in kernel space 259 */ 260 li t0, SR_FS 261 csrc CSR_STATUS, t0 262 263#ifdef CONFIG_RISCV_BOOT_SPINWAIT 264 li t0, CONFIG_NR_CPUS 265 blt a0, t0, .Lgood_cores 266 tail .Lsecondary_park 267.Lgood_cores: 268 269 /* The lottery system is only required for spinwait booting method */ 270#ifndef CONFIG_XIP_KERNEL 271 /* Pick one hart to run the main boot sequence */ 272 la a3, hart_lottery 273 li a2, 1 274 amoadd.w a3, a2, (a3) 275 bnez a3, .Lsecondary_start 276 277#else 278 /* hart_lottery in flash contains a magic number */ 279 la a3, hart_lottery 280 mv a2, a3 281 XIP_FIXUP_OFFSET a2 282 XIP_FIXUP_FLASH_OFFSET a3 283 lw t1, (a3) 284 amoswap.w t0, t1, (a2) 285 /* first time here if hart_lottery in RAM is not set */ 286 beq t0, t1, .Lsecondary_start 287 288#endif /* CONFIG_XIP */ 289#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ 290 291#ifdef CONFIG_XIP_KERNEL 292 la sp, _end + THREAD_SIZE 293 XIP_FIXUP_OFFSET sp 294 mv s0, a0 295 call __copy_data 296 297 /* Restore a0 copy */ 298 mv a0, s0 299#endif 300 301#ifndef CONFIG_XIP_KERNEL 302 /* Clear BSS for flat non-ELF images */ 303 la a3, __bss_start 304 la a4, __bss_stop 305 ble a4, a3, clear_bss_done 306clear_bss: 307 REG_S zero, (a3) 308 add a3, a3, RISCV_SZPTR 309 blt a3, a4, clear_bss 310clear_bss_done: 311#endif 312 /* Save hart ID and DTB physical address */ 313 mv s0, a0 314 mv s1, a1 315 316 la a2, boot_cpu_hartid 317 XIP_FIXUP_OFFSET a2 318 REG_S a0, (a2) 319 320 /* Initialize page tables and relocate to virtual addresses */ 321 la sp, init_thread_union + THREAD_SIZE 322 XIP_FIXUP_OFFSET sp 323#ifdef CONFIG_BUILTIN_DTB 324 la a0, __dtb_start 325 XIP_FIXUP_OFFSET a0 326#else 327 mv a0, s1 328#endif /* CONFIG_BUILTIN_DTB */ 329 call setup_vm 330#ifdef CONFIG_MMU 331 la a0, early_pg_dir 332 XIP_FIXUP_OFFSET a0 333 call relocate_enable_mmu 334#endif /* CONFIG_MMU */ 335 336 call setup_trap_vector 337 /* Restore C environment */ 338 la tp, init_task 339 la sp, init_thread_union + THREAD_SIZE 340 341#ifdef CONFIG_KASAN 342 call kasan_early_init 343#endif 344 /* Start the kernel */ 345 call soc_early_init 346 tail start_kernel 347 348#if CONFIG_RISCV_BOOT_SPINWAIT 349.Lsecondary_start: 350 /* Set trap vector to spin forever to help debug */ 351 la a3, .Lsecondary_park 352 csrw CSR_TVEC, a3 353 354 slli a3, a0, LGREG 355 la a1, __cpu_spinwait_stack_pointer 356 XIP_FIXUP_OFFSET a1 357 la a2, __cpu_spinwait_task_pointer 358 XIP_FIXUP_OFFSET a2 359 add a1, a3, a1 360 add a2, a3, a2 361 362 /* 363 * This hart didn't win the lottery, so we wait for the winning hart to 364 * get far enough along the boot process that it should continue. 365 */ 366.Lwait_for_cpu_up: 367 /* FIXME: We should WFI to save some energy here. */ 368 REG_L sp, (a1) 369 REG_L tp, (a2) 370 beqz sp, .Lwait_for_cpu_up 371 beqz tp, .Lwait_for_cpu_up 372 fence 373 374 tail .Lsecondary_start_common 375#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ 376 377END(_start_kernel) 378 379#ifdef CONFIG_RISCV_M_MODE 380ENTRY(reset_regs) 381 li sp, 0 382 li gp, 0 383 li tp, 0 384 li t0, 0 385 li t1, 0 386 li t2, 0 387 li s0, 0 388 li s1, 0 389 li a2, 0 390 li a3, 0 391 li a4, 0 392 li a5, 0 393 li a6, 0 394 li a7, 0 395 li s2, 0 396 li s3, 0 397 li s4, 0 398 li s5, 0 399 li s6, 0 400 li s7, 0 401 li s8, 0 402 li s9, 0 403 li s10, 0 404 li s11, 0 405 li t3, 0 406 li t4, 0 407 li t5, 0 408 li t6, 0 409 csrw CSR_SCRATCH, 0 410 411#ifdef CONFIG_FPU 412 csrr t0, CSR_MISA 413 andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D) 414 beqz t0, .Lreset_regs_done 415 416 li t1, SR_FS 417 csrs CSR_STATUS, t1 418 fmv.s.x f0, zero 419 fmv.s.x f1, zero 420 fmv.s.x f2, zero 421 fmv.s.x f3, zero 422 fmv.s.x f4, zero 423 fmv.s.x f5, zero 424 fmv.s.x f6, zero 425 fmv.s.x f7, zero 426 fmv.s.x f8, zero 427 fmv.s.x f9, zero 428 fmv.s.x f10, zero 429 fmv.s.x f11, zero 430 fmv.s.x f12, zero 431 fmv.s.x f13, zero 432 fmv.s.x f14, zero 433 fmv.s.x f15, zero 434 fmv.s.x f16, zero 435 fmv.s.x f17, zero 436 fmv.s.x f18, zero 437 fmv.s.x f19, zero 438 fmv.s.x f20, zero 439 fmv.s.x f21, zero 440 fmv.s.x f22, zero 441 fmv.s.x f23, zero 442 fmv.s.x f24, zero 443 fmv.s.x f25, zero 444 fmv.s.x f26, zero 445 fmv.s.x f27, zero 446 fmv.s.x f28, zero 447 fmv.s.x f29, zero 448 fmv.s.x f30, zero 449 fmv.s.x f31, zero 450 csrw fcsr, 0 451 /* note that the caller must clear SR_FS */ 452#endif /* CONFIG_FPU */ 453.Lreset_regs_done: 454 ret 455END(reset_regs) 456#endif /* CONFIG_RISCV_M_MODE */ 457