1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6#include <asm/asm-offsets.h> 7#include <asm/asm.h> 8#include <linux/init.h> 9#include <linux/linkage.h> 10#include <asm/thread_info.h> 11#include <asm/page.h> 12#include <asm/pgtable.h> 13#include <asm/csr.h> 14#include <asm/hwcap.h> 15#include <asm/image.h> 16#include <asm/scs.h> 17#include <asm/xip_fixup.h> 18#include "efi-header.S" 19 20__HEAD 21SYM_CODE_START(_start) 22 /* 23 * Image header expected by Linux boot-loaders. The image header data 24 * structure is described in asm/image.h. 25 * Do not modify it without modifying the structure and all bootloaders 26 * that expects this header format!! 27 */ 28#ifdef CONFIG_EFI 29 /* 30 * This instruction decodes to "MZ" ASCII required by UEFI. 31 */ 32 c.li s4,-13 33 j _start_kernel 34#else 35 /* jump to start kernel */ 36 j _start_kernel 37 /* reserved */ 38 .word 0 39#endif 40 .balign 8 41#ifdef CONFIG_RISCV_M_MODE 42 /* Image load offset (0MB) from start of RAM for M-mode */ 43 .dword 0 44#else 45#if __riscv_xlen == 64 46 /* Image load offset(2MB) from start of RAM */ 47 .dword 0x200000 48#else 49 /* Image load offset(4MB) from start of RAM */ 50 .dword 0x400000 51#endif 52#endif 53 /* Effective size of kernel image */ 54 .dword _end - _start 55 .dword __HEAD_FLAGS 56 .word RISCV_HEADER_VERSION 57 .word 0 58 .dword 0 59 .ascii RISCV_IMAGE_MAGIC 60 .balign 4 61 .ascii RISCV_IMAGE_MAGIC2 62#ifdef CONFIG_EFI 63 .word pe_head_start - _start 64pe_head_start: 65 66 __EFI_PE_HEADER 67#else 68 .word 0 69#endif 70 71.align 2 72#ifdef CONFIG_MMU 73 .global relocate_enable_mmu 74relocate_enable_mmu: 75 /* Relocate return address */ 76 la a1, kernel_map 77 XIP_FIXUP_OFFSET a1 78 REG_L a1, KERNEL_MAP_VIRT_ADDR(a1) 79 la a2, _start 80 sub a1, a1, a2 81 add ra, ra, a1 82 83 /* Point stvec to virtual address of intruction after satp write */ 84 la a2, 1f 85 add a2, a2, a1 86 csrw CSR_TVEC, a2 87 88 /* Compute satp for kernel page tables, but don't load it yet */ 89 srl a2, a0, PAGE_SHIFT 90 la a1, satp_mode 91 XIP_FIXUP_OFFSET a1 92 REG_L a1, 0(a1) 93 or a2, a2, a1 94 95 /* 96 * Load trampoline page directory, which will cause us to trap to 97 * stvec if VA != PA, or simply fall through if VA == PA. We need a 98 * full fence here because setup_vm() just wrote these PTEs and we need 99 * to ensure the new translations are in use. 100 */ 101 la a0, trampoline_pg_dir 102 XIP_FIXUP_OFFSET a0 103 srl a0, a0, PAGE_SHIFT 104 or a0, a0, a1 105 sfence.vma 106 csrw CSR_SATP, a0 107.align 2 1081: 109 /* Set trap vector to spin forever to help debug */ 110 la a0, .Lsecondary_park 111 csrw CSR_TVEC, a0 112 113 /* Reload the global pointer */ 114 load_global_pointer 115 116 /* 117 * Switch to kernel page tables. A full fence is necessary in order to 118 * avoid using the trampoline translations, which are only correct for 119 * the first superpage. Fetching the fence is guaranteed to work 120 * because that first superpage is translated the same way. 121 */ 122 csrw CSR_SATP, a2 123 sfence.vma 124 125 ret 126#endif /* CONFIG_MMU */ 127#ifdef CONFIG_SMP 128 .global secondary_start_sbi 129secondary_start_sbi: 130 /* Mask all interrupts */ 131 csrw CSR_IE, zero 132 csrw CSR_IP, zero 133 134 /* Load the global pointer */ 135 load_global_pointer 136 137 /* 138 * Disable FPU & VECTOR to detect illegal usage of 139 * floating point or vector in kernel space 140 */ 141 li t0, SR_FS_VS 142 csrc CSR_STATUS, t0 143 144 /* Set trap vector to spin forever to help debug */ 145 la a3, .Lsecondary_park 146 csrw CSR_TVEC, a3 147 148 /* a0 contains the hartid & a1 contains boot data */ 149 li a2, SBI_HART_BOOT_TASK_PTR_OFFSET 150 XIP_FIXUP_OFFSET a2 151 add a2, a2, a1 152 REG_L tp, (a2) 153 li a3, SBI_HART_BOOT_STACK_PTR_OFFSET 154 XIP_FIXUP_OFFSET a3 155 add a3, a3, a1 156 REG_L sp, (a3) 157 158.Lsecondary_start_common: 159 160#ifdef CONFIG_MMU 161 /* Enable virtual memory and relocate to virtual address */ 162 la a0, swapper_pg_dir 163 XIP_FIXUP_OFFSET a0 164 call relocate_enable_mmu 165#endif 166 call .Lsetup_trap_vector 167 scs_load_current 168 call smp_callin 169#endif /* CONFIG_SMP */ 170 171.align 2 172.Lsecondary_park: 173 /* 174 * Park this hart if we: 175 * - have too many harts on CONFIG_RISCV_BOOT_SPINWAIT 176 * - receive an early trap, before setup_trap_vector finished 177 * - fail in smp_callin(), as a successful one wouldn't return 178 */ 179 wfi 180 j .Lsecondary_park 181 182.align 2 183.Lsetup_trap_vector: 184 /* Set trap vector to exception handler */ 185 la a0, handle_exception 186 csrw CSR_TVEC, a0 187 188 /* 189 * Set sup0 scratch register to 0, indicating to exception vector that 190 * we are presently executing in kernel. 191 */ 192 csrw CSR_SCRATCH, zero 193 ret 194 195SYM_CODE_END(_start) 196 197SYM_CODE_START(_start_kernel) 198 /* Mask all interrupts */ 199 csrw CSR_IE, zero 200 csrw CSR_IP, zero 201 202#ifdef CONFIG_RISCV_M_MODE 203 /* flush the instruction cache */ 204 fence.i 205 206 /* Reset all registers except ra, a0, a1 */ 207 call reset_regs 208 209 /* 210 * Setup a PMP to permit access to all of memory. Some machines may 211 * not implement PMPs, so we set up a quick trap handler to just skip 212 * touching the PMPs on any trap. 213 */ 214 la a0, .Lpmp_done 215 csrw CSR_TVEC, a0 216 217 li a0, -1 218 csrw CSR_PMPADDR0, a0 219 li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X) 220 csrw CSR_PMPCFG0, a0 221.align 2 222.Lpmp_done: 223 224 /* 225 * The hartid in a0 is expected later on, and we have no firmware 226 * to hand it to us. 227 */ 228 csrr a0, CSR_MHARTID 229#endif /* CONFIG_RISCV_M_MODE */ 230 231 /* Load the global pointer */ 232 load_global_pointer 233 234 /* 235 * Disable FPU & VECTOR to detect illegal usage of 236 * floating point or vector in kernel space 237 */ 238 li t0, SR_FS_VS 239 csrc CSR_STATUS, t0 240 241#ifdef CONFIG_RISCV_BOOT_SPINWAIT 242 li t0, CONFIG_NR_CPUS 243 blt a0, t0, .Lgood_cores 244 tail .Lsecondary_park 245.Lgood_cores: 246 247 /* The lottery system is only required for spinwait booting method */ 248#ifndef CONFIG_XIP_KERNEL 249 /* Pick one hart to run the main boot sequence */ 250 la a3, hart_lottery 251 li a2, 1 252 amoadd.w a3, a2, (a3) 253 bnez a3, .Lsecondary_start 254 255#else 256 /* hart_lottery in flash contains a magic number */ 257 la a3, hart_lottery 258 mv a2, a3 259 XIP_FIXUP_OFFSET a2 260 XIP_FIXUP_FLASH_OFFSET a3 261 lw t1, (a3) 262 amoswap.w t0, t1, (a2) 263 /* first time here if hart_lottery in RAM is not set */ 264 beq t0, t1, .Lsecondary_start 265 266#endif /* CONFIG_XIP */ 267#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ 268 269#ifdef CONFIG_XIP_KERNEL 270 la sp, _end + THREAD_SIZE 271 XIP_FIXUP_OFFSET sp 272 mv s0, a0 273 mv s1, a1 274 call __copy_data 275 276 /* Restore a0 & a1 copy */ 277 mv a0, s0 278 mv a1, s1 279#endif 280 281#ifndef CONFIG_XIP_KERNEL 282 /* Clear BSS for flat non-ELF images */ 283 la a3, __bss_start 284 la a4, __bss_stop 285 ble a4, a3, .Lclear_bss_done 286.Lclear_bss: 287 REG_S zero, (a3) 288 add a3, a3, RISCV_SZPTR 289 blt a3, a4, .Lclear_bss 290.Lclear_bss_done: 291#endif 292 la a2, boot_cpu_hartid 293 XIP_FIXUP_OFFSET a2 294 REG_S a0, (a2) 295 296 /* Initialize page tables and relocate to virtual addresses */ 297 la tp, init_task 298 la sp, init_thread_union + THREAD_SIZE 299 XIP_FIXUP_OFFSET sp 300 addi sp, sp, -PT_SIZE_ON_STACK 301 scs_load_init_stack 302#ifdef CONFIG_BUILTIN_DTB 303 la a0, __dtb_start 304 XIP_FIXUP_OFFSET a0 305#else 306 mv a0, a1 307#endif /* CONFIG_BUILTIN_DTB */ 308 /* Set trap vector to spin forever to help debug */ 309 la a3, .Lsecondary_park 310 csrw CSR_TVEC, a3 311 call setup_vm 312#ifdef CONFIG_MMU 313 la a0, early_pg_dir 314 XIP_FIXUP_OFFSET a0 315 call relocate_enable_mmu 316#endif /* CONFIG_MMU */ 317 318 call .Lsetup_trap_vector 319 /* Restore C environment */ 320 la tp, init_task 321 la sp, init_thread_union + THREAD_SIZE 322 addi sp, sp, -PT_SIZE_ON_STACK 323 scs_load_current 324 325#ifdef CONFIG_KASAN 326 call kasan_early_init 327#endif 328 /* Start the kernel */ 329 call soc_early_init 330 tail start_kernel 331 332#ifdef CONFIG_RISCV_BOOT_SPINWAIT 333.Lsecondary_start: 334 /* Set trap vector to spin forever to help debug */ 335 la a3, .Lsecondary_park 336 csrw CSR_TVEC, a3 337 338 slli a3, a0, LGREG 339 la a1, __cpu_spinwait_stack_pointer 340 XIP_FIXUP_OFFSET a1 341 la a2, __cpu_spinwait_task_pointer 342 XIP_FIXUP_OFFSET a2 343 add a1, a3, a1 344 add a2, a3, a2 345 346 /* 347 * This hart didn't win the lottery, so we wait for the winning hart to 348 * get far enough along the boot process that it should continue. 349 */ 350.Lwait_for_cpu_up: 351 /* FIXME: We should WFI to save some energy here. */ 352 REG_L sp, (a1) 353 REG_L tp, (a2) 354 beqz sp, .Lwait_for_cpu_up 355 beqz tp, .Lwait_for_cpu_up 356 fence 357 358 tail .Lsecondary_start_common 359#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ 360 361SYM_CODE_END(_start_kernel) 362 363#ifdef CONFIG_RISCV_M_MODE 364SYM_CODE_START_LOCAL(reset_regs) 365 li sp, 0 366 li gp, 0 367 li tp, 0 368 li t0, 0 369 li t1, 0 370 li t2, 0 371 li s0, 0 372 li s1, 0 373 li a2, 0 374 li a3, 0 375 li a4, 0 376 li a5, 0 377 li a6, 0 378 li a7, 0 379 li s2, 0 380 li s3, 0 381 li s4, 0 382 li s5, 0 383 li s6, 0 384 li s7, 0 385 li s8, 0 386 li s9, 0 387 li s10, 0 388 li s11, 0 389 li t3, 0 390 li t4, 0 391 li t5, 0 392 li t6, 0 393 csrw CSR_SCRATCH, 0 394 395#ifdef CONFIG_FPU 396 csrr t0, CSR_MISA 397 andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D) 398 beqz t0, .Lreset_regs_done_fpu 399 400 li t1, SR_FS 401 csrs CSR_STATUS, t1 402 fmv.s.x f0, zero 403 fmv.s.x f1, zero 404 fmv.s.x f2, zero 405 fmv.s.x f3, zero 406 fmv.s.x f4, zero 407 fmv.s.x f5, zero 408 fmv.s.x f6, zero 409 fmv.s.x f7, zero 410 fmv.s.x f8, zero 411 fmv.s.x f9, zero 412 fmv.s.x f10, zero 413 fmv.s.x f11, zero 414 fmv.s.x f12, zero 415 fmv.s.x f13, zero 416 fmv.s.x f14, zero 417 fmv.s.x f15, zero 418 fmv.s.x f16, zero 419 fmv.s.x f17, zero 420 fmv.s.x f18, zero 421 fmv.s.x f19, zero 422 fmv.s.x f20, zero 423 fmv.s.x f21, zero 424 fmv.s.x f22, zero 425 fmv.s.x f23, zero 426 fmv.s.x f24, zero 427 fmv.s.x f25, zero 428 fmv.s.x f26, zero 429 fmv.s.x f27, zero 430 fmv.s.x f28, zero 431 fmv.s.x f29, zero 432 fmv.s.x f30, zero 433 fmv.s.x f31, zero 434 csrw fcsr, 0 435 /* note that the caller must clear SR_FS */ 436.Lreset_regs_done_fpu: 437#endif /* CONFIG_FPU */ 438 439#ifdef CONFIG_RISCV_ISA_V 440 csrr t0, CSR_MISA 441 li t1, COMPAT_HWCAP_ISA_V 442 and t0, t0, t1 443 beqz t0, .Lreset_regs_done_vector 444 445 /* 446 * Clear vector registers and reset vcsr 447 * VLMAX has a defined value, VLEN is a constant, 448 * and this form of vsetvli is defined to set vl to VLMAX. 449 */ 450 li t1, SR_VS 451 csrs CSR_STATUS, t1 452 csrs CSR_VCSR, x0 453 vsetvli t1, x0, e8, m8, ta, ma 454 vmv.v.i v0, 0 455 vmv.v.i v8, 0 456 vmv.v.i v16, 0 457 vmv.v.i v24, 0 458 /* note that the caller must clear SR_VS */ 459.Lreset_regs_done_vector: 460#endif /* CONFIG_RISCV_ISA_V */ 461 ret 462SYM_CODE_END(reset_regs) 463#endif /* CONFIG_RISCV_M_MODE */ 464