1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6#include <asm/asm-offsets.h> 7#include <asm/asm.h> 8#include <linux/init.h> 9#include <linux/linkage.h> 10#include <asm/thread_info.h> 11#include <asm/page.h> 12#include <asm/pgtable.h> 13#include <asm/csr.h> 14#include <asm/cpu_ops_sbi.h> 15#include <asm/hwcap.h> 16#include <asm/image.h> 17#include <asm/scs.h> 18#include <asm/xip_fixup.h> 19#include "efi-header.S" 20 21__HEAD 22SYM_CODE_START(_start) 23 /* 24 * Image header expected by Linux boot-loaders. The image header data 25 * structure is described in asm/image.h. 26 * Do not modify it without modifying the structure and all bootloaders 27 * that expects this header format!! 28 */ 29#ifdef CONFIG_EFI 30 /* 31 * This instruction decodes to "MZ" ASCII required by UEFI. 32 */ 33 c.li s4,-13 34 j _start_kernel 35#else 36 /* jump to start kernel */ 37 j _start_kernel 38 /* reserved */ 39 .word 0 40#endif 41 .balign 8 42#ifdef CONFIG_RISCV_M_MODE 43 /* Image load offset (0MB) from start of RAM for M-mode */ 44 .dword 0 45#else 46#if __riscv_xlen == 64 47 /* Image load offset(2MB) from start of RAM */ 48 .dword 0x200000 49#else 50 /* Image load offset(4MB) from start of RAM */ 51 .dword 0x400000 52#endif 53#endif 54 /* Effective size of kernel image */ 55 .dword _end - _start 56 .dword __HEAD_FLAGS 57 .word RISCV_HEADER_VERSION 58 .word 0 59 .dword 0 60 .ascii RISCV_IMAGE_MAGIC 61 .balign 4 62 .ascii RISCV_IMAGE_MAGIC2 63#ifdef CONFIG_EFI 64 .word pe_head_start - _start 65pe_head_start: 66 67 __EFI_PE_HEADER 68#else 69 .word 0 70#endif 71 72.align 2 73#ifdef CONFIG_MMU 74 .global relocate_enable_mmu 75relocate_enable_mmu: 76 /* Relocate return address */ 77 la a1, kernel_map 78 XIP_FIXUP_OFFSET a1 79 REG_L a1, KERNEL_MAP_VIRT_ADDR(a1) 80 la a2, _start 81 sub a1, a1, a2 82 add ra, ra, a1 83 84 /* Point stvec to virtual address of intruction after satp write */ 85 la a2, 1f 86 add a2, a2, a1 87 csrw CSR_TVEC, a2 88 89 /* Compute satp for kernel page tables, but don't load it yet */ 90 srl a2, a0, PAGE_SHIFT 91 la a1, satp_mode 92 REG_L a1, 0(a1) 93 or a2, a2, a1 94 95 /* 96 * Load trampoline page directory, which will cause us to trap to 97 * stvec if VA != PA, or simply fall through if VA == PA. We need a 98 * full fence here because setup_vm() just wrote these PTEs and we need 99 * to ensure the new translations are in use. 100 */ 101 la a0, trampoline_pg_dir 102 XIP_FIXUP_OFFSET a0 103 srl a0, a0, PAGE_SHIFT 104 or a0, a0, a1 105 sfence.vma 106 csrw CSR_SATP, a0 107.align 2 1081: 109 /* Set trap vector to spin forever to help debug */ 110 la a0, .Lsecondary_park 111 csrw CSR_TVEC, a0 112 113 /* Reload the global pointer */ 114 load_global_pointer 115 116 /* 117 * Switch to kernel page tables. A full fence is necessary in order to 118 * avoid using the trampoline translations, which are only correct for 119 * the first superpage. Fetching the fence is guaranteed to work 120 * because that first superpage is translated the same way. 121 */ 122 csrw CSR_SATP, a2 123 sfence.vma 124 125 ret 126#endif /* CONFIG_MMU */ 127#ifdef CONFIG_SMP 128 .global secondary_start_sbi 129secondary_start_sbi: 130 /* Mask all interrupts */ 131 csrw CSR_IE, zero 132 csrw CSR_IP, zero 133 134 /* Load the global pointer */ 135 load_global_pointer 136 137 /* 138 * Disable FPU & VECTOR to detect illegal usage of 139 * floating point or vector in kernel space 140 */ 141 li t0, SR_FS_VS 142 csrc CSR_STATUS, t0 143 144 /* Set trap vector to spin forever to help debug */ 145 la a3, .Lsecondary_park 146 csrw CSR_TVEC, a3 147 148 /* a0 contains the hartid & a1 contains boot data */ 149 li a2, SBI_HART_BOOT_TASK_PTR_OFFSET 150 XIP_FIXUP_OFFSET a2 151 add a2, a2, a1 152 REG_L tp, (a2) 153 li a3, SBI_HART_BOOT_STACK_PTR_OFFSET 154 XIP_FIXUP_OFFSET a3 155 add a3, a3, a1 156 REG_L sp, (a3) 157 scs_load_current 158 159.Lsecondary_start_common: 160 161#ifdef CONFIG_MMU 162 /* Enable virtual memory and relocate to virtual address */ 163 la a0, swapper_pg_dir 164 XIP_FIXUP_OFFSET a0 165 call relocate_enable_mmu 166#endif 167 call .Lsetup_trap_vector 168 tail smp_callin 169#endif /* CONFIG_SMP */ 170 171.align 2 172.Lsetup_trap_vector: 173 /* Set trap vector to exception handler */ 174 la a0, handle_exception 175 csrw CSR_TVEC, a0 176 177 /* 178 * Set sup0 scratch register to 0, indicating to exception vector that 179 * we are presently executing in kernel. 180 */ 181 csrw CSR_SCRATCH, zero 182 ret 183 184.align 2 185.Lsecondary_park: 186 /* We lack SMP support or have too many harts, so park this hart */ 187 wfi 188 j .Lsecondary_park 189 190SYM_CODE_END(_start) 191 192SYM_CODE_START(_start_kernel) 193 /* Mask all interrupts */ 194 csrw CSR_IE, zero 195 csrw CSR_IP, zero 196 197#ifdef CONFIG_RISCV_M_MODE 198 /* flush the instruction cache */ 199 fence.i 200 201 /* Reset all registers except ra, a0, a1 */ 202 call reset_regs 203 204 /* 205 * Setup a PMP to permit access to all of memory. Some machines may 206 * not implement PMPs, so we set up a quick trap handler to just skip 207 * touching the PMPs on any trap. 208 */ 209 la a0, .Lpmp_done 210 csrw CSR_TVEC, a0 211 212 li a0, -1 213 csrw CSR_PMPADDR0, a0 214 li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X) 215 csrw CSR_PMPCFG0, a0 216.align 2 217.Lpmp_done: 218 219 /* 220 * The hartid in a0 is expected later on, and we have no firmware 221 * to hand it to us. 222 */ 223 csrr a0, CSR_MHARTID 224#endif /* CONFIG_RISCV_M_MODE */ 225 226 /* Load the global pointer */ 227 load_global_pointer 228 229 /* 230 * Disable FPU & VECTOR to detect illegal usage of 231 * floating point or vector in kernel space 232 */ 233 li t0, SR_FS_VS 234 csrc CSR_STATUS, t0 235 236#ifdef CONFIG_RISCV_BOOT_SPINWAIT 237 li t0, CONFIG_NR_CPUS 238 blt a0, t0, .Lgood_cores 239 tail .Lsecondary_park 240.Lgood_cores: 241 242 /* The lottery system is only required for spinwait booting method */ 243#ifndef CONFIG_XIP_KERNEL 244 /* Pick one hart to run the main boot sequence */ 245 la a3, hart_lottery 246 li a2, 1 247 amoadd.w a3, a2, (a3) 248 bnez a3, .Lsecondary_start 249 250#else 251 /* hart_lottery in flash contains a magic number */ 252 la a3, hart_lottery 253 mv a2, a3 254 XIP_FIXUP_OFFSET a2 255 XIP_FIXUP_FLASH_OFFSET a3 256 lw t1, (a3) 257 amoswap.w t0, t1, (a2) 258 /* first time here if hart_lottery in RAM is not set */ 259 beq t0, t1, .Lsecondary_start 260 261#endif /* CONFIG_XIP */ 262#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ 263 264#ifdef CONFIG_XIP_KERNEL 265 la sp, _end + THREAD_SIZE 266 XIP_FIXUP_OFFSET sp 267 mv s0, a0 268 call __copy_data 269 270 /* Restore a0 copy */ 271 mv a0, s0 272#endif 273 274#ifndef CONFIG_XIP_KERNEL 275 /* Clear BSS for flat non-ELF images */ 276 la a3, __bss_start 277 la a4, __bss_stop 278 ble a4, a3, .Lclear_bss_done 279.Lclear_bss: 280 REG_S zero, (a3) 281 add a3, a3, RISCV_SZPTR 282 blt a3, a4, .Lclear_bss 283.Lclear_bss_done: 284#endif 285 la a2, boot_cpu_hartid 286 XIP_FIXUP_OFFSET a2 287 REG_S a0, (a2) 288 289 /* Initialize page tables and relocate to virtual addresses */ 290 la tp, init_task 291 la sp, init_thread_union + THREAD_SIZE 292 XIP_FIXUP_OFFSET sp 293 addi sp, sp, -PT_SIZE_ON_STACK 294 scs_load_init_stack 295#ifdef CONFIG_BUILTIN_DTB 296 la a0, __dtb_start 297 XIP_FIXUP_OFFSET a0 298#else 299 mv a0, a1 300#endif /* CONFIG_BUILTIN_DTB */ 301 call setup_vm 302#ifdef CONFIG_MMU 303 la a0, early_pg_dir 304 XIP_FIXUP_OFFSET a0 305 call relocate_enable_mmu 306#endif /* CONFIG_MMU */ 307 308 call .Lsetup_trap_vector 309 /* Restore C environment */ 310 la tp, init_task 311 la sp, init_thread_union + THREAD_SIZE 312 addi sp, sp, -PT_SIZE_ON_STACK 313 scs_load_current 314 315#ifdef CONFIG_KASAN 316 call kasan_early_init 317#endif 318 /* Start the kernel */ 319 call soc_early_init 320 tail start_kernel 321 322#ifdef CONFIG_RISCV_BOOT_SPINWAIT 323.Lsecondary_start: 324 /* Set trap vector to spin forever to help debug */ 325 la a3, .Lsecondary_park 326 csrw CSR_TVEC, a3 327 328 slli a3, a0, LGREG 329 la a1, __cpu_spinwait_stack_pointer 330 XIP_FIXUP_OFFSET a1 331 la a2, __cpu_spinwait_task_pointer 332 XIP_FIXUP_OFFSET a2 333 add a1, a3, a1 334 add a2, a3, a2 335 336 /* 337 * This hart didn't win the lottery, so we wait for the winning hart to 338 * get far enough along the boot process that it should continue. 339 */ 340.Lwait_for_cpu_up: 341 /* FIXME: We should WFI to save some energy here. */ 342 REG_L sp, (a1) 343 REG_L tp, (a2) 344 beqz sp, .Lwait_for_cpu_up 345 beqz tp, .Lwait_for_cpu_up 346 fence 347 348 tail .Lsecondary_start_common 349#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ 350 351SYM_CODE_END(_start_kernel) 352 353#ifdef CONFIG_RISCV_M_MODE 354SYM_CODE_START_LOCAL(reset_regs) 355 li sp, 0 356 li gp, 0 357 li tp, 0 358 li t0, 0 359 li t1, 0 360 li t2, 0 361 li s0, 0 362 li s1, 0 363 li a2, 0 364 li a3, 0 365 li a4, 0 366 li a5, 0 367 li a6, 0 368 li a7, 0 369 li s2, 0 370 li s3, 0 371 li s4, 0 372 li s5, 0 373 li s6, 0 374 li s7, 0 375 li s8, 0 376 li s9, 0 377 li s10, 0 378 li s11, 0 379 li t3, 0 380 li t4, 0 381 li t5, 0 382 li t6, 0 383 csrw CSR_SCRATCH, 0 384 385#ifdef CONFIG_FPU 386 csrr t0, CSR_MISA 387 andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D) 388 beqz t0, .Lreset_regs_done_fpu 389 390 li t1, SR_FS 391 csrs CSR_STATUS, t1 392 fmv.s.x f0, zero 393 fmv.s.x f1, zero 394 fmv.s.x f2, zero 395 fmv.s.x f3, zero 396 fmv.s.x f4, zero 397 fmv.s.x f5, zero 398 fmv.s.x f6, zero 399 fmv.s.x f7, zero 400 fmv.s.x f8, zero 401 fmv.s.x f9, zero 402 fmv.s.x f10, zero 403 fmv.s.x f11, zero 404 fmv.s.x f12, zero 405 fmv.s.x f13, zero 406 fmv.s.x f14, zero 407 fmv.s.x f15, zero 408 fmv.s.x f16, zero 409 fmv.s.x f17, zero 410 fmv.s.x f18, zero 411 fmv.s.x f19, zero 412 fmv.s.x f20, zero 413 fmv.s.x f21, zero 414 fmv.s.x f22, zero 415 fmv.s.x f23, zero 416 fmv.s.x f24, zero 417 fmv.s.x f25, zero 418 fmv.s.x f26, zero 419 fmv.s.x f27, zero 420 fmv.s.x f28, zero 421 fmv.s.x f29, zero 422 fmv.s.x f30, zero 423 fmv.s.x f31, zero 424 csrw fcsr, 0 425 /* note that the caller must clear SR_FS */ 426.Lreset_regs_done_fpu: 427#endif /* CONFIG_FPU */ 428 429#ifdef CONFIG_RISCV_ISA_V 430 csrr t0, CSR_MISA 431 li t1, COMPAT_HWCAP_ISA_V 432 and t0, t0, t1 433 beqz t0, .Lreset_regs_done_vector 434 435 /* 436 * Clear vector registers and reset vcsr 437 * VLMAX has a defined value, VLEN is a constant, 438 * and this form of vsetvli is defined to set vl to VLMAX. 439 */ 440 li t1, SR_VS 441 csrs CSR_STATUS, t1 442 csrs CSR_VCSR, x0 443 vsetvli t1, x0, e8, m8, ta, ma 444 vmv.v.i v0, 0 445 vmv.v.i v8, 0 446 vmv.v.i v16, 0 447 vmv.v.i v24, 0 448 /* note that the caller must clear SR_VS */ 449.Lreset_regs_done_vector: 450#endif /* CONFIG_RISCV_ISA_V */ 451 ret 452SYM_CODE_END(reset_regs) 453#endif /* CONFIG_RISCV_M_MODE */ 454