1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6#include <asm/asm-offsets.h> 7#include <asm/asm.h> 8#include <linux/init.h> 9#include <linux/linkage.h> 10#include <asm/thread_info.h> 11#include <asm/page.h> 12#include <asm/pgtable.h> 13#include <asm/csr.h> 14#include <asm/hwcap.h> 15#include <asm/image.h> 16#include <asm/scs.h> 17#include <asm/xip_fixup.h> 18#include "efi-header.S" 19 20__HEAD 21SYM_CODE_START(_start) 22 /* 23 * Image header expected by Linux boot-loaders. The image header data 24 * structure is described in asm/image.h. 25 * Do not modify it without modifying the structure and all bootloaders 26 * that expects this header format!! 27 */ 28#ifdef CONFIG_EFI 29 /* 30 * This instruction decodes to "MZ" ASCII required by UEFI. 31 */ 32 c.li s4,-13 33 j _start_kernel 34#else 35 /* jump to start kernel */ 36 j _start_kernel 37 /* reserved */ 38 .word 0 39#endif 40 .balign 8 41#ifdef CONFIG_RISCV_M_MODE 42 /* Image load offset (0MB) from start of RAM for M-mode */ 43 .dword 0 44#else 45#if __riscv_xlen == 64 46 /* Image load offset(2MB) from start of RAM */ 47 .dword 0x200000 48#else 49 /* Image load offset(4MB) from start of RAM */ 50 .dword 0x400000 51#endif 52#endif 53 /* Effective size of kernel image */ 54 .dword _end - _start 55 .dword __HEAD_FLAGS 56 .word RISCV_HEADER_VERSION 57 .word 0 58 .dword 0 59 .ascii RISCV_IMAGE_MAGIC 60 .balign 4 61 .ascii RISCV_IMAGE_MAGIC2 62#ifdef CONFIG_EFI 63 .word pe_head_start - _start 64pe_head_start: 65 66 __EFI_PE_HEADER 67#else 68 .word 0 69#endif 70 71.align 2 72#ifdef CONFIG_MMU 73 .global relocate_enable_mmu 74relocate_enable_mmu: 75 /* Relocate return address */ 76 la a1, kernel_map 77 XIP_FIXUP_OFFSET a1 78 REG_L a1, KERNEL_MAP_VIRT_ADDR(a1) 79 la a2, _start 80 sub a1, a1, a2 81 add ra, ra, a1 82 83 /* Point stvec to virtual address of intruction after satp write */ 84 la a2, 1f 85 add a2, a2, a1 86 csrw CSR_TVEC, a2 87 88 /* Compute satp for kernel page tables, but don't load it yet */ 89 srl a2, a0, PAGE_SHIFT 90 la a1, satp_mode 91 XIP_FIXUP_OFFSET a1 92 REG_L a1, 0(a1) 93 or a2, a2, a1 94 95 /* 96 * Load trampoline page directory, which will cause us to trap to 97 * stvec if VA != PA, or simply fall through if VA == PA. We need a 98 * full fence here because setup_vm() just wrote these PTEs and we need 99 * to ensure the new translations are in use. 100 */ 101 la a0, trampoline_pg_dir 102 XIP_FIXUP_OFFSET a0 103 srl a0, a0, PAGE_SHIFT 104 or a0, a0, a1 105 sfence.vma 106 csrw CSR_SATP, a0 107.align 2 1081: 109 /* Set trap vector to spin forever to help debug */ 110 la a0, .Lsecondary_park 111 csrw CSR_TVEC, a0 112 113 /* Reload the global pointer */ 114 load_global_pointer 115 116 /* 117 * Switch to kernel page tables. A full fence is necessary in order to 118 * avoid using the trampoline translations, which are only correct for 119 * the first superpage. Fetching the fence is guaranteed to work 120 * because that first superpage is translated the same way. 121 */ 122 csrw CSR_SATP, a2 123 sfence.vma 124 125 ret 126#endif /* CONFIG_MMU */ 127#ifdef CONFIG_SMP 128 .global secondary_start_sbi 129secondary_start_sbi: 130 /* Mask all interrupts */ 131 csrw CSR_IE, zero 132 csrw CSR_IP, zero 133 134 /* Load the global pointer */ 135 load_global_pointer 136 137 /* 138 * Disable FPU & VECTOR to detect illegal usage of 139 * floating point or vector in kernel space 140 */ 141 li t0, SR_FS_VS 142 csrc CSR_STATUS, t0 143 144 /* Set trap vector to spin forever to help debug */ 145 la a3, .Lsecondary_park 146 csrw CSR_TVEC, a3 147 148 /* a0 contains the hartid & a1 contains boot data */ 149 li a2, SBI_HART_BOOT_TASK_PTR_OFFSET 150 XIP_FIXUP_OFFSET a2 151 add a2, a2, a1 152 REG_L tp, (a2) 153 li a3, SBI_HART_BOOT_STACK_PTR_OFFSET 154 XIP_FIXUP_OFFSET a3 155 add a3, a3, a1 156 REG_L sp, (a3) 157 158.Lsecondary_start_common: 159 160#ifdef CONFIG_MMU 161 /* Enable virtual memory and relocate to virtual address */ 162 la a0, swapper_pg_dir 163 XIP_FIXUP_OFFSET a0 164 call relocate_enable_mmu 165#endif 166 call .Lsetup_trap_vector 167 scs_load_current 168 tail smp_callin 169#endif /* CONFIG_SMP */ 170 171.align 2 172.Lsetup_trap_vector: 173 /* Set trap vector to exception handler */ 174 la a0, handle_exception 175 csrw CSR_TVEC, a0 176 177 /* 178 * Set sup0 scratch register to 0, indicating to exception vector that 179 * we are presently executing in kernel. 180 */ 181 csrw CSR_SCRATCH, zero 182 ret 183 184.align 2 185.Lsecondary_park: 186 /* We lack SMP support or have too many harts, so park this hart */ 187 wfi 188 j .Lsecondary_park 189 190SYM_CODE_END(_start) 191 192SYM_CODE_START(_start_kernel) 193 /* Mask all interrupts */ 194 csrw CSR_IE, zero 195 csrw CSR_IP, zero 196 197#ifdef CONFIG_RISCV_M_MODE 198 /* flush the instruction cache */ 199 fence.i 200 201 /* Reset all registers except ra, a0, a1 */ 202 call reset_regs 203 204 /* 205 * Setup a PMP to permit access to all of memory. Some machines may 206 * not implement PMPs, so we set up a quick trap handler to just skip 207 * touching the PMPs on any trap. 208 */ 209 la a0, .Lpmp_done 210 csrw CSR_TVEC, a0 211 212 li a0, -1 213 csrw CSR_PMPADDR0, a0 214 li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X) 215 csrw CSR_PMPCFG0, a0 216.align 2 217.Lpmp_done: 218 219 /* 220 * The hartid in a0 is expected later on, and we have no firmware 221 * to hand it to us. 222 */ 223 csrr a0, CSR_MHARTID 224#endif /* CONFIG_RISCV_M_MODE */ 225 226 /* Load the global pointer */ 227 load_global_pointer 228 229 /* 230 * Disable FPU & VECTOR to detect illegal usage of 231 * floating point or vector in kernel space 232 */ 233 li t0, SR_FS_VS 234 csrc CSR_STATUS, t0 235 236#ifdef CONFIG_RISCV_BOOT_SPINWAIT 237 li t0, CONFIG_NR_CPUS 238 blt a0, t0, .Lgood_cores 239 tail .Lsecondary_park 240.Lgood_cores: 241 242 /* The lottery system is only required for spinwait booting method */ 243#ifndef CONFIG_XIP_KERNEL 244 /* Pick one hart to run the main boot sequence */ 245 la a3, hart_lottery 246 li a2, 1 247 amoadd.w a3, a2, (a3) 248 bnez a3, .Lsecondary_start 249 250#else 251 /* hart_lottery in flash contains a magic number */ 252 la a3, hart_lottery 253 mv a2, a3 254 XIP_FIXUP_OFFSET a2 255 XIP_FIXUP_FLASH_OFFSET a3 256 lw t1, (a3) 257 amoswap.w t0, t1, (a2) 258 /* first time here if hart_lottery in RAM is not set */ 259 beq t0, t1, .Lsecondary_start 260 261#endif /* CONFIG_XIP */ 262#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ 263 264#ifdef CONFIG_XIP_KERNEL 265 la sp, _end + THREAD_SIZE 266 XIP_FIXUP_OFFSET sp 267 mv s0, a0 268 mv s1, a1 269 call __copy_data 270 271 /* Restore a0 & a1 copy */ 272 mv a0, s0 273 mv a1, s1 274#endif 275 276#ifndef CONFIG_XIP_KERNEL 277 /* Clear BSS for flat non-ELF images */ 278 la a3, __bss_start 279 la a4, __bss_stop 280 ble a4, a3, .Lclear_bss_done 281.Lclear_bss: 282 REG_S zero, (a3) 283 add a3, a3, RISCV_SZPTR 284 blt a3, a4, .Lclear_bss 285.Lclear_bss_done: 286#endif 287 la a2, boot_cpu_hartid 288 XIP_FIXUP_OFFSET a2 289 REG_S a0, (a2) 290 291 /* Initialize page tables and relocate to virtual addresses */ 292 la tp, init_task 293 la sp, init_thread_union + THREAD_SIZE 294 XIP_FIXUP_OFFSET sp 295 addi sp, sp, -PT_SIZE_ON_STACK 296 scs_load_init_stack 297#ifdef CONFIG_BUILTIN_DTB 298 la a0, __dtb_start 299 XIP_FIXUP_OFFSET a0 300#else 301 mv a0, a1 302#endif /* CONFIG_BUILTIN_DTB */ 303 call setup_vm 304#ifdef CONFIG_MMU 305 la a0, early_pg_dir 306 XIP_FIXUP_OFFSET a0 307 call relocate_enable_mmu 308#endif /* CONFIG_MMU */ 309 310 call .Lsetup_trap_vector 311 /* Restore C environment */ 312 la tp, init_task 313 la sp, init_thread_union + THREAD_SIZE 314 addi sp, sp, -PT_SIZE_ON_STACK 315 scs_load_current 316 317#ifdef CONFIG_KASAN 318 call kasan_early_init 319#endif 320 /* Start the kernel */ 321 call soc_early_init 322 tail start_kernel 323 324#ifdef CONFIG_RISCV_BOOT_SPINWAIT 325.Lsecondary_start: 326 /* Set trap vector to spin forever to help debug */ 327 la a3, .Lsecondary_park 328 csrw CSR_TVEC, a3 329 330 slli a3, a0, LGREG 331 la a1, __cpu_spinwait_stack_pointer 332 XIP_FIXUP_OFFSET a1 333 la a2, __cpu_spinwait_task_pointer 334 XIP_FIXUP_OFFSET a2 335 add a1, a3, a1 336 add a2, a3, a2 337 338 /* 339 * This hart didn't win the lottery, so we wait for the winning hart to 340 * get far enough along the boot process that it should continue. 341 */ 342.Lwait_for_cpu_up: 343 /* FIXME: We should WFI to save some energy here. */ 344 REG_L sp, (a1) 345 REG_L tp, (a2) 346 beqz sp, .Lwait_for_cpu_up 347 beqz tp, .Lwait_for_cpu_up 348 fence 349 350 tail .Lsecondary_start_common 351#endif /* CONFIG_RISCV_BOOT_SPINWAIT */ 352 353SYM_CODE_END(_start_kernel) 354 355#ifdef CONFIG_RISCV_M_MODE 356SYM_CODE_START_LOCAL(reset_regs) 357 li sp, 0 358 li gp, 0 359 li tp, 0 360 li t0, 0 361 li t1, 0 362 li t2, 0 363 li s0, 0 364 li s1, 0 365 li a2, 0 366 li a3, 0 367 li a4, 0 368 li a5, 0 369 li a6, 0 370 li a7, 0 371 li s2, 0 372 li s3, 0 373 li s4, 0 374 li s5, 0 375 li s6, 0 376 li s7, 0 377 li s8, 0 378 li s9, 0 379 li s10, 0 380 li s11, 0 381 li t3, 0 382 li t4, 0 383 li t5, 0 384 li t6, 0 385 csrw CSR_SCRATCH, 0 386 387#ifdef CONFIG_FPU 388 csrr t0, CSR_MISA 389 andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D) 390 beqz t0, .Lreset_regs_done_fpu 391 392 li t1, SR_FS 393 csrs CSR_STATUS, t1 394 fmv.s.x f0, zero 395 fmv.s.x f1, zero 396 fmv.s.x f2, zero 397 fmv.s.x f3, zero 398 fmv.s.x f4, zero 399 fmv.s.x f5, zero 400 fmv.s.x f6, zero 401 fmv.s.x f7, zero 402 fmv.s.x f8, zero 403 fmv.s.x f9, zero 404 fmv.s.x f10, zero 405 fmv.s.x f11, zero 406 fmv.s.x f12, zero 407 fmv.s.x f13, zero 408 fmv.s.x f14, zero 409 fmv.s.x f15, zero 410 fmv.s.x f16, zero 411 fmv.s.x f17, zero 412 fmv.s.x f18, zero 413 fmv.s.x f19, zero 414 fmv.s.x f20, zero 415 fmv.s.x f21, zero 416 fmv.s.x f22, zero 417 fmv.s.x f23, zero 418 fmv.s.x f24, zero 419 fmv.s.x f25, zero 420 fmv.s.x f26, zero 421 fmv.s.x f27, zero 422 fmv.s.x f28, zero 423 fmv.s.x f29, zero 424 fmv.s.x f30, zero 425 fmv.s.x f31, zero 426 csrw fcsr, 0 427 /* note that the caller must clear SR_FS */ 428.Lreset_regs_done_fpu: 429#endif /* CONFIG_FPU */ 430 431#ifdef CONFIG_RISCV_ISA_V 432 csrr t0, CSR_MISA 433 li t1, COMPAT_HWCAP_ISA_V 434 and t0, t0, t1 435 beqz t0, .Lreset_regs_done_vector 436 437 /* 438 * Clear vector registers and reset vcsr 439 * VLMAX has a defined value, VLEN is a constant, 440 * and this form of vsetvli is defined to set vl to VLMAX. 441 */ 442 li t1, SR_VS 443 csrs CSR_STATUS, t1 444 csrs CSR_VCSR, x0 445 vsetvli t1, x0, e8, m8, ta, ma 446 vmv.v.i v0, 0 447 vmv.v.i v8, 0 448 vmv.v.i v16, 0 449 vmv.v.i v24, 0 450 /* note that the caller must clear SR_VS */ 451.Lreset_regs_done_vector: 452#endif /* CONFIG_RISCV_ISA_V */ 453 ret 454SYM_CODE_END(reset_regs) 455#endif /* CONFIG_RISCV_M_MODE */ 456