1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM 4 * 5 * Created by: Nicolas Pitre, March 2012 6 * Copyright: (C) 2012-2013 Linaro Limited 7 * 8 * Refer to Documentation/arm/cluster-pm-race-avoidance.rst 9 * for details of the synchronisation algorithms used here. 10 */ 11 12#include <linux/linkage.h> 13#include <asm/mcpm.h> 14#include <asm/assembler.h> 15 16#include "vlock.h" 17 18.if MCPM_SYNC_CLUSTER_CPUS 19.error "cpus must be the first member of struct mcpm_sync_struct" 20.endif 21 22 .macro pr_dbg string 23#if defined(CONFIG_DEBUG_LL) && defined(DEBUG) 24 b 1901f 251902: .asciz "CPU" 261903: .asciz " cluster" 271904: .asciz ": \string" 28 .align 291901: adr r0, 1902b 30 bl printascii 31 mov r0, r9 32 bl printhex2 33 adr r0, 1903b 34 bl printascii 35 mov r0, r10 36 bl printhex2 37 adr r0, 1904b 38 bl printascii 39#endif 40 .endm 41 42 .arm 43 .align 44 45ENTRY(mcpm_entry_point) 46 47 ARM_BE8(setend be) 48 THUMB( badr r12, 1f ) 49 THUMB( bx r12 ) 50 THUMB( .thumb ) 511: 52 mrc p15, 0, r0, c0, c0, 5 @ MPIDR 53 ubfx r9, r0, #0, #8 @ r9 = cpu 54 ubfx r10, r0, #8, #8 @ r10 = cluster 55 mov r3, #MAX_CPUS_PER_CLUSTER 56 mla r4, r3, r10, r9 @ r4 = canonical CPU index 57 cmp r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS) 58 blo 2f 59 60 /* We didn't expect this CPU. Try to cheaply make it quiet. */ 611: wfi 62 wfe 63 b 1b 64 652: pr_dbg "kernel mcpm_entry_point\n" 66 67 /* 68 * MMU is off so we need to get to various variables in a 69 * position independent way. 70 */ 71 adr r5, 3f 72 ldmia r5, {r0, r6, r7, r8, r11} 73 add r0, r5, r0 @ r0 = mcpm_entry_early_pokes 74 add r6, r5, r6 @ r6 = mcpm_entry_vectors 75 ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys 76 add r8, r5, r8 @ r8 = mcpm_sync 77 add r11, r5, r11 @ r11 = first_man_locks 78 79 @ Perform an early poke, if any 80 add r0, r0, r4, lsl #3 81 ldmia r0, {r0, r1} 82 teq r0, #0 83 strne r1, [r0] 84 85 mov r0, #MCPM_SYNC_CLUSTER_SIZE 86 mla r8, r0, r10, r8 @ r8 = sync cluster base 87 88 @ Signal that this CPU is coming UP: 89 mov r0, #CPU_COMING_UP 90 mov r5, #MCPM_SYNC_CPU_SIZE 91 mla r5, r9, r5, r8 @ r5 = sync cpu address 92 strb r0, [r5] 93 94 @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN 95 @ state, because there is at least one active CPU (this CPU). 96 97 mov r0, #VLOCK_SIZE 98 mla r11, r0, r10, r11 @ r11 = cluster first man lock 99 mov r0, r11 100 mov r1, r9 @ cpu 101 bl vlock_trylock @ implies DMB 102 103 cmp r0, #0 @ failed to get the lock? 104 bne mcpm_setup_wait @ wait for cluster setup if so 105 106 ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] 107 cmp r0, #CLUSTER_UP @ cluster already up? 108 bne mcpm_setup @ if not, set up the cluster 109 110 @ Otherwise, release the first man lock and skip setup: 111 mov r0, r11 112 bl vlock_unlock 113 b mcpm_setup_complete 114 115mcpm_setup: 116 @ Control dependency implies strb not observable before previous ldrb. 117 118 @ Signal that the cluster is being brought up: 119 mov r0, #INBOUND_COMING_UP 120 strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] 121 dmb 122 123 @ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this 124 @ point onwards will observe INBOUND_COMING_UP and abort. 125 126 @ Wait for any previously-pending cluster teardown operations to abort 127 @ or complete: 128mcpm_teardown_wait: 129 ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] 130 cmp r0, #CLUSTER_GOING_DOWN 131 bne first_man_setup 132 wfe 133 b mcpm_teardown_wait 134 135first_man_setup: 136 dmb 137 138 @ If the outbound gave up before teardown started, skip cluster setup: 139 140 cmp r0, #CLUSTER_UP 141 beq mcpm_setup_leave 142 143 @ power_up_setup is now responsible for setting up the cluster: 144 145 cmp r7, #0 146 mov r0, #1 @ second (cluster) affinity level 147 blxne r7 @ Call power_up_setup if defined 148 dmb 149 150 mov r0, #CLUSTER_UP 151 strb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] 152 dmb 153 154mcpm_setup_leave: 155 @ Leave the cluster setup critical section: 156 157 mov r0, #INBOUND_NOT_COMING_UP 158 strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] 159 dsb st 160 sev 161 162 mov r0, r11 163 bl vlock_unlock @ implies DMB 164 b mcpm_setup_complete 165 166 @ In the contended case, non-first men wait here for cluster setup 167 @ to complete: 168mcpm_setup_wait: 169 ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] 170 cmp r0, #CLUSTER_UP 171 wfene 172 bne mcpm_setup_wait 173 dmb 174 175mcpm_setup_complete: 176 @ If a platform-specific CPU setup hook is needed, it is 177 @ called from here. 178 179 cmp r7, #0 180 mov r0, #0 @ first (CPU) affinity level 181 blxne r7 @ Call power_up_setup if defined 182 dmb 183 184 @ Mark the CPU as up: 185 186 mov r0, #CPU_UP 187 strb r0, [r5] 188 189 @ Observability order of CPU_UP and opening of the gate does not matter. 190 191mcpm_entry_gated: 192 ldr r5, [r6, r4, lsl #2] @ r5 = CPU entry vector 193 cmp r5, #0 194 wfeeq 195 beq mcpm_entry_gated 196 dmb 197 198 pr_dbg "released\n" 199 bx r5 200 201 .align 2 202 2033: .word mcpm_entry_early_pokes - . 204 .word mcpm_entry_vectors - 3b 205 .word mcpm_power_up_setup_phys - 3b 206 .word mcpm_sync - 3b 207 .word first_man_locks - 3b 208 209ENDPROC(mcpm_entry_point) 210 211 .bss 212 213 .align CACHE_WRITEBACK_ORDER 214 .type first_man_locks, #object 215first_man_locks: 216 .space VLOCK_SIZE * MAX_NR_CLUSTERS 217 .align CACHE_WRITEBACK_ORDER 218 219 .type mcpm_entry_vectors, #object 220ENTRY(mcpm_entry_vectors) 221 .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER 222 223 .type mcpm_entry_early_pokes, #object 224ENTRY(mcpm_entry_early_pokes) 225 .space 8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER 226 227 .type mcpm_power_up_setup_phys, #object 228ENTRY(mcpm_power_up_setup_phys) 229 .space 4 @ set by mcpm_sync_init() 230