1/*- 2 * Copyright 2014 Svatopluk Kraus <onwahe@gmail.com> 3 * Copyright 2014 Michal Meloun <meloun@miracle.cz> 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27#include "assym.inc" 28 29#include <machine/asm.h> 30#include <machine/asmacros.h> 31#include <machine/armreg.h> 32#include <machine/sysreg.h> 33 34#define GET_PCB(tmp) \ 35 mrc CP15_TPIDRPRW(tmp); \ 36 add tmp, tmp, #(TD_PCB) 37 38/* 39 * Define cache functions used by startup code, which counts on the fact that 40 * only r0-r3,r12 (ip) are modified and no stack space is used. These functions 41 * must be called with interrupts disabled. Moreover, these work only with 42 * caches integrated to CPU (accessible via CP15); systems with an external L2 43 * cache controller such as a PL310 need separate calls to that device driver 44 * to affect L2 caches. This is not a factor during early kernel startup, as 45 * any external L2 cache controller has not been enabled yet. 46 */ 47 48/* Invalidate D cache to PoC. (aka all cache levels)*/ 49ASENTRY_NP(dcache_inv_poc_all) 50#if __ARM_ARCH == 6 51 mcr CP15_DCIALL 52 DSB 53 bx lr 54#else 55 mrc CP15_CLIDR(r0) 56 ands r0, r0, #0x07000000 57 mov r0, r0, lsr #23 /* Get LoC 'naturally' aligned for */ 58 beq 4f /* use in the CSSELR register below */ 59 601: sub r0, #2 61 mcr CP15_CSSELR(r0) /* set cache level */ 62 isb 63 mrc CP15_CCSIDR(r0) /* read CCSIDR */ 64 65 ubfx r2, r0, #13, #15 /* get num sets - 1 from CCSIDR */ 66 ubfx r3, r0, #3, #10 /* get num ways - 1 from CCSIDR */ 67 clz r1, r3 /* number of bits to MSB of way */ 68 lsl r3, r3, r1 /* shift into position */ 69 mov ip, #1 70 lsl ip, ip, r1 /* ip now contains the way decr */ 71 72 ubfx r0, r0, #0, #3 /* get linesize from CCSIDR */ 73 add r0, r0, #4 /* apply bias */ 74 lsl r2, r2, r0 /* shift sets by log2(linesize) */ 75 add r3, r3, r2 /* merge numsets - 1 with numways - 1 */ 76 sub ip, ip, r2 /* subtract numsets - 1 from way decr */ 77 mov r1, #1 78 lsl r1, r1, r0 /* r1 now contains the set decr */ 79 mov r2, ip /* r2 now contains set way decr */ 80 81 /* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */ 822: mcr CP15_DCISW(r3) /* invalidate line */ 83 movs r0, r3 /* get current way/set */ 84 beq 3f /* at 0 means we are done */ 85 movs r0, r0, lsl #10 /* clear way bits leaving only set bits*/ 86 subne r3, r3, r1 /* non-zero?, decrement set */ 87 subeq r3, r3, r2 /* zero?, decrement way and restore set count */ 88 b 2b 89 903: 91 mrc CP15_CSSELR(r0) /* get cache level */ 92 teq r0, #0 93 bne 1b 94 954: dsb /* wait for stores to finish */ 96 mov r0, #0 97 mcr CP15_CSSELR(r0) 98 isb 99 bx lr 100#endif /* __ARM_ARCH == 6 */ 101END(dcache_inv_poc_all) 102 103/* Invalidate D cache to PoU. (aka L1 cache only)*/ 104ASENTRY_NP(dcache_inv_pou_all) 105#if __ARM_ARCH == 6 106 mcr CP15_DCIALL 107 DSB 108 bx lr 109#else 110 mrc CP15_CLIDR(r0) 111 ands r0, r0, #0x38000000 112 mov r0, r0, lsr #26 /* Get LoUU (naturally aligned) */ 113 beq 4f 114 1151: sub r0, #2 116 mcr CP15_CSSELR(r0) /* set cache level */ 117 isb 118 mrc CP15_CCSIDR(r0) /* read CCSIDR */ 119 120 ubfx r2, r0, #13, #15 /* get num sets - 1 from CCSIDR */ 121 ubfx r3, r0, #3, #10 /* get num ways - 1 from CCSIDR */ 122 clz r1, r3 /* number of bits to MSB of way */ 123 lsl r3, r3, r1 /* shift into position */ 124 mov ip, #1 125 lsl ip, ip, r1 /* ip now contains the way decr */ 126 127 ubfx r0, r0, #0, #3 /* get linesize from CCSIDR */ 128 add r0, r0, #4 /* apply bias */ 129 lsl r2, r2, r0 /* shift sets by log2(linesize) */ 130 add r3, r3, r2 /* merge numsets - 1 with numways - 1 */ 131 sub ip, ip, r2 /* subtract numsets - 1 from way decr */ 132 mov r1, #1 133 lsl r1, r1, r0 /* r1 now contains the set decr */ 134 mov r2, ip /* r2 now contains set way decr */ 135 136 /* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */ 1372: mcr CP15_DCISW(r3) /* invalidate line */ 138 movs r0, r3 /* get current way/set */ 139 beq 3f /* at 0 means we are done */ 140 movs r0, r0, lsl #10 /* clear way bits leaving only set bits*/ 141 subne r3, r3, r1 /* non-zero?, decrement set */ 142 subeq r3, r3, r2 /* zero?, decrement way and restore set count */ 143 b 2b 144 1453: 146 mrc CP15_CSSELR(r0) /* get cache level */ 147 teq r0, #0 148 bne 1b 149 1504: dsb /* wait for stores to finish */ 151 mov r0, #0 152 mcr CP15_CSSELR(r0) 153 bx lr 154#endif 155END(dcache_inv_pou_all) 156 157/* Write back and Invalidate D cache to PoC. */ 158ASENTRY_NP(dcache_wbinv_poc_all) 159#if __ARM_ARCH == 6 160 mcr CP15_DCCIALL 161 DSB 162 bx lr 163#else 164 mrc CP15_CLIDR(r0) 165 ands r0, r0, #0x07000000 166 beq 4f 167 mov r0, #0 /* Clean from inner to outer levels */ 168 1691: mcr CP15_CSSELR(r0) /* set cache level */ 170 isb 171 mrc CP15_CCSIDR(r0) /* read CCSIDR */ 172 173 ubfx r2, r0, #13, #15 /* get num sets - 1 from CCSIDR */ 174 ubfx r3, r0, #3, #10 /* get num ways - 1 from CCSIDR */ 175 clz r1, r3 /* number of bits to MSB of way */ 176 lsl r3, r3, r1 /* shift into position */ 177 mov ip, #1 178 lsl ip, ip, r1 /* ip now contains the way decr */ 179 180 ubfx r0, r0, #0, #3 /* get linesize from CCSIDR */ 181 add r0, r0, #4 /* apply bias */ 182 lsl r2, r2, r0 /* shift sets by log2(linesize) */ 183 add r3, r3, r2 /* merge numsets - 1 with numways - 1 */ 184 sub ip, ip, r2 /* subtract numsets - 1 from way decr */ 185 mov r1, #1 186 lsl r1, r1, r0 /* r1 now contains the set decr */ 187 mov r2, ip /* r2 now contains set way decr */ 188 189 /* r3 = ways/sets, r2 = way decr, r1 = set decr, r0 and ip are free */ 1902: mcr CP15_DCCISW(r3) /* clean and invalidate line */ 191 movs r0, r3 /* get current way/set */ 192 beq 3f /* at 0 means we are done */ 193 movs r0, r0, lsl #10 /* clear way bits leaving only set bits*/ 194 subne r3, r3, r1 /* non-zero?, decrement set */ 195 subeq r3, r3, r2 /* zero?, decrement way and restore set count */ 196 b 2b 197 1983: 199 mrc CP15_CSSELR(r0) /* get cache level */ 200 add r0, r0, #2 /* next level */ 201 mrc CP15_CLIDR(r1) 202 ands r1, r1, #0x07000000 203 mov r1, r1, lsr #23 /* Get LoC (naturally aligned) */ 204 cmp r1, r0 205 bne 1b 206 2074: dsb /* wait for stores to finish */ 208 mov r0, #0 209 mcr CP15_CSSELR(r0) 210 bx lr 211#endif /* __ARM_ARCH == 6 */ 212END(dcache_wbinv_poc_all) 213 214ASENTRY_NP(dcache_wb_pou_checked) 215 ldr ip, .Lcpuinfo 216 ldr ip, [ip, #DCACHE_LINE_SIZE] 217 218 GET_PCB(r2) 219 ldr r2, [r2] 220 221 adr r3, _C_LABEL(cachebailout) 222 str r3, [r2, #PCB_ONFAULT] 2231: 224 mcr CP15_DCCMVAC(r0) 225 add r0, r0, ip 226 subs r1, r1, ip 227 bhi 1b 228 DSB 229 mov r0, #0 230 str r0, [r2, #PCB_ONFAULT] 231 mov r0, #1 /* cannot be faulting address */ 232 RET 233 234.Lcpuinfo: 235 .word cpuinfo 236END(dcache_wb_pou_checked) 237 238ASENTRY_NP(icache_inv_pou_checked) 239 ldr ip, .Lcpuinfo 240 ldr ip, [ip, #ICACHE_LINE_SIZE] 241 242 GET_PCB(r2) 243 ldr r2, [r2] 244 245 adr r3, _C_LABEL(cachebailout) 246 str r3, [r2, #PCB_ONFAULT] 247 2481: 249 mcr CP15_ICIMVAU(r0) 250 add r0, r0, ip 251 subs r1, r1, ip 252 bhi 1b 253 DSB 254 ISB 255 mov r0, #0 256 str r0, [r2, #PCB_ONFAULT] 257 mov r0, #1 /* cannot be faulting address */ 258 RET 259END(icache_inv_pou_checked) 260 261/* label must be global as trap-v6.c references it */ 262 .global _C_LABEL(cachebailout) 263_C_LABEL(cachebailout): 264 DSB 265 ISB 266 mov r1, #0 267 str r1, [r2, #PCB_ONFAULT] 268 RET 269