1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * linux/arch/arm/mm/cache-v7m.S 4 * 5 * Based on linux/arch/arm/mm/cache-v7.S 6 * 7 * Copyright (C) 2001 Deep Blue Solutions Ltd. 8 * Copyright (C) 2005 ARM Ltd. 9 * 10 * This is the "shell" of the ARMv7M processor support. 11 */ 12#include <linux/linkage.h> 13#include <linux/init.h> 14#include <asm/assembler.h> 15#include <asm/errno.h> 16#include <asm/unwind.h> 17#include <asm/v7m.h> 18 19#include "proc-macros.S" 20 21/* Generic V7M read/write macros for memory mapped cache operations */ 22.macro v7m_cache_read, rt, reg 23 movw \rt, #:lower16:BASEADDR_V7M_SCB + \reg 24 movt \rt, #:upper16:BASEADDR_V7M_SCB + \reg 25 ldr \rt, [\rt] 26.endm 27 28.macro v7m_cacheop, rt, tmp, op, c = al 29 movw\c \tmp, #:lower16:BASEADDR_V7M_SCB + \op 30 movt\c \tmp, #:upper16:BASEADDR_V7M_SCB + \op 31 str\c \rt, [\tmp] 32.endm 33 34 35.macro read_ccsidr, rt 36 v7m_cache_read \rt, V7M_SCB_CCSIDR 37.endm 38 39.macro read_clidr, rt 40 v7m_cache_read \rt, V7M_SCB_CLIDR 41.endm 42 43.macro write_csselr, rt, tmp 44 v7m_cacheop \rt, \tmp, V7M_SCB_CSSELR 45.endm 46 47/* 48 * dcisw: Invalidate data cache by set/way 49 */ 50.macro dcisw, rt, tmp 51 v7m_cacheop \rt, \tmp, V7M_SCB_DCISW 52.endm 53 54/* 55 * dccisw: Clean and invalidate data cache by set/way 56 */ 57.macro dccisw, rt, tmp 58 v7m_cacheop \rt, \tmp, V7M_SCB_DCCISW 59.endm 60 61/* 62 * dccimvac: Clean and invalidate data cache line by MVA to PoC. 63 */ 64.irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo 65.macro dccimvac\c, rt, tmp 66 v7m_cacheop \rt, \tmp, V7M_SCB_DCCIMVAC, \c 67.endm 68.endr 69 70/* 71 * dcimvac: Invalidate data cache line by MVA to PoC 72 */ 73.irp c,,eq,ne,cs,cc,mi,pl,vs,vc,hi,ls,ge,lt,gt,le,hs,lo 74.macro dcimvac\c, rt, tmp 75 v7m_cacheop \rt, \tmp, V7M_SCB_DCIMVAC, \c 76.endm 77.endr 78 79/* 80 * dccmvau: Clean data cache line by MVA to PoU 81 */ 82.macro dccmvau, rt, tmp 83 v7m_cacheop \rt, \tmp, V7M_SCB_DCCMVAU 84.endm 85 86/* 87 * dccmvac: Clean data cache line by MVA to PoC 88 */ 89.macro dccmvac, rt, tmp 90 v7m_cacheop \rt, \tmp, V7M_SCB_DCCMVAC 91.endm 92 93/* 94 * icimvau: Invalidate instruction caches by MVA to PoU 95 */ 96.macro icimvau, rt, tmp 97 v7m_cacheop \rt, \tmp, V7M_SCB_ICIMVAU 98.endm 99 100/* 101 * Invalidate the icache, inner shareable if SMP, invalidate BTB for UP. 102 * rt data ignored by ICIALLU(IS), so can be used for the address 103 */ 104.macro invalidate_icache, rt 105 v7m_cacheop \rt, \rt, V7M_SCB_ICIALLU 106 mov \rt, #0 107.endm 108 109/* 110 * Invalidate the BTB, inner shareable if SMP. 111 * rt data ignored by BPIALL, so it can be used for the address 112 */ 113.macro invalidate_bp, rt 114 v7m_cacheop \rt, \rt, V7M_SCB_BPIALL 115 mov \rt, #0 116.endm 117 118ENTRY(v7m_invalidate_l1) 119 mov r0, #0 120 121 write_csselr r0, r1 122 read_ccsidr r0 123 124 movw r1, #0x7fff 125 and r2, r1, r0, lsr #13 126 127 movw r1, #0x3ff 128 129 and r3, r1, r0, lsr #3 @ NumWays - 1 130 add r2, r2, #1 @ NumSets 131 132 and r0, r0, #0x7 133 add r0, r0, #4 @ SetShift 134 135 clz r1, r3 @ WayShift 136 add r4, r3, #1 @ NumWays 1371: sub r2, r2, #1 @ NumSets-- 138 mov r3, r4 @ Temp = NumWays 1392: subs r3, r3, #1 @ Temp-- 140 mov r5, r3, lsl r1 141 mov r6, r2, lsl r0 142 orr r5, r5, r6 @ Reg = (Temp<<WayShift)|(NumSets<<SetShift) 143 dcisw r5, r6 144 bgt 2b 145 cmp r2, #0 146 bgt 1b 147 dsb st 148 isb 149 ret lr 150ENDPROC(v7m_invalidate_l1) 151 152/* 153 * v7m_flush_icache_all() 154 * 155 * Flush the whole I-cache. 156 * 157 * Registers: 158 * r0 - set to 0 159 */ 160ENTRY(v7m_flush_icache_all) 161 invalidate_icache r0 162 ret lr 163ENDPROC(v7m_flush_icache_all) 164 165/* 166 * v7m_flush_dcache_all() 167 * 168 * Flush the whole D-cache. 169 * 170 * Corrupted registers: r0-r7, r9-r11 171 */ 172ENTRY(v7m_flush_dcache_all) 173 dmb @ ensure ordering with previous memory accesses 174 read_clidr r0 175 mov r3, r0, lsr #23 @ move LoC into position 176 ands r3, r3, #7 << 1 @ extract LoC*2 from clidr 177 beq finished @ if loc is 0, then no need to clean 178start_flush_levels: 179 mov r10, #0 @ start clean at cache level 0 180flush_levels: 181 add r2, r10, r10, lsr #1 @ work out 3x current cache level 182 mov r1, r0, lsr r2 @ extract cache type bits from clidr 183 and r1, r1, #7 @ mask of the bits for current cache only 184 cmp r1, #2 @ see what cache we have at this level 185 blt skip @ skip if no cache, or just i-cache 186#ifdef CONFIG_PREEMPTION 187 save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic 188#endif 189 write_csselr r10, r1 @ set current cache level 190 isb @ isb to sych the new cssr&csidr 191 read_ccsidr r1 @ read the new csidr 192#ifdef CONFIG_PREEMPTION 193 restore_irqs_notrace r9 194#endif 195 and r2, r1, #7 @ extract the length of the cache lines 196 add r2, r2, #4 @ add 4 (line length offset) 197 movw r4, #0x3ff 198 ands r4, r4, r1, lsr #3 @ find maximum number on the way size 199 clz r5, r4 @ find bit position of way size increment 200 movw r7, #0x7fff 201 ands r7, r7, r1, lsr #13 @ extract max number of the index size 202loop1: 203 mov r9, r7 @ create working copy of max index 204loop2: 205 lsl r6, r4, r5 206 orr r11, r10, r6 @ factor way and cache number into r11 207 lsl r6, r9, r2 208 orr r11, r11, r6 @ factor index number into r11 209 dccisw r11, r6 @ clean/invalidate by set/way 210 subs r9, r9, #1 @ decrement the index 211 bge loop2 212 subs r4, r4, #1 @ decrement the way 213 bge loop1 214skip: 215 add r10, r10, #2 @ increment cache number 216 cmp r3, r10 217 bgt flush_levels 218finished: 219 mov r10, #0 @ switch back to cache level 0 220 write_csselr r10, r3 @ select current cache level in cssr 221 dsb st 222 isb 223 ret lr 224ENDPROC(v7m_flush_dcache_all) 225 226/* 227 * v7m_flush_cache_all() 228 * 229 * Flush the entire cache system. 230 * The data cache flush is now achieved using atomic clean / invalidates 231 * working outwards from L1 cache. This is done using Set/Way based cache 232 * maintenance instructions. 233 * The instruction cache can still be invalidated back to the point of 234 * unification in a single instruction. 235 * 236 */ 237ENTRY(v7m_flush_kern_cache_all) 238 stmfd sp!, {r4-r7, r9-r11, lr} 239 bl v7m_flush_dcache_all 240 invalidate_icache r0 241 ldmfd sp!, {r4-r7, r9-r11, lr} 242 ret lr 243ENDPROC(v7m_flush_kern_cache_all) 244 245/* 246 * v7m_flush_cache_all() 247 * 248 * Flush all TLB entries in a particular address space 249 * 250 * - mm - mm_struct describing address space 251 */ 252ENTRY(v7m_flush_user_cache_all) 253 /*FALLTHROUGH*/ 254 255/* 256 * v7m_flush_cache_range(start, end, flags) 257 * 258 * Flush a range of TLB entries in the specified address space. 259 * 260 * - start - start address (may not be aligned) 261 * - end - end address (exclusive, may not be aligned) 262 * - flags - vm_area_struct flags describing address space 263 * 264 * It is assumed that: 265 * - we have a VIPT cache. 266 */ 267ENTRY(v7m_flush_user_cache_range) 268 ret lr 269ENDPROC(v7m_flush_user_cache_all) 270ENDPROC(v7m_flush_user_cache_range) 271 272/* 273 * v7m_coherent_kern_range(start,end) 274 * 275 * Ensure that the I and D caches are coherent within specified 276 * region. This is typically used when code has been written to 277 * a memory region, and will be executed. 278 * 279 * - start - virtual start address of region 280 * - end - virtual end address of region 281 * 282 * It is assumed that: 283 * - the Icache does not read data from the write buffer 284 */ 285ENTRY(v7m_coherent_kern_range) 286 /* FALLTHROUGH */ 287 288/* 289 * v7m_coherent_user_range(start,end) 290 * 291 * Ensure that the I and D caches are coherent within specified 292 * region. This is typically used when code has been written to 293 * a memory region, and will be executed. 294 * 295 * - start - virtual start address of region 296 * - end - virtual end address of region 297 * 298 * It is assumed that: 299 * - the Icache does not read data from the write buffer 300 */ 301ENTRY(v7m_coherent_user_range) 302 UNWIND(.fnstart ) 303 dcache_line_size r2, r3 304 sub r3, r2, #1 305 bic r12, r0, r3 3061: 307/* 308 * We use open coded version of dccmvau otherwise USER() would 309 * point at movw instruction. 310 */ 311 dccmvau r12, r3 312 add r12, r12, r2 313 cmp r12, r1 314 blo 1b 315 dsb ishst 316 icache_line_size r2, r3 317 sub r3, r2, #1 318 bic r12, r0, r3 3192: 320 icimvau r12, r3 321 add r12, r12, r2 322 cmp r12, r1 323 blo 2b 324 invalidate_bp r0 325 dsb ishst 326 isb 327 ret lr 328 UNWIND(.fnend ) 329ENDPROC(v7m_coherent_kern_range) 330ENDPROC(v7m_coherent_user_range) 331 332/* 333 * v7m_flush_kern_dcache_area(void *addr, size_t size) 334 * 335 * Ensure that the data held in the page kaddr is written back 336 * to the page in question. 337 * 338 * - addr - kernel address 339 * - size - region size 340 */ 341ENTRY(v7m_flush_kern_dcache_area) 342 dcache_line_size r2, r3 343 add r1, r0, r1 344 sub r3, r2, #1 345 bic r0, r0, r3 3461: 347 dccimvac r0, r3 @ clean & invalidate D line / unified line 348 add r0, r0, r2 349 cmp r0, r1 350 blo 1b 351 dsb st 352 ret lr 353ENDPROC(v7m_flush_kern_dcache_area) 354 355/* 356 * v7m_dma_inv_range(start,end) 357 * 358 * Invalidate the data cache within the specified region; we will 359 * be performing a DMA operation in this region and we want to 360 * purge old data in the cache. 361 * 362 * - start - virtual start address of region 363 * - end - virtual end address of region 364 */ 365v7m_dma_inv_range: 366 dcache_line_size r2, r3 367 sub r3, r2, #1 368 tst r0, r3 369 bic r0, r0, r3 370 dccimvacne r0, r3 371 addne r0, r0, r2 372 subne r3, r2, #1 @ restore r3, corrupted by v7m's dccimvac 373 tst r1, r3 374 bic r1, r1, r3 375 dccimvacne r1, r3 376 cmp r0, r1 3771: 378 dcimvaclo r0, r3 379 addlo r0, r0, r2 380 cmplo r0, r1 381 blo 1b 382 dsb st 383 ret lr 384ENDPROC(v7m_dma_inv_range) 385 386/* 387 * v7m_dma_clean_range(start,end) 388 * - start - virtual start address of region 389 * - end - virtual end address of region 390 */ 391v7m_dma_clean_range: 392 dcache_line_size r2, r3 393 sub r3, r2, #1 394 bic r0, r0, r3 3951: 396 dccmvac r0, r3 @ clean D / U line 397 add r0, r0, r2 398 cmp r0, r1 399 blo 1b 400 dsb st 401 ret lr 402ENDPROC(v7m_dma_clean_range) 403 404/* 405 * v7m_dma_flush_range(start,end) 406 * - start - virtual start address of region 407 * - end - virtual end address of region 408 */ 409ENTRY(v7m_dma_flush_range) 410 dcache_line_size r2, r3 411 sub r3, r2, #1 412 bic r0, r0, r3 4131: 414 dccimvac r0, r3 @ clean & invalidate D / U line 415 add r0, r0, r2 416 cmp r0, r1 417 blo 1b 418 dsb st 419 ret lr 420ENDPROC(v7m_dma_flush_range) 421 422/* 423 * dma_map_area(start, size, dir) 424 * - start - kernel virtual start address 425 * - size - size of region 426 * - dir - DMA direction 427 */ 428ENTRY(v7m_dma_map_area) 429 add r1, r1, r0 430 teq r2, #DMA_FROM_DEVICE 431 beq v7m_dma_inv_range 432 b v7m_dma_clean_range 433ENDPROC(v7m_dma_map_area) 434 435/* 436 * dma_unmap_area(start, size, dir) 437 * - start - kernel virtual start address 438 * - size - size of region 439 * - dir - DMA direction 440 */ 441ENTRY(v7m_dma_unmap_area) 442 add r1, r1, r0 443 teq r2, #DMA_TO_DEVICE 444 bne v7m_dma_inv_range 445 ret lr 446ENDPROC(v7m_dma_unmap_area) 447 448 .globl v7m_flush_kern_cache_louis 449 .equ v7m_flush_kern_cache_louis, v7m_flush_kern_cache_all 450 451 __INITDATA 452 453 @ define struct cpu_cache_fns (see <asm/cacheflush.h> and proc-macros.S) 454 define_cache_functions v7m 455