1d2912cb1SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 2bbe88886SCatalin Marinas/* 3bbe88886SCatalin Marinas * linux/arch/arm/mm/cache-v7.S 4bbe88886SCatalin Marinas * 5bbe88886SCatalin Marinas * Copyright (C) 2001 Deep Blue Solutions Ltd. 6bbe88886SCatalin Marinas * Copyright (C) 2005 ARM Ltd. 7bbe88886SCatalin Marinas * 8bbe88886SCatalin Marinas * This is the "shell" of the ARMv7 processor support. 9bbe88886SCatalin Marinas */ 10bbe88886SCatalin Marinas#include <linux/linkage.h> 11bbe88886SCatalin Marinas#include <linux/init.h> 121036b895SLinus Walleij#include <linux/cfi_types.h> 13bbe88886SCatalin Marinas#include <asm/assembler.h> 14c5102f59SWill Deacon#include <asm/errno.h> 1532cfb1b1SCatalin Marinas#include <asm/unwind.h> 161238c4fdSFlorian Fainelli#include <asm/hardware/cache-b15-rac.h> 17bbe88886SCatalin Marinas 18bbe88886SCatalin Marinas#include "proc-macros.S" 19bbe88886SCatalin Marinas 20a2faac39SNick Desaulniers.arch armv7-a 21a2faac39SNick Desaulniers 225f41f919SMarek Szyprowski#ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND 235f41f919SMarek Szyprowski.globl icache_size 245f41f919SMarek Szyprowski .data 255f41f919SMarek Szyprowski .align 2 265f41f919SMarek Szyprowskiicache_size: 275f41f919SMarek Szyprowski .long 64 285f41f919SMarek Szyprowski .text 295f41f919SMarek Szyprowski#endif 30bbe88886SCatalin Marinas/* 31c08e20d2SDinh Nguyen * The secondary kernel init calls v7_flush_dcache_all before it enables 32c08e20d2SDinh Nguyen * the L1; however, the L1 comes out of reset in an undefined state, so 33c08e20d2SDinh Nguyen * the clean + invalidate performed by v7_flush_dcache_all causes a bunch 34c08e20d2SDinh Nguyen * of cache lines with uninitialized data and uninitialized tags to get 35c08e20d2SDinh Nguyen * written out to memory, which does really unpleasant things to the main 36c08e20d2SDinh Nguyen * processor. We fix this by performing an invalidate, rather than a 37c08e20d2SDinh Nguyen * clean + invalidate, before jumping into the kernel. 38c08e20d2SDinh Nguyen * 39f9e7a99fSArd Biesheuvel * This function needs to be called for both secondary cores startup and 40f9e7a99fSArd Biesheuvel * primary core resume procedures. 41c08e20d2SDinh Nguyen */ 42c08e20d2SDinh NguyenENTRY(v7_invalidate_l1) 43c08e20d2SDinh Nguyen mov r0, #0 44c0e50736SArd Biesheuvel mcr p15, 2, r0, c0, c0, 0 @ select L1 data cache in CSSELR 45c0e50736SArd Biesheuvel isb 46c0e50736SArd Biesheuvel mrc p15, 1, r0, c0, c0, 0 @ read cache geometry from CCSIDR 47c08e20d2SDinh Nguyen 48f9e7a99fSArd Biesheuvel movw r3, #0x3ff 49f9e7a99fSArd Biesheuvel and r3, r3, r0, lsr #3 @ 'Associativity' in CCSIDR[12:3] 50c08e20d2SDinh Nguyen clz r1, r3 @ WayShift 51f9e7a99fSArd Biesheuvel mov r2, #1 52f9e7a99fSArd Biesheuvel mov r3, r3, lsl r1 @ NumWays-1 shifted into bits [31:...] 53f9e7a99fSArd Biesheuvel movs r1, r2, lsl r1 @ #1 shifted left by same amount 54f9e7a99fSArd Biesheuvel moveq r1, #1 @ r1 needs value > 0 even if only 1 way 55f9e7a99fSArd Biesheuvel 56f9e7a99fSArd Biesheuvel and r2, r0, #0x7 57f9e7a99fSArd Biesheuvel add r2, r2, #4 @ SetShift 58f9e7a99fSArd Biesheuvel 5995731b8eSArd Biesheuvel1: movw ip, #0x7fff 6095731b8eSArd Biesheuvel and r0, ip, r0, lsr #13 @ 'NumSets' in CCSIDR[27:13] 61f9e7a99fSArd Biesheuvel 6295731b8eSArd Biesheuvel2: mov ip, r0, lsl r2 @ NumSet << SetShift 6395731b8eSArd Biesheuvel orr ip, ip, r3 @ Reg = (Temp<<WayShift)|(NumSets<<SetShift) 6495731b8eSArd Biesheuvel mcr p15, 0, ip, c7, c6, 2 65f9e7a99fSArd Biesheuvel subs r0, r0, #1 @ Set-- 66f9e7a99fSArd Biesheuvel bpl 2b 67f9e7a99fSArd Biesheuvel subs r3, r3, r1 @ Way-- 68f9e7a99fSArd Biesheuvel bcc 3f 69f9e7a99fSArd Biesheuvel mrc p15, 1, r0, c0, c0, 0 @ re-read cache geometry from CCSIDR 70f9e7a99fSArd Biesheuvel b 1b 71f9e7a99fSArd Biesheuvel3: dsb st 72c08e20d2SDinh Nguyen isb 736ebbf2ceSRussell King ret lr 74c08e20d2SDinh NguyenENDPROC(v7_invalidate_l1) 75c08e20d2SDinh Nguyen 76c08e20d2SDinh Nguyen/* 7781d11955STony Lindgren * v7_flush_icache_all() 7881d11955STony Lindgren * 7981d11955STony Lindgren * Flush the whole I-cache. 8081d11955STony Lindgren * 8181d11955STony Lindgren * Registers: 8281d11955STony Lindgren * r0 - set to 0 8381d11955STony Lindgren */ 841036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_flush_icache_all) 8581d11955STony Lindgren mov r0, #0 8681d11955STony Lindgren ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable 8781d11955STony Lindgren ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate 886ebbf2ceSRussell King ret lr 891036b895SLinus WalleijSYM_FUNC_END(v7_flush_icache_all) 9081d11955STony Lindgren 9181d11955STony Lindgren /* 92031bd879SLorenzo Pieralisi * v7_flush_dcache_louis() 93031bd879SLorenzo Pieralisi * 94031bd879SLorenzo Pieralisi * Flush the D-cache up to the Level of Unification Inner Shareable 95031bd879SLorenzo Pieralisi * 961f640552SArd Biesheuvel * Corrupted registers: r0-r6, r9-r10 97031bd879SLorenzo Pieralisi */ 98031bd879SLorenzo Pieralisi 99031bd879SLorenzo PieralisiENTRY(v7_flush_dcache_louis) 100031bd879SLorenzo Pieralisi dmb @ ensure ordering with previous memory accesses 101031bd879SLorenzo Pieralisi mrc p15, 1, r0, c0, c0, 1 @ read clidr, r0 = clidr 10247b8484eSRussell KingALT_SMP(mov r3, r0, lsr #20) @ move LoUIS into position 10347b8484eSRussell KingALT_UP( mov r3, r0, lsr #26) @ move LoUU into position 10447b8484eSRussell King ands r3, r3, #7 << 1 @ extract LoU*2 field from clidr 105d3cd451dSRussell King bne start_flush_levels @ LoU != 0, start flushing 10669155794SJon Medhurst#ifdef CONFIG_ARM_ERRATA_643719 107d3cd451dSRussell KingALT_SMP(mrc p15, 0, r2, c0, c0, 0) @ read main ID register 108d3cd451dSRussell KingALT_UP( ret lr) @ LoUU is zero, so nothing to do 109aaf4b5d9SRussell King movw r1, #:lower16:(0x410fc090 >> 4) @ ID of ARM Cortex A9 r0p? 110aaf4b5d9SRussell King movt r1, #:upper16:(0x410fc090 >> 4) 111aaf4b5d9SRussell King teq r1, r2, lsr #4 @ test for errata affected core and if so... 112d3cd451dSRussell King moveq r3, #1 << 1 @ fix LoUIS value 113d3cd451dSRussell King beq start_flush_levels @ start flushing cache levels 11469155794SJon Medhurst#endif 115d3cd451dSRussell King ret lr 116031bd879SLorenzo PieralisiENDPROC(v7_flush_dcache_louis) 117031bd879SLorenzo Pieralisi 118031bd879SLorenzo Pieralisi/* 119bbe88886SCatalin Marinas * v7_flush_dcache_all() 120bbe88886SCatalin Marinas * 121bbe88886SCatalin Marinas * Flush the whole D-cache. 122bbe88886SCatalin Marinas * 1231f640552SArd Biesheuvel * Corrupted registers: r0-r6, r9-r10 124bbe88886SCatalin Marinas * 125bbe88886SCatalin Marinas * - mm - mm_struct describing address space 126bbe88886SCatalin Marinas */ 127bbe88886SCatalin MarinasENTRY(v7_flush_dcache_all) 128c30c2f99SCatalin Marinas dmb @ ensure ordering with previous memory accesses 129bbe88886SCatalin Marinas mrc p15, 1, r0, c0, c0, 1 @ read clidr 13047b8484eSRussell King mov r3, r0, lsr #23 @ move LoC into position 13147b8484eSRussell King ands r3, r3, #7 << 1 @ extract LoC*2 from clidr 132bbe88886SCatalin Marinas beq finished @ if loc is 0, then no need to clean 133cd8b24d9SRussell Kingstart_flush_levels: 134bbe88886SCatalin Marinas mov r10, #0 @ start clean at cache level 0 1353287be8cSLorenzo Pieralisiflush_levels: 136bbe88886SCatalin Marinas add r2, r10, r10, lsr #1 @ work out 3x current cache level 137bbe88886SCatalin Marinas mov r1, r0, lsr r2 @ extract cache type bits from clidr 138bbe88886SCatalin Marinas and r1, r1, #7 @ mask of the bits for current cache only 139bbe88886SCatalin Marinas cmp r1, #2 @ see what cache we have at this level 140bbe88886SCatalin Marinas blt skip @ skip if no cache, or just i-cache 141e7289c6dSThomas Gleixner#ifdef CONFIG_PREEMPTION 1428e43a905SRabin Vincent save_and_disable_irqs_notrace r9 @ make cssr&csidr read atomic 143b46c0f74SStephen Boyd#endif 144bbe88886SCatalin Marinas mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr 145bbe88886SCatalin Marinas isb @ isb to sych the new cssr&csidr 146bbe88886SCatalin Marinas mrc p15, 1, r1, c0, c0, 0 @ read the new csidr 147e7289c6dSThomas Gleixner#ifdef CONFIG_PREEMPTION 148b46c0f74SStephen Boyd restore_irqs_notrace r9 149b46c0f74SStephen Boyd#endif 150bbe88886SCatalin Marinas and r2, r1, #7 @ extract the length of the cache lines 151bbe88886SCatalin Marinas add r2, r2, #4 @ add 4 (line length offset) 1525aca3708SRussell King movw r4, #0x3ff 153bbe88886SCatalin Marinas ands r4, r4, r1, lsr #3 @ find maximum number on the way size 154bbe88886SCatalin Marinas clz r5, r4 @ find bit position of way size increment 1551f640552SArd Biesheuvel movw r6, #0x7fff 1561f640552SArd Biesheuvel and r1, r6, r1, lsr #13 @ extract max number of the index size 1571f640552SArd Biesheuvel mov r6, #1 1581f640552SArd Biesheuvel movne r4, r4, lsl r5 @ # of ways shifted into bits [31:...] 1591f640552SArd Biesheuvel movne r6, r6, lsl r5 @ 1 shifted left by same amount 1603287be8cSLorenzo Pieralisiloop1: 1611f640552SArd Biesheuvel mov r9, r1 @ create working copy of max index 1623287be8cSLorenzo Pieralisiloop2: 1631f640552SArd Biesheuvel mov r5, r9, lsl r2 @ factor set number into r5 1641f640552SArd Biesheuvel orr r5, r5, r4 @ factor way number into r5 1651f640552SArd Biesheuvel orr r5, r5, r10 @ factor cache level into r5 1661f640552SArd Biesheuvel mcr p15, 0, r5, c7, c14, 2 @ clean & invalidate by set/way 16770f665feSLorenzo Pieralisi subs r9, r9, #1 @ decrement the index 168bbe88886SCatalin Marinas bge loop2 1691f640552SArd Biesheuvel subs r4, r4, r6 @ decrement the way 1701f640552SArd Biesheuvel bcs loop1 171bbe88886SCatalin Marinasskip: 172bbe88886SCatalin Marinas add r10, r10, #2 @ increment cache number 173bbe88886SCatalin Marinas cmp r3, r10 174779eb41cSBenjamin Gaignard#ifdef CONFIG_ARM_ERRATA_814220 175779eb41cSBenjamin Gaignard dsb 176779eb41cSBenjamin Gaignard#endif 1773287be8cSLorenzo Pieralisi bgt flush_levels 178bbe88886SCatalin Marinasfinished: 17908a7e621SMasahiro Yamada mov r10, #0 @ switch back to cache level 0 180bbe88886SCatalin Marinas mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr 1819581960aSWill Deacon dsb st 182bbe88886SCatalin Marinas isb 1836ebbf2ceSRussell King ret lr 18493ed3970SCatalin MarinasENDPROC(v7_flush_dcache_all) 185bbe88886SCatalin Marinas 186bbe88886SCatalin Marinas/* 187bbe88886SCatalin Marinas * v7_flush_cache_all() 188bbe88886SCatalin Marinas * 189bbe88886SCatalin Marinas * Flush the entire cache system. 190bbe88886SCatalin Marinas * The data cache flush is now achieved using atomic clean / invalidates 191bbe88886SCatalin Marinas * working outwards from L1 cache. This is done using Set/Way based cache 19225985edcSLucas De Marchi * maintenance instructions. 193bbe88886SCatalin Marinas * The instruction cache can still be invalidated back to the point of 194bbe88886SCatalin Marinas * unification in a single instruction. 195bbe88886SCatalin Marinas * 196bbe88886SCatalin Marinas */ 1971036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_flush_kern_cache_all) 1981f640552SArd Biesheuvel stmfd sp!, {r4-r6, r9-r10, lr} 199bbe88886SCatalin Marinas bl v7_flush_dcache_all 200bbe88886SCatalin Marinas mov r0, #0 201f00ec48fSRussell King ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable 202f00ec48fSRussell King ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate 2031f640552SArd Biesheuvel ldmfd sp!, {r4-r6, r9-r10, lr} 2046ebbf2ceSRussell King ret lr 2051036b895SLinus WalleijSYM_FUNC_END(v7_flush_kern_cache_all) 206bbe88886SCatalin Marinas 207bbe88886SCatalin Marinas /* 208031bd879SLorenzo Pieralisi * v7_flush_kern_cache_louis(void) 209031bd879SLorenzo Pieralisi * 210031bd879SLorenzo Pieralisi * Flush the data cache up to Level of Unification Inner Shareable. 211031bd879SLorenzo Pieralisi * Invalidate the I-cache to the point of unification. 212031bd879SLorenzo Pieralisi */ 2131036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_flush_kern_cache_louis) 2141f640552SArd Biesheuvel stmfd sp!, {r4-r6, r9-r10, lr} 215031bd879SLorenzo Pieralisi bl v7_flush_dcache_louis 216031bd879SLorenzo Pieralisi mov r0, #0 217031bd879SLorenzo Pieralisi ALT_SMP(mcr p15, 0, r0, c7, c1, 0) @ invalidate I-cache inner shareable 218031bd879SLorenzo Pieralisi ALT_UP(mcr p15, 0, r0, c7, c5, 0) @ I+BTB cache invalidate 2191f640552SArd Biesheuvel ldmfd sp!, {r4-r6, r9-r10, lr} 2206ebbf2ceSRussell King ret lr 2211036b895SLinus WalleijSYM_FUNC_END(v7_flush_kern_cache_louis) 222031bd879SLorenzo Pieralisi 223031bd879SLorenzo Pieralisi/* 224bbe88886SCatalin Marinas * v7_flush_cache_all() 225bbe88886SCatalin Marinas * 226bbe88886SCatalin Marinas * Flush all TLB entries in a particular address space 227bbe88886SCatalin Marinas * 228bbe88886SCatalin Marinas * - mm - mm_struct describing address space 229bbe88886SCatalin Marinas */ 2301036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_flush_user_cache_all) 2311036b895SLinus Walleij ret lr 2321036b895SLinus WalleijSYM_FUNC_END(v7_flush_user_cache_all) 233bbe88886SCatalin Marinas 234bbe88886SCatalin Marinas/* 235bbe88886SCatalin Marinas * v7_flush_cache_range(start, end, flags) 236bbe88886SCatalin Marinas * 237bbe88886SCatalin Marinas * Flush a range of TLB entries in the specified address space. 238bbe88886SCatalin Marinas * 239bbe88886SCatalin Marinas * - start - start address (may not be aligned) 240bbe88886SCatalin Marinas * - end - end address (exclusive, may not be aligned) 241bbe88886SCatalin Marinas * - flags - vm_area_struct flags describing address space 242bbe88886SCatalin Marinas * 243bbe88886SCatalin Marinas * It is assumed that: 244bbe88886SCatalin Marinas * - we have a VIPT cache. 245bbe88886SCatalin Marinas */ 2461036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_flush_user_cache_range) 2476ebbf2ceSRussell King ret lr 2481036b895SLinus WalleijSYM_FUNC_END(v7_flush_user_cache_range) 249bbe88886SCatalin Marinas 250bbe88886SCatalin Marinas/* 251bbe88886SCatalin Marinas * v7_coherent_kern_range(start,end) 252bbe88886SCatalin Marinas * 253bbe88886SCatalin Marinas * Ensure that the I and D caches are coherent within specified 254bbe88886SCatalin Marinas * region. This is typically used when code has been written to 255bbe88886SCatalin Marinas * a memory region, and will be executed. 256bbe88886SCatalin Marinas * 257bbe88886SCatalin Marinas * - start - virtual start address of region 258bbe88886SCatalin Marinas * - end - virtual end address of region 259bbe88886SCatalin Marinas * 260bbe88886SCatalin Marinas * It is assumed that: 261bbe88886SCatalin Marinas * - the Icache does not read data from the write buffer 262bbe88886SCatalin Marinas */ 2631036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_coherent_kern_range) 264*7b749aadSLinus Walleij#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ 2651036b895SLinus Walleij b v7_coherent_user_range 266*7b749aadSLinus Walleij#endif 2671036b895SLinus WalleijSYM_FUNC_END(v7_coherent_kern_range) 268bbe88886SCatalin Marinas 269bbe88886SCatalin Marinas/* 270bbe88886SCatalin Marinas * v7_coherent_user_range(start,end) 271bbe88886SCatalin Marinas * 272bbe88886SCatalin Marinas * Ensure that the I and D caches are coherent within specified 273bbe88886SCatalin Marinas * region. This is typically used when code has been written to 274bbe88886SCatalin Marinas * a memory region, and will be executed. 275bbe88886SCatalin Marinas * 276bbe88886SCatalin Marinas * - start - virtual start address of region 277bbe88886SCatalin Marinas * - end - virtual end address of region 278bbe88886SCatalin Marinas * 279bbe88886SCatalin Marinas * It is assumed that: 280bbe88886SCatalin Marinas * - the Icache does not read data from the write buffer 281bbe88886SCatalin Marinas */ 2821036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_coherent_user_range) 28332cfb1b1SCatalin Marinas UNWIND(.fnstart ) 284bbe88886SCatalin Marinas dcache_line_size r2, r3 285bbe88886SCatalin Marinas sub r3, r2, #1 286da30e0acSCatalin Marinas bic r12, r0, r3 287f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 288f630c1bdSWill Deacon ALT_SMP(W(dsb)) 289f630c1bdSWill Deacon ALT_UP(W(nop)) 290f630c1bdSWill Deacon#endif 29132cfb1b1SCatalin Marinas1: 292da30e0acSCatalin Marinas USER( mcr p15, 0, r12, c7, c11, 1 ) @ clean D line to the point of unification 293da30e0acSCatalin Marinas add r12, r12, r2 294da30e0acSCatalin Marinas cmp r12, r1 295bbe88886SCatalin Marinas blo 1b 2966abdd491SWill Deacon dsb ishst 2975f41f919SMarek Szyprowski#ifdef CONFIG_CPU_ICACHE_MISMATCH_WORKAROUND 2985f41f919SMarek Szyprowski ldr r3, =icache_size 2995f41f919SMarek Szyprowski ldr r2, [r3, #0] 3005f41f919SMarek Szyprowski#else 301da30e0acSCatalin Marinas icache_line_size r2, r3 3025f41f919SMarek Szyprowski#endif 303da30e0acSCatalin Marinas sub r3, r2, #1 304da30e0acSCatalin Marinas bic r12, r0, r3 305da30e0acSCatalin Marinas2: 306da30e0acSCatalin Marinas USER( mcr p15, 0, r12, c7, c5, 1 ) @ invalidate I line 307da30e0acSCatalin Marinas add r12, r12, r2 308da30e0acSCatalin Marinas cmp r12, r1 309da30e0acSCatalin Marinas blo 2b 310bbe88886SCatalin Marinas mov r0, #0 311f00ec48fSRussell King ALT_SMP(mcr p15, 0, r0, c7, c1, 6) @ invalidate BTB Inner Shareable 312f00ec48fSRussell King ALT_UP(mcr p15, 0, r0, c7, c5, 6) @ invalidate BTB 3136abdd491SWill Deacon dsb ishst 314bbe88886SCatalin Marinas isb 3156ebbf2ceSRussell King ret lr 31632cfb1b1SCatalin Marinas 31732cfb1b1SCatalin Marinas/* 31832cfb1b1SCatalin Marinas * Fault handling for the cache operation above. If the virtual address in r0 319c5102f59SWill Deacon * isn't mapped, fail with -EFAULT. 32032cfb1b1SCatalin Marinas */ 32132cfb1b1SCatalin Marinas9001: 3227253b85cSSimon Horman#ifdef CONFIG_ARM_ERRATA_775420 3237253b85cSSimon Horman dsb 3247253b85cSSimon Horman#endif 325c5102f59SWill Deacon mov r0, #-EFAULT 3266ebbf2ceSRussell King ret lr 32732cfb1b1SCatalin Marinas UNWIND(.fnend ) 3281036b895SLinus WalleijSYM_FUNC_END(v7_coherent_user_range) 329bbe88886SCatalin Marinas 330bbe88886SCatalin Marinas/* 3312c9b9c84SRussell King * v7_flush_kern_dcache_area(void *addr, size_t size) 332bbe88886SCatalin Marinas * 333bbe88886SCatalin Marinas * Ensure that the data held in the page kaddr is written back 334bbe88886SCatalin Marinas * to the page in question. 335bbe88886SCatalin Marinas * 3362c9b9c84SRussell King * - addr - kernel address 3372c9b9c84SRussell King * - size - region size 338bbe88886SCatalin Marinas */ 3391036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_flush_kern_dcache_area) 340bbe88886SCatalin Marinas dcache_line_size r2, r3 3412c9b9c84SRussell King add r1, r0, r1 342a248b13bSWill Deacon sub r3, r2, #1 343a248b13bSWill Deacon bic r0, r0, r3 344f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 345f630c1bdSWill Deacon ALT_SMP(W(dsb)) 346f630c1bdSWill Deacon ALT_UP(W(nop)) 347f630c1bdSWill Deacon#endif 348bbe88886SCatalin Marinas1: 349bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line / unified line 350bbe88886SCatalin Marinas add r0, r0, r2 351bbe88886SCatalin Marinas cmp r0, r1 352bbe88886SCatalin Marinas blo 1b 3539581960aSWill Deacon dsb st 3546ebbf2ceSRussell King ret lr 3551036b895SLinus WalleijSYM_FUNC_END(v7_flush_kern_dcache_area) 356bbe88886SCatalin Marinas 357bbe88886SCatalin Marinas/* 358bbe88886SCatalin Marinas * v7_dma_inv_range(start,end) 359bbe88886SCatalin Marinas * 360bbe88886SCatalin Marinas * Invalidate the data cache within the specified region; we will 361bbe88886SCatalin Marinas * be performing a DMA operation in this region and we want to 362bbe88886SCatalin Marinas * purge old data in the cache. 363bbe88886SCatalin Marinas * 364bbe88886SCatalin Marinas * - start - virtual start address of region 365bbe88886SCatalin Marinas * - end - virtual end address of region 366bbe88886SCatalin Marinas */ 367702b94bfSRussell Kingv7_dma_inv_range: 368bbe88886SCatalin Marinas dcache_line_size r2, r3 369bbe88886SCatalin Marinas sub r3, r2, #1 370bbe88886SCatalin Marinas tst r0, r3 371bbe88886SCatalin Marinas bic r0, r0, r3 372f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 373f630c1bdSWill Deacon ALT_SMP(W(dsb)) 374f630c1bdSWill Deacon ALT_UP(W(nop)) 375f630c1bdSWill Deacon#endif 376bbe88886SCatalin Marinas mcrne p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line 377a1208f6aSChris Cole addne r0, r0, r2 378bbe88886SCatalin Marinas 379bbe88886SCatalin Marinas tst r1, r3 380bbe88886SCatalin Marinas bic r1, r1, r3 381bbe88886SCatalin Marinas mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D / U line 382bbe88886SCatalin Marinas cmp r0, r1 383a1208f6aSChris Cole1: 384a1208f6aSChris Cole mcrlo p15, 0, r0, c7, c6, 1 @ invalidate D / U line 385a1208f6aSChris Cole addlo r0, r0, r2 386a1208f6aSChris Cole cmplo r0, r1 387bbe88886SCatalin Marinas blo 1b 3889581960aSWill Deacon dsb st 3896ebbf2ceSRussell King ret lr 39093ed3970SCatalin MarinasENDPROC(v7_dma_inv_range) 391bbe88886SCatalin Marinas 392bbe88886SCatalin Marinas/* 393bbe88886SCatalin Marinas * v7_dma_clean_range(start,end) 394bbe88886SCatalin Marinas * - start - virtual start address of region 395bbe88886SCatalin Marinas * - end - virtual end address of region 396bbe88886SCatalin Marinas */ 397702b94bfSRussell Kingv7_dma_clean_range: 398bbe88886SCatalin Marinas dcache_line_size r2, r3 399bbe88886SCatalin Marinas sub r3, r2, #1 400bbe88886SCatalin Marinas bic r0, r0, r3 401f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 402f630c1bdSWill Deacon ALT_SMP(W(dsb)) 403f630c1bdSWill Deacon ALT_UP(W(nop)) 404f630c1bdSWill Deacon#endif 405bbe88886SCatalin Marinas1: 406bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c10, 1 @ clean D / U line 407bbe88886SCatalin Marinas add r0, r0, r2 408bbe88886SCatalin Marinas cmp r0, r1 409bbe88886SCatalin Marinas blo 1b 4109581960aSWill Deacon dsb st 4116ebbf2ceSRussell King ret lr 41293ed3970SCatalin MarinasENDPROC(v7_dma_clean_range) 413bbe88886SCatalin Marinas 414bbe88886SCatalin Marinas/* 415bbe88886SCatalin Marinas * v7_dma_flush_range(start,end) 416bbe88886SCatalin Marinas * - start - virtual start address of region 417bbe88886SCatalin Marinas * - end - virtual end address of region 418bbe88886SCatalin Marinas */ 4191036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_dma_flush_range) 420bbe88886SCatalin Marinas dcache_line_size r2, r3 421bbe88886SCatalin Marinas sub r3, r2, #1 422bbe88886SCatalin Marinas bic r0, r0, r3 423f630c1bdSWill Deacon#ifdef CONFIG_ARM_ERRATA_764369 424f630c1bdSWill Deacon ALT_SMP(W(dsb)) 425f630c1bdSWill Deacon ALT_UP(W(nop)) 426f630c1bdSWill Deacon#endif 427bbe88886SCatalin Marinas1: 428bbe88886SCatalin Marinas mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D / U line 429bbe88886SCatalin Marinas add r0, r0, r2 430bbe88886SCatalin Marinas cmp r0, r1 431bbe88886SCatalin Marinas blo 1b 4329581960aSWill Deacon dsb st 4336ebbf2ceSRussell King ret lr 4341036b895SLinus WalleijSYM_FUNC_END(v7_dma_flush_range) 435bbe88886SCatalin Marinas 436a9c9147eSRussell King/* 437a9c9147eSRussell King * dma_map_area(start, size, dir) 438a9c9147eSRussell King * - start - kernel virtual start address 439a9c9147eSRussell King * - size - size of region 440a9c9147eSRussell King * - dir - DMA direction 441a9c9147eSRussell King */ 4421036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_dma_map_area) 443a9c9147eSRussell King add r1, r1, r0 4442ffe2da3SRussell King teq r2, #DMA_FROM_DEVICE 4452ffe2da3SRussell King beq v7_dma_inv_range 4462ffe2da3SRussell King b v7_dma_clean_range 4471036b895SLinus WalleijSYM_FUNC_END(v7_dma_map_area) 448a9c9147eSRussell King 449a9c9147eSRussell King/* 450a9c9147eSRussell King * dma_unmap_area(start, size, dir) 451a9c9147eSRussell King * - start - kernel virtual start address 452a9c9147eSRussell King * - size - size of region 453a9c9147eSRussell King * - dir - DMA direction 454a9c9147eSRussell King */ 4551036b895SLinus WalleijSYM_TYPED_FUNC_START(v7_dma_unmap_area) 4562ffe2da3SRussell King add r1, r1, r0 4572ffe2da3SRussell King teq r2, #DMA_TO_DEVICE 4582ffe2da3SRussell King bne v7_dma_inv_range 4596ebbf2ceSRussell King ret lr 4601036b895SLinus WalleijSYM_FUNC_END(v7_dma_unmap_area) 461