1d2912cb1SThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 21da177e4SLinus Torvalds/* 31da177e4SLinus Torvalds * linux/arch/arm/mm/cache-v4wb.S 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Copyright (C) 1997-2002 Russell king 61da177e4SLinus Torvalds */ 71da177e4SLinus Torvalds#include <linux/linkage.h> 81da177e4SLinus Torvalds#include <linux/init.h> 91036b895SLinus Walleij#include <linux/cfi_types.h> 106ebbf2ceSRussell King#include <asm/assembler.h> 111da177e4SLinus Torvalds#include <asm/page.h> 121da177e4SLinus Torvalds#include "proc-macros.S" 131da177e4SLinus Torvalds 141da177e4SLinus Torvalds/* 151da177e4SLinus Torvalds * The size of one data cache line. 161da177e4SLinus Torvalds */ 171da177e4SLinus Torvalds#define CACHE_DLINESIZE 32 181da177e4SLinus Torvalds 191da177e4SLinus Torvalds/* 201da177e4SLinus Torvalds * The total size of the data cache. 211da177e4SLinus Torvalds */ 221da177e4SLinus Torvalds#if defined(CONFIG_CPU_SA110) 231da177e4SLinus Torvalds# define CACHE_DSIZE 16384 241da177e4SLinus Torvalds#elif defined(CONFIG_CPU_SA1100) 251da177e4SLinus Torvalds# define CACHE_DSIZE 8192 261da177e4SLinus Torvalds#else 271da177e4SLinus Torvalds# error Unknown cache size 281da177e4SLinus Torvalds#endif 291da177e4SLinus Torvalds 301da177e4SLinus Torvalds/* 311da177e4SLinus Torvalds * This is the size at which it becomes more efficient to 321da177e4SLinus Torvalds * clean the whole cache, rather than using the individual 3325985edcSLucas De Marchi * cache line maintenance instructions. 341da177e4SLinus Torvalds * 351da177e4SLinus Torvalds * Size Clean (ticks) Dirty (ticks) 361da177e4SLinus Torvalds * 4096 21 20 21 53 55 54 371da177e4SLinus Torvalds * 8192 40 41 40 106 100 102 381da177e4SLinus Torvalds * 16384 77 77 76 140 140 138 391da177e4SLinus Torvalds * 32768 150 149 150 214 216 212 <--- 401da177e4SLinus Torvalds * 65536 296 297 296 351 358 361 411da177e4SLinus Torvalds * 131072 591 591 591 656 657 651 421da177e4SLinus Torvalds * Whole 132 136 132 221 217 207 <--- 431da177e4SLinus Torvalds */ 441da177e4SLinus Torvalds#define CACHE_DLIMIT (CACHE_DSIZE * 4) 451da177e4SLinus Torvalds 4695f3df6bSRussell King .data 471abd3502SRussell King .align 2 4895f3df6bSRussell Kingflush_base: 4995f3df6bSRussell King .long FLUSH_BASE 5095f3df6bSRussell King .text 5195f3df6bSRussell King 521da177e4SLinus Torvalds/* 53c8c90860SMika Westerberg * flush_icache_all() 54c8c90860SMika Westerberg * 55c8c90860SMika Westerberg * Unconditionally clean and invalidate the entire icache. 56c8c90860SMika Westerberg */ 571036b895SLinus WalleijSYM_TYPED_FUNC_START(v4wb_flush_icache_all) 58c8c90860SMika Westerberg mov r0, #0 59c8c90860SMika Westerberg mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache 606ebbf2ceSRussell King ret lr 611036b895SLinus WalleijSYM_FUNC_END(v4wb_flush_icache_all) 62c8c90860SMika Westerberg 63c8c90860SMika Westerberg/* 641da177e4SLinus Torvalds * flush_user_cache_all() 651da177e4SLinus Torvalds * 661da177e4SLinus Torvalds * Clean and invalidate all cache entries in a particular address 671da177e4SLinus Torvalds * space. 681da177e4SLinus Torvalds */ 692074beebSLinus WalleijSYM_FUNC_ALIAS(v4wb_flush_user_cache_all, v4wb_flush_kern_cache_all) 701036b895SLinus Walleij 711da177e4SLinus Torvalds/* 721da177e4SLinus Torvalds * flush_kern_cache_all() 731da177e4SLinus Torvalds * 741da177e4SLinus Torvalds * Clean and invalidate the entire cache. 751da177e4SLinus Torvalds */ 761036b895SLinus WalleijSYM_TYPED_FUNC_START(v4wb_flush_kern_cache_all) 771da177e4SLinus Torvalds mov ip, #0 781da177e4SLinus Torvalds mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache 791da177e4SLinus Torvalds__flush_whole_cache: 8095f3df6bSRussell King ldr r3, =flush_base 8195f3df6bSRussell King ldr r1, [r3, #0] 8295f3df6bSRussell King eor r1, r1, #CACHE_DSIZE 8395f3df6bSRussell King str r1, [r3, #0] 8495f3df6bSRussell King add r2, r1, #CACHE_DSIZE 8595f3df6bSRussell King1: ldr r3, [r1], #32 8695f3df6bSRussell King cmp r1, r2 871da177e4SLinus Torvalds blo 1b 8895f3df6bSRussell King#ifdef FLUSH_BASE_MINICACHE 8995f3df6bSRussell King add r2, r2, #FLUSH_BASE_MINICACHE - FLUSH_BASE 9095f3df6bSRussell King sub r1, r2, #512 @ only 512 bytes 9195f3df6bSRussell King1: ldr r3, [r1], #32 9295f3df6bSRussell King cmp r1, r2 9395f3df6bSRussell King blo 1b 9495f3df6bSRussell King#endif 951da177e4SLinus Torvalds mcr p15, 0, ip, c7, c10, 4 @ drain write buffer 966ebbf2ceSRussell King ret lr 971036b895SLinus WalleijSYM_FUNC_END(v4wb_flush_kern_cache_all) 981da177e4SLinus Torvalds 991da177e4SLinus Torvalds/* 1001da177e4SLinus Torvalds * flush_user_cache_range(start, end, flags) 1011da177e4SLinus Torvalds * 1021da177e4SLinus Torvalds * Invalidate a range of cache entries in the specified 1031da177e4SLinus Torvalds * address space. 1041da177e4SLinus Torvalds * 1051da177e4SLinus Torvalds * - start - start address (inclusive, page aligned) 1061da177e4SLinus Torvalds * - end - end address (exclusive, page aligned) 1071da177e4SLinus Torvalds * - flags - vma_area_struct flags describing address space 1081da177e4SLinus Torvalds */ 1091036b895SLinus WalleijSYM_TYPED_FUNC_START(v4wb_flush_user_cache_range) 11095f3df6bSRussell King mov ip, #0 1111da177e4SLinus Torvalds sub r3, r1, r0 @ calculate total size 1121da177e4SLinus Torvalds tst r2, #VM_EXEC @ executable region? 1131da177e4SLinus Torvalds mcrne p15, 0, ip, c7, c5, 0 @ invalidate I cache 1141da177e4SLinus Torvalds 1151da177e4SLinus Torvalds cmp r3, #CACHE_DLIMIT @ total size >= limit? 1161da177e4SLinus Torvalds bhs __flush_whole_cache @ flush whole D cache 1171da177e4SLinus Torvalds 1181da177e4SLinus Torvalds1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry 1191da177e4SLinus Torvalds mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry 1201da177e4SLinus Torvalds add r0, r0, #CACHE_DLINESIZE 1211da177e4SLinus Torvalds cmp r0, r1 1221da177e4SLinus Torvalds blo 1b 1231da177e4SLinus Torvalds tst r2, #VM_EXEC 1241da177e4SLinus Torvalds mcrne p15, 0, ip, c7, c10, 4 @ drain write buffer 1256ebbf2ceSRussell King ret lr 1261036b895SLinus WalleijSYM_FUNC_END(v4wb_flush_user_cache_range) 1271da177e4SLinus Torvalds 1281da177e4SLinus Torvalds/* 1292c9b9c84SRussell King * flush_kern_dcache_area(void *addr, size_t size) 1301da177e4SLinus Torvalds * 1311da177e4SLinus Torvalds * Ensure no D cache aliasing occurs, either with itself or 1321da177e4SLinus Torvalds * the I cache 1331da177e4SLinus Torvalds * 1342c9b9c84SRussell King * - addr - kernel address 1352c9b9c84SRussell King * - size - region size 1361da177e4SLinus Torvalds */ 1371036b895SLinus WalleijSYM_TYPED_FUNC_START(v4wb_flush_kern_dcache_area) 1382c9b9c84SRussell King add r1, r0, r1 139*7b749aadSLinus Walleij#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ 1401036b895SLinus Walleij b v4wb_coherent_user_range 141*7b749aadSLinus Walleij#endif 1421036b895SLinus WalleijSYM_FUNC_END(v4wb_flush_kern_dcache_area) 1431da177e4SLinus Torvalds 1441da177e4SLinus Torvalds/* 1451da177e4SLinus Torvalds * coherent_kern_range(start, end) 1461da177e4SLinus Torvalds * 1471da177e4SLinus Torvalds * Ensure coherency between the Icache and the Dcache in the 1481da177e4SLinus Torvalds * region described by start. If you have non-snooping 1491da177e4SLinus Torvalds * Harvard caches, you need to implement this function. 1501da177e4SLinus Torvalds * 1511da177e4SLinus Torvalds * - start - virtual start address 1521da177e4SLinus Torvalds * - end - virtual end address 1531da177e4SLinus Torvalds */ 1541036b895SLinus WalleijSYM_TYPED_FUNC_START(v4wb_coherent_kern_range) 155*7b749aadSLinus Walleij#ifdef CONFIG_CFI_CLANG /* Fallthrough if !CFI */ 1561036b895SLinus Walleij b v4wb_coherent_user_range 157*7b749aadSLinus Walleij#endif 1581036b895SLinus WalleijSYM_FUNC_END(v4wb_coherent_kern_range) 1591da177e4SLinus Torvalds 1601da177e4SLinus Torvalds/* 1611da177e4SLinus Torvalds * coherent_user_range(start, end) 1621da177e4SLinus Torvalds * 1631da177e4SLinus Torvalds * Ensure coherency between the Icache and the Dcache in the 1641da177e4SLinus Torvalds * region described by start. If you have non-snooping 1651da177e4SLinus Torvalds * Harvard caches, you need to implement this function. 1661da177e4SLinus Torvalds * 1671da177e4SLinus Torvalds * - start - virtual start address 1681da177e4SLinus Torvalds * - end - virtual end address 1691da177e4SLinus Torvalds */ 1701036b895SLinus WalleijSYM_TYPED_FUNC_START(v4wb_coherent_user_range) 1711da177e4SLinus Torvalds bic r0, r0, #CACHE_DLINESIZE - 1 1721da177e4SLinus Torvalds1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry 1731da177e4SLinus Torvalds mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry 1741da177e4SLinus Torvalds add r0, r0, #CACHE_DLINESIZE 1751da177e4SLinus Torvalds cmp r0, r1 1761da177e4SLinus Torvalds blo 1b 177c5102f59SWill Deacon mov r0, #0 178c5102f59SWill Deacon mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache 179c5102f59SWill Deacon mcr p15, 0, r0, c7, c10, 4 @ drain WB 1806ebbf2ceSRussell King ret lr 1811036b895SLinus WalleijSYM_FUNC_END(v4wb_coherent_user_range) 1821da177e4SLinus Torvalds 1831da177e4SLinus Torvalds/* 1841da177e4SLinus Torvalds * dma_inv_range(start, end) 1851da177e4SLinus Torvalds * 1861da177e4SLinus Torvalds * Invalidate (discard) the specified virtual address range. 1871da177e4SLinus Torvalds * May not write back any entries. If 'start' or 'end' 1881da177e4SLinus Torvalds * are not cache line aligned, those lines must be written 1891da177e4SLinus Torvalds * back. 1901da177e4SLinus Torvalds * 1911da177e4SLinus Torvalds * - start - virtual start address 1921da177e4SLinus Torvalds * - end - virtual end address 1931da177e4SLinus Torvalds */ 194702b94bfSRussell Kingv4wb_dma_inv_range: 1951da177e4SLinus Torvalds tst r0, #CACHE_DLINESIZE - 1 1961da177e4SLinus Torvalds bic r0, r0, #CACHE_DLINESIZE - 1 1971da177e4SLinus Torvalds mcrne p15, 0, r0, c7, c10, 1 @ clean D entry 1981da177e4SLinus Torvalds tst r1, #CACHE_DLINESIZE - 1 1991da177e4SLinus Torvalds mcrne p15, 0, r1, c7, c10, 1 @ clean D entry 2001da177e4SLinus Torvalds1: mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry 2011da177e4SLinus Torvalds add r0, r0, #CACHE_DLINESIZE 2021da177e4SLinus Torvalds cmp r0, r1 2031da177e4SLinus Torvalds blo 1b 2041da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2056ebbf2ceSRussell King ret lr 2061da177e4SLinus Torvalds 2071da177e4SLinus Torvalds/* 2081da177e4SLinus Torvalds * dma_clean_range(start, end) 2091da177e4SLinus Torvalds * 2101da177e4SLinus Torvalds * Clean (write back) the specified virtual address range. 2111da177e4SLinus Torvalds * 2121da177e4SLinus Torvalds * - start - virtual start address 2131da177e4SLinus Torvalds * - end - virtual end address 2141da177e4SLinus Torvalds */ 215702b94bfSRussell Kingv4wb_dma_clean_range: 2161da177e4SLinus Torvalds bic r0, r0, #CACHE_DLINESIZE - 1 2171da177e4SLinus Torvalds1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry 2181da177e4SLinus Torvalds add r0, r0, #CACHE_DLINESIZE 2191da177e4SLinus Torvalds cmp r0, r1 2201da177e4SLinus Torvalds blo 1b 2211da177e4SLinus Torvalds mcr p15, 0, r0, c7, c10, 4 @ drain write buffer 2226ebbf2ceSRussell King ret lr 2231da177e4SLinus Torvalds 2241da177e4SLinus Torvalds/* 2251da177e4SLinus Torvalds * dma_flush_range(start, end) 2261da177e4SLinus Torvalds * 2271da177e4SLinus Torvalds * Clean and invalidate the specified virtual address range. 2281da177e4SLinus Torvalds * 2291da177e4SLinus Torvalds * - start - virtual start address 2301da177e4SLinus Torvalds * - end - virtual end address 2311da177e4SLinus Torvalds * 2321da177e4SLinus Torvalds * This is actually the same as v4wb_coherent_kern_range() 2331da177e4SLinus Torvalds */ 2341da177e4SLinus Torvalds .globl v4wb_dma_flush_range 2351da177e4SLinus Torvalds .set v4wb_dma_flush_range, v4wb_coherent_kern_range 2361da177e4SLinus Torvalds 237a9c9147eSRussell King/* 238a9c9147eSRussell King * dma_map_area(start, size, dir) 239a9c9147eSRussell King * - start - kernel virtual start address 240a9c9147eSRussell King * - size - size of region 241a9c9147eSRussell King * - dir - DMA direction 242a9c9147eSRussell King */ 2431036b895SLinus WalleijSYM_TYPED_FUNC_START(v4wb_dma_map_area) 244a9c9147eSRussell King add r1, r1, r0 245a9c9147eSRussell King cmp r2, #DMA_TO_DEVICE 246a9c9147eSRussell King beq v4wb_dma_clean_range 247a9c9147eSRussell King bcs v4wb_dma_inv_range 248a9c9147eSRussell King b v4wb_dma_flush_range 2491036b895SLinus WalleijSYM_FUNC_END(v4wb_dma_map_area) 250a9c9147eSRussell King 251a9c9147eSRussell King/* 252a9c9147eSRussell King * dma_unmap_area(start, size, dir) 253a9c9147eSRussell King * - start - kernel virtual start address 254a9c9147eSRussell King * - size - size of region 255a9c9147eSRussell King * - dir - DMA direction 256a9c9147eSRussell King */ 2571036b895SLinus WalleijSYM_TYPED_FUNC_START(v4wb_dma_unmap_area) 2586ebbf2ceSRussell King ret lr 2591036b895SLinus WalleijSYM_FUNC_END(v4wb_dma_unmap_area) 260