1384740dcSRalf Baechle /*
2384740dcSRalf Baechle * This file is subject to the terms and conditions of the GNU General Public
3384740dcSRalf Baechle * License. See the file "COPYING" in the main directory of this archive
4384740dcSRalf Baechle * for more details.
5384740dcSRalf Baechle *
6384740dcSRalf Baechle * Inline assembly cache operations.
7384740dcSRalf Baechle *
879add627SJustin P. Mattock * Copyright (C) 1996 David S. Miller (davem@davemloft.net)
9384740dcSRalf Baechle * Copyright (C) 1997 - 2002 Ralf Baechle (ralf@gnu.org)
10384740dcSRalf Baechle * Copyright (C) 2004 Ralf Baechle (ralf@linux-mips.org)
11384740dcSRalf Baechle */
12384740dcSRalf Baechle #ifndef _ASM_R4KCACHE_H
13384740dcSRalf Baechle #define _ASM_R4KCACHE_H
14384740dcSRalf Baechle
15f6b39ae6SMarkos Chandras #include <linux/stringify.h>
16f6b39ae6SMarkos Chandras
17384740dcSRalf Baechle #include <asm/asm.h>
186baaeadaSPaul Burton #include <asm/asm-eva.h>
19384740dcSRalf Baechle #include <asm/cacheops.h>
20934c7923SMarkos Chandras #include <asm/compiler.h>
21384740dcSRalf Baechle #include <asm/cpu-features.h>
2214bd8c08SRalf Baechle #include <asm/cpu-type.h>
23384740dcSRalf Baechle #include <asm/mipsmtregs.h>
24bb53fdf3SHuacai Chen #include <asm/mmzone.h>
256baaeadaSPaul Burton #include <asm/unroll.h>
26384740dcSRalf Baechle
27*66445677SArnd Bergmann extern void r5k_sc_init(void);
28*66445677SArnd Bergmann extern void rm7k_sc_init(void);
29*66445677SArnd Bergmann extern int mips_sc_init(void);
30*66445677SArnd Bergmann
31d116e812SDeng-Cheng Zhu extern void (*r4k_blast_dcache)(void);
32d116e812SDeng-Cheng Zhu extern void (*r4k_blast_icache)(void);
33d116e812SDeng-Cheng Zhu
34384740dcSRalf Baechle /*
35384740dcSRalf Baechle * This macro return a properly sign-extended address suitable as base address
36384740dcSRalf Baechle * for indexed cache operations. Two issues here:
37384740dcSRalf Baechle *
38384740dcSRalf Baechle * - The MIPS32 and MIPS64 specs permit an implementation to directly derive
39384740dcSRalf Baechle * the index bits from the virtual address. This breaks with tradition
40384740dcSRalf Baechle * set by the R4000. To keep unpleasant surprises from happening we pick
41384740dcSRalf Baechle * an address in KSEG0 / CKSEG0.
42384740dcSRalf Baechle * - We need a properly sign extended address for 64-bit code. To get away
43384740dcSRalf Baechle * without ifdefs we let the compiler do it by a type cast.
44384740dcSRalf Baechle */
45384740dcSRalf Baechle #define INDEX_BASE CKSEG0
46384740dcSRalf Baechle
476baaeadaSPaul Burton #define _cache_op(insn, op, addr) \
48384740dcSRalf Baechle __asm__ __volatile__( \
49384740dcSRalf Baechle " .set push \n" \
50384740dcSRalf Baechle " .set noreorder \n" \
51934c7923SMarkos Chandras " .set "MIPS_ISA_ARCH_LEVEL" \n" \
526baaeadaSPaul Burton " " insn("%0", "%1") " \n" \
53384740dcSRalf Baechle " .set pop \n" \
54384740dcSRalf Baechle : \
55384740dcSRalf Baechle : "i" (op), "R" (*(unsigned char *)(addr)))
56384740dcSRalf Baechle
576baaeadaSPaul Burton #define cache_op(op, addr) \
586baaeadaSPaul Burton _cache_op(kernel_cache, op, addr)
596baaeadaSPaul Burton
flush_icache_line_indexed(unsigned long addr)60384740dcSRalf Baechle static inline void flush_icache_line_indexed(unsigned long addr)
61384740dcSRalf Baechle {
62384740dcSRalf Baechle cache_op(Index_Invalidate_I, addr);
63384740dcSRalf Baechle }
64384740dcSRalf Baechle
flush_dcache_line_indexed(unsigned long addr)65384740dcSRalf Baechle static inline void flush_dcache_line_indexed(unsigned long addr)
66384740dcSRalf Baechle {
67384740dcSRalf Baechle cache_op(Index_Writeback_Inv_D, addr);
68384740dcSRalf Baechle }
69384740dcSRalf Baechle
flush_scache_line_indexed(unsigned long addr)70384740dcSRalf Baechle static inline void flush_scache_line_indexed(unsigned long addr)
71384740dcSRalf Baechle {
72384740dcSRalf Baechle cache_op(Index_Writeback_Inv_SD, addr);
73384740dcSRalf Baechle }
74384740dcSRalf Baechle
flush_icache_line(unsigned long addr)75384740dcSRalf Baechle static inline void flush_icache_line(unsigned long addr)
76384740dcSRalf Baechle {
7714bd8c08SRalf Baechle switch (boot_cpu_type()) {
78268a2d60SJiaxun Yang case CPU_LOONGSON2EF:
79bad009feSHuacai Chen cache_op(Hit_Invalidate_I_Loongson2, addr);
8014bd8c08SRalf Baechle break;
8114bd8c08SRalf Baechle
8214bd8c08SRalf Baechle default:
83384740dcSRalf Baechle cache_op(Hit_Invalidate_I, addr);
8414bd8c08SRalf Baechle break;
8514bd8c08SRalf Baechle }
86384740dcSRalf Baechle }
87384740dcSRalf Baechle
flush_dcache_line(unsigned long addr)88384740dcSRalf Baechle static inline void flush_dcache_line(unsigned long addr)
89384740dcSRalf Baechle {
90384740dcSRalf Baechle cache_op(Hit_Writeback_Inv_D, addr);
91384740dcSRalf Baechle }
92384740dcSRalf Baechle
invalidate_dcache_line(unsigned long addr)93384740dcSRalf Baechle static inline void invalidate_dcache_line(unsigned long addr)
94384740dcSRalf Baechle {
95384740dcSRalf Baechle cache_op(Hit_Invalidate_D, addr);
96384740dcSRalf Baechle }
97384740dcSRalf Baechle
invalidate_scache_line(unsigned long addr)98384740dcSRalf Baechle static inline void invalidate_scache_line(unsigned long addr)
99384740dcSRalf Baechle {
100384740dcSRalf Baechle cache_op(Hit_Invalidate_SD, addr);
101384740dcSRalf Baechle }
102384740dcSRalf Baechle
flush_scache_line(unsigned long addr)103384740dcSRalf Baechle static inline void flush_scache_line(unsigned long addr)
104384740dcSRalf Baechle {
105384740dcSRalf Baechle cache_op(Hit_Writeback_Inv_SD, addr);
106384740dcSRalf Baechle }
107384740dcSRalf Baechle
108f1b0bf57SThomas Bogendoerfer #ifdef CONFIG_EVA
109f1b0bf57SThomas Bogendoerfer
110384740dcSRalf Baechle #define protected_cache_op(op, addr) \
1117170bdc7SJames Hogan ({ \
1127170bdc7SJames Hogan int __err = 0; \
113384740dcSRalf Baechle __asm__ __volatile__( \
114384740dcSRalf Baechle " .set push \n" \
115384740dcSRalf Baechle " .set noreorder \n" \
116a8053854SLeonid Yegoshin " .set mips0 \n" \
117a8053854SLeonid Yegoshin " .set eva \n" \
1187170bdc7SJames Hogan "1: cachee %1, (%2) \n" \
119f229454dSPaul Burton "2: .insn \n" \
120f229454dSPaul Burton " .set pop \n" \
1217170bdc7SJames Hogan " .section .fixup,\"ax\" \n" \
1227170bdc7SJames Hogan "3: li %0, %3 \n" \
1237170bdc7SJames Hogan " j 2b \n" \
1247170bdc7SJames Hogan " .previous \n" \
125a8053854SLeonid Yegoshin " .section __ex_table,\"a\" \n" \
126fa62f39dSThomas Bogendoerfer " "STR(PTR_WD)" 1b, 3b \n" \
127a8053854SLeonid Yegoshin " .previous" \
1287170bdc7SJames Hogan : "+r" (__err) \
1297170bdc7SJames Hogan : "i" (op), "r" (addr), "i" (-EFAULT)); \
1307170bdc7SJames Hogan __err; \
1317170bdc7SJames Hogan })
132f1b0bf57SThomas Bogendoerfer #else
133f1b0bf57SThomas Bogendoerfer
134f1b0bf57SThomas Bogendoerfer #define protected_cache_op(op, addr) \
135f1b0bf57SThomas Bogendoerfer ({ \
136f1b0bf57SThomas Bogendoerfer int __err = 0; \
137f1b0bf57SThomas Bogendoerfer __asm__ __volatile__( \
138f1b0bf57SThomas Bogendoerfer " .set push \n" \
139f1b0bf57SThomas Bogendoerfer " .set noreorder \n" \
140f1b0bf57SThomas Bogendoerfer " .set "MIPS_ISA_ARCH_LEVEL" \n" \
141f1b0bf57SThomas Bogendoerfer "1: cache %1, (%2) \n" \
142f1b0bf57SThomas Bogendoerfer "2: .insn \n" \
143f1b0bf57SThomas Bogendoerfer " .set pop \n" \
144f1b0bf57SThomas Bogendoerfer " .section .fixup,\"ax\" \n" \
145f1b0bf57SThomas Bogendoerfer "3: li %0, %3 \n" \
146f1b0bf57SThomas Bogendoerfer " j 2b \n" \
147f1b0bf57SThomas Bogendoerfer " .previous \n" \
148f1b0bf57SThomas Bogendoerfer " .section __ex_table,\"a\" \n" \
149fa62f39dSThomas Bogendoerfer " "STR(PTR_WD)" 1b, 3b \n" \
150f1b0bf57SThomas Bogendoerfer " .previous" \
151f1b0bf57SThomas Bogendoerfer : "+r" (__err) \
152f1b0bf57SThomas Bogendoerfer : "i" (op), "r" (addr), "i" (-EFAULT)); \
153f1b0bf57SThomas Bogendoerfer __err; \
154f1b0bf57SThomas Bogendoerfer })
155f1b0bf57SThomas Bogendoerfer #endif
156a8053854SLeonid Yegoshin
157384740dcSRalf Baechle /*
158384740dcSRalf Baechle * The next two are for badland addresses like signal trampolines.
159384740dcSRalf Baechle */
protected_flush_icache_line(unsigned long addr)1607170bdc7SJames Hogan static inline int protected_flush_icache_line(unsigned long addr)
161384740dcSRalf Baechle {
16214bd8c08SRalf Baechle switch (boot_cpu_type()) {
163268a2d60SJiaxun Yang case CPU_LOONGSON2EF:
1647170bdc7SJames Hogan return protected_cache_op(Hit_Invalidate_I_Loongson2, addr);
16514bd8c08SRalf Baechle
16614bd8c08SRalf Baechle default:
1677170bdc7SJames Hogan return protected_cache_op(Hit_Invalidate_I, addr);
16814bd8c08SRalf Baechle }
169384740dcSRalf Baechle }
170384740dcSRalf Baechle
171384740dcSRalf Baechle /*
172384740dcSRalf Baechle * R10000 / R12000 hazard - these processors don't support the Hit_Writeback_D
173384740dcSRalf Baechle * cacheop so we use Hit_Writeback_Inv_D which is supported by all R4000-style
174384740dcSRalf Baechle * caches. We're talking about one cacheline unnecessarily getting invalidated
175384740dcSRalf Baechle * here so the penalty isn't overly hard.
176384740dcSRalf Baechle */
protected_writeback_dcache_line(unsigned long addr)1777170bdc7SJames Hogan static inline int protected_writeback_dcache_line(unsigned long addr)
178384740dcSRalf Baechle {
1797170bdc7SJames Hogan return protected_cache_op(Hit_Writeback_Inv_D, addr);
180384740dcSRalf Baechle }
181384740dcSRalf Baechle
protected_writeback_scache_line(unsigned long addr)1827170bdc7SJames Hogan static inline int protected_writeback_scache_line(unsigned long addr)
183384740dcSRalf Baechle {
1847170bdc7SJames Hogan return protected_cache_op(Hit_Writeback_Inv_SD, addr);
185384740dcSRalf Baechle }
186384740dcSRalf Baechle
187384740dcSRalf Baechle /*
188384740dcSRalf Baechle * This one is RM7000-specific
189384740dcSRalf Baechle */
invalidate_tcache_page(unsigned long addr)190384740dcSRalf Baechle static inline void invalidate_tcache_page(unsigned long addr)
191384740dcSRalf Baechle {
192384740dcSRalf Baechle cache_op(Page_Invalidate_T, addr);
193384740dcSRalf Baechle }
194384740dcSRalf Baechle
1956baaeadaSPaul Burton #define cache_unroll(times, insn, op, addr, lsize) do { \
1966baaeadaSPaul Burton int i = 0; \
1976baaeadaSPaul Burton unroll(times, _cache_op, insn, op, (addr) + (i++ * (lsize))); \
1986baaeadaSPaul Burton } while (0)
199de8974e3SLeonid Yegoshin
200384740dcSRalf Baechle /* build blast_xxx, blast_xxx_page, blast_xxx_page_indexed */
20143a06847SAaro Koskinen #define __BUILD_BLAST_CACHE(pfx, desc, indexop, hitop, lsize, extra) \
20243a06847SAaro Koskinen static inline void extra##blast_##pfx##cache##lsize(void) \
203384740dcSRalf Baechle { \
204384740dcSRalf Baechle unsigned long start = INDEX_BASE; \
205384740dcSRalf Baechle unsigned long end = start + current_cpu_data.desc.waysize; \
206384740dcSRalf Baechle unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \
207384740dcSRalf Baechle unsigned long ws_end = current_cpu_data.desc.ways << \
208384740dcSRalf Baechle current_cpu_data.desc.waybit; \
209384740dcSRalf Baechle unsigned long ws, addr; \
210384740dcSRalf Baechle \
211384740dcSRalf Baechle for (ws = 0; ws < ws_end; ws += ws_inc) \
212384740dcSRalf Baechle for (addr = start; addr < end; addr += lsize * 32) \
2136baaeadaSPaul Burton cache_unroll(32, kernel_cache, indexop, \
2146baaeadaSPaul Burton addr | ws, lsize); \
215384740dcSRalf Baechle } \
216384740dcSRalf Baechle \
21743a06847SAaro Koskinen static inline void extra##blast_##pfx##cache##lsize##_page(unsigned long page) \
218384740dcSRalf Baechle { \
219384740dcSRalf Baechle unsigned long start = page; \
220384740dcSRalf Baechle unsigned long end = page + PAGE_SIZE; \
221384740dcSRalf Baechle \
222384740dcSRalf Baechle do { \
2236baaeadaSPaul Burton cache_unroll(32, kernel_cache, hitop, start, lsize); \
224384740dcSRalf Baechle start += lsize * 32; \
225384740dcSRalf Baechle } while (start < end); \
226384740dcSRalf Baechle } \
227384740dcSRalf Baechle \
22843a06847SAaro Koskinen static inline void extra##blast_##pfx##cache##lsize##_page_indexed(unsigned long page) \
229384740dcSRalf Baechle { \
230384740dcSRalf Baechle unsigned long indexmask = current_cpu_data.desc.waysize - 1; \
231384740dcSRalf Baechle unsigned long start = INDEX_BASE + (page & indexmask); \
232384740dcSRalf Baechle unsigned long end = start + PAGE_SIZE; \
233384740dcSRalf Baechle unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \
234384740dcSRalf Baechle unsigned long ws_end = current_cpu_data.desc.ways << \
235384740dcSRalf Baechle current_cpu_data.desc.waybit; \
236384740dcSRalf Baechle unsigned long ws, addr; \
237384740dcSRalf Baechle \
238384740dcSRalf Baechle for (ws = 0; ws < ws_end; ws += ws_inc) \
239384740dcSRalf Baechle for (addr = start; addr < end; addr += lsize * 32) \
2406baaeadaSPaul Burton cache_unroll(32, kernel_cache, indexop, \
2416baaeadaSPaul Burton addr | ws, lsize); \
242384740dcSRalf Baechle }
243384740dcSRalf Baechle
24443a06847SAaro Koskinen __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 16, )
24543a06847SAaro Koskinen __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16, )
24643a06847SAaro Koskinen __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 16, )
24743a06847SAaro Koskinen __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 32, )
24843a06847SAaro Koskinen __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32, )
24943a06847SAaro Koskinen __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I_Loongson2, 32, loongson2_)
25043a06847SAaro Koskinen __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 32, )
25143a06847SAaro Koskinen __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 64, )
25243a06847SAaro Koskinen __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64, )
25343a06847SAaro Koskinen __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 64, )
25418a8cd63SDavid Daney __BUILD_BLAST_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D, 128, )
25518a8cd63SDavid Daney __BUILD_BLAST_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 128, )
25643a06847SAaro Koskinen __BUILD_BLAST_CACHE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 128, )
257384740dcSRalf Baechle
25843a06847SAaro Koskinen __BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 16, )
25943a06847SAaro Koskinen __BUILD_BLAST_CACHE(inv_d, dcache, Index_Writeback_Inv_D, Hit_Invalidate_D, 32, )
26043a06847SAaro Koskinen __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 16, )
26143a06847SAaro Koskinen __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 32, )
26243a06847SAaro Koskinen __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 64, )
26343a06847SAaro Koskinen __BUILD_BLAST_CACHE(inv_s, scache, Index_Writeback_Inv_SD, Hit_Invalidate_SD, 128, )
264384740dcSRalf Baechle
265de8974e3SLeonid Yegoshin #define __BUILD_BLAST_USER_CACHE(pfx, desc, indexop, hitop, lsize) \
266de8974e3SLeonid Yegoshin static inline void blast_##pfx##cache##lsize##_user_page(unsigned long page) \
267de8974e3SLeonid Yegoshin { \
268de8974e3SLeonid Yegoshin unsigned long start = page; \
269de8974e3SLeonid Yegoshin unsigned long end = page + PAGE_SIZE; \
270de8974e3SLeonid Yegoshin \
271de8974e3SLeonid Yegoshin do { \
2726baaeadaSPaul Burton cache_unroll(32, user_cache, hitop, start, lsize); \
273de8974e3SLeonid Yegoshin start += lsize * 32; \
274de8974e3SLeonid Yegoshin } while (start < end); \
275de8974e3SLeonid Yegoshin }
276de8974e3SLeonid Yegoshin
277de8974e3SLeonid Yegoshin __BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D,
278de8974e3SLeonid Yegoshin 16)
279de8974e3SLeonid Yegoshin __BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 16)
280de8974e3SLeonid Yegoshin __BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D,
281de8974e3SLeonid Yegoshin 32)
282de8974e3SLeonid Yegoshin __BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 32)
283de8974e3SLeonid Yegoshin __BUILD_BLAST_USER_CACHE(d, dcache, Index_Writeback_Inv_D, Hit_Writeback_Inv_D,
284de8974e3SLeonid Yegoshin 64)
285de8974e3SLeonid Yegoshin __BUILD_BLAST_USER_CACHE(i, icache, Index_Invalidate_I, Hit_Invalidate_I, 64)
286de8974e3SLeonid Yegoshin
287384740dcSRalf Baechle /* build blast_xxx_range, protected_blast_xxx_range */
28814bd8c08SRalf Baechle #define __BUILD_BLAST_CACHE_RANGE(pfx, desc, hitop, prot, extra) \
28914bd8c08SRalf Baechle static inline void prot##extra##blast_##pfx##cache##_range(unsigned long start, \
290384740dcSRalf Baechle unsigned long end) \
291384740dcSRalf Baechle { \
292384740dcSRalf Baechle unsigned long lsize = cpu_##desc##_line_size(); \
293384740dcSRalf Baechle unsigned long addr = start & ~(lsize - 1); \
294384740dcSRalf Baechle unsigned long aend = (end - 1) & ~(lsize - 1); \
295384740dcSRalf Baechle \
296384740dcSRalf Baechle while (1) { \
297384740dcSRalf Baechle prot##cache_op(hitop, addr); \
298384740dcSRalf Baechle if (addr == aend) \
299384740dcSRalf Baechle break; \
300384740dcSRalf Baechle addr += lsize; \
301384740dcSRalf Baechle } \
302384740dcSRalf Baechle }
303384740dcSRalf Baechle
30414bd8c08SRalf Baechle __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, protected_, )
30514bd8c08SRalf Baechle __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, protected_, )
306de8974e3SLeonid Yegoshin __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, protected_, )
307bad009feSHuacai Chen __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I_Loongson2, \
308bad009feSHuacai Chen protected_, loongson2_)
30914bd8c08SRalf Baechle __BUILD_BLAST_CACHE_RANGE(d, dcache, Hit_Writeback_Inv_D, , )
31041e62b04SLeonid Yegoshin __BUILD_BLAST_CACHE_RANGE(i, icache, Hit_Invalidate_I, , )
31114bd8c08SRalf Baechle __BUILD_BLAST_CACHE_RANGE(s, scache, Hit_Writeback_Inv_SD, , )
312384740dcSRalf Baechle /* blast_inv_dcache_range */
31314bd8c08SRalf Baechle __BUILD_BLAST_CACHE_RANGE(inv_d, dcache, Hit_Invalidate_D, , )
31414bd8c08SRalf Baechle __BUILD_BLAST_CACHE_RANGE(inv_s, scache, Hit_Invalidate_SD, , )
315384740dcSRalf Baechle
316bb53fdf3SHuacai Chen /* Currently, this is very specific to Loongson-3 */
317bb53fdf3SHuacai Chen #define __BUILD_BLAST_CACHE_NODE(pfx, desc, indexop, hitop, lsize) \
318bb53fdf3SHuacai Chen static inline void blast_##pfx##cache##lsize##_node(long node) \
319bb53fdf3SHuacai Chen { \
320bb53fdf3SHuacai Chen unsigned long start = CAC_BASE | nid_to_addrbase(node); \
321bb53fdf3SHuacai Chen unsigned long end = start + current_cpu_data.desc.waysize; \
322bb53fdf3SHuacai Chen unsigned long ws_inc = 1UL << current_cpu_data.desc.waybit; \
323bb53fdf3SHuacai Chen unsigned long ws_end = current_cpu_data.desc.ways << \
324bb53fdf3SHuacai Chen current_cpu_data.desc.waybit; \
325bb53fdf3SHuacai Chen unsigned long ws, addr; \
326bb53fdf3SHuacai Chen \
327bb53fdf3SHuacai Chen for (ws = 0; ws < ws_end; ws += ws_inc) \
328bb53fdf3SHuacai Chen for (addr = start; addr < end; addr += lsize * 32) \
3296baaeadaSPaul Burton cache_unroll(32, kernel_cache, indexop, \
3306baaeadaSPaul Burton addr | ws, lsize); \
331bb53fdf3SHuacai Chen }
332bb53fdf3SHuacai Chen
333bb53fdf3SHuacai Chen __BUILD_BLAST_CACHE_NODE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 16)
334bb53fdf3SHuacai Chen __BUILD_BLAST_CACHE_NODE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 32)
335bb53fdf3SHuacai Chen __BUILD_BLAST_CACHE_NODE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 64)
336bb53fdf3SHuacai Chen __BUILD_BLAST_CACHE_NODE(s, scache, Index_Writeback_Inv_SD, Hit_Writeback_Inv_SD, 128)
337bb53fdf3SHuacai Chen
338384740dcSRalf Baechle #endif /* _ASM_R4KCACHE_H */
339