1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 *
5 * Derived from MIPS:
6 * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org)
7 * Copyright (C) 2007 MIPS Technologies, Inc.
8 */
9 #include <linux/cacheinfo.h>
10 #include <linux/export.h>
11 #include <linux/fs.h>
12 #include <linux/highmem.h>
13 #include <linux/kernel.h>
14 #include <linux/linkage.h>
15 #include <linux/mm.h>
16 #include <linux/sched.h>
17 #include <linux/syscalls.h>
18
19 #include <asm/bootinfo.h>
20 #include <asm/cacheflush.h>
21 #include <asm/cpu.h>
22 #include <asm/cpu-features.h>
23 #include <asm/loongarch.h>
24 #include <asm/numa.h>
25 #include <asm/processor.h>
26 #include <asm/setup.h>
27
cache_error_setup(void)28 void cache_error_setup(void)
29 {
30 extern char __weak except_vec_cex;
31 set_merr_handler(0x0, &except_vec_cex, 0x80);
32 }
33
flush_cache_leaf(unsigned int leaf)34 static void flush_cache_leaf(unsigned int leaf)
35 {
36 int i, j, nr_nodes;
37 uint64_t addr = CSR_DMW0_BASE;
38 struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf;
39
40 nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes;
41
42 do {
43 for (i = 0; i < cdesc->sets; i++) {
44 for (j = 0; j < cdesc->ways; j++) {
45 flush_cache_line(leaf, addr);
46 addr++;
47 }
48
49 addr -= cdesc->ways;
50 addr += cdesc->linesz;
51 }
52 addr += (1ULL << NODE_ADDRSPACE_SHIFT);
53 } while (--nr_nodes > 0);
54 }
55
__flush_cache_all(void)56 asmlinkage __visible void __flush_cache_all(void)
57 {
58 int leaf;
59 struct cache_desc *cdesc = current_cpu_data.cache_leaves;
60 unsigned int cache_present = current_cpu_data.cache_leaves_present;
61
62 leaf = cache_present - 1;
63 if (cache_inclusive(cdesc + leaf)) {
64 flush_cache_leaf(leaf);
65 return;
66 }
67
68 for (leaf = 0; leaf < cache_present; leaf++)
69 flush_cache_leaf(leaf);
70 }
71
72 #define L1IUPRE (1 << 0)
73 #define L1IUUNIFY (1 << 1)
74 #define L1DPRE (1 << 2)
75
76 #define LXIUPRE (1 << 0)
77 #define LXIUUNIFY (1 << 1)
78 #define LXIUPRIV (1 << 2)
79 #define LXIUINCL (1 << 3)
80 #define LXDPRE (1 << 4)
81 #define LXDPRIV (1 << 5)
82 #define LXDINCL (1 << 6)
83
84 #define populate_cache_properties(cfg0, cdesc, level, leaf) \
85 do { \
86 unsigned int cfg1; \
87 \
88 cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \
89 if (level == 1) { \
90 cdesc->flags |= CACHE_PRIVATE; \
91 } else { \
92 if (cfg0 & LXIUPRIV) \
93 cdesc->flags |= CACHE_PRIVATE; \
94 if (cfg0 & LXIUINCL) \
95 cdesc->flags |= CACHE_INCLUSIVE; \
96 } \
97 cdesc->level = level; \
98 cdesc->flags |= CACHE_PRESENT; \
99 cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \
100 cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \
101 cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \
102 cdesc++; leaf++; \
103 } while (0)
104
cpu_cache_init(void)105 void cpu_cache_init(void)
106 {
107 unsigned int leaf = 0, level = 1;
108 unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16);
109 struct cache_desc *cdesc = current_cpu_data.cache_leaves;
110
111 if (config & L1IUPRE) {
112 if (config & L1IUUNIFY)
113 cdesc->type = CACHE_TYPE_UNIFIED;
114 else
115 cdesc->type = CACHE_TYPE_INST;
116 populate_cache_properties(config, cdesc, level, leaf);
117 }
118
119 if (config & L1DPRE) {
120 cdesc->type = CACHE_TYPE_DATA;
121 populate_cache_properties(config, cdesc, level, leaf);
122 }
123
124 config = config >> 3;
125 for (level = 2; level <= CACHE_LEVEL_MAX; level++) {
126 if (!config)
127 break;
128
129 if (config & LXIUPRE) {
130 if (config & LXIUUNIFY)
131 cdesc->type = CACHE_TYPE_UNIFIED;
132 else
133 cdesc->type = CACHE_TYPE_INST;
134 populate_cache_properties(config, cdesc, level, leaf);
135 }
136
137 if (config & LXDPRE) {
138 cdesc->type = CACHE_TYPE_DATA;
139 populate_cache_properties(config, cdesc, level, leaf);
140 }
141
142 config = config >> 7;
143 }
144
145 BUG_ON(leaf > CACHE_LEAVES_MAX);
146
147 current_cpu_data.cache_leaves_present = leaf;
148 current_cpu_data.options |= LOONGARCH_CPU_PREFETCH;
149 }
150
151 static const pgprot_t protection_map[16] = {
152 [VM_NONE] = __pgprot(_CACHE_CC | _PAGE_USER |
153 _PAGE_NO_EXEC | _PAGE_NO_READ |
154 (_PAGE_PROTNONE ? : _PAGE_PRESENT)),
155 [VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
156 _PAGE_USER | _PAGE_PRESENT |
157 _PAGE_NO_EXEC),
158 [VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID |
159 _PAGE_USER | _PAGE_PRESENT |
160 _PAGE_NO_EXEC),
161 [VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
162 _PAGE_USER | _PAGE_PRESENT |
163 _PAGE_NO_EXEC),
164 [VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID |
165 _PAGE_USER | _PAGE_PRESENT),
166 [VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
167 _PAGE_USER | _PAGE_PRESENT),
168 [VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID |
169 _PAGE_USER | _PAGE_PRESENT),
170 [VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
171 _PAGE_USER | _PAGE_PRESENT),
172 [VM_SHARED] = __pgprot(_CACHE_CC | _PAGE_USER |
173 _PAGE_NO_EXEC | _PAGE_NO_READ |
174 (_PAGE_PROTNONE ? : _PAGE_PRESENT)),
175 [VM_SHARED | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
176 _PAGE_USER | _PAGE_PRESENT |
177 _PAGE_NO_EXEC),
178 [VM_SHARED | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID |
179 _PAGE_USER | _PAGE_PRESENT |
180 _PAGE_NO_EXEC | _PAGE_WRITE),
181 [VM_SHARED | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
182 _PAGE_USER | _PAGE_PRESENT |
183 _PAGE_NO_EXEC | _PAGE_WRITE),
184 [VM_SHARED | VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID |
185 _PAGE_USER | _PAGE_PRESENT),
186 [VM_SHARED | VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
187 _PAGE_USER | _PAGE_PRESENT),
188 [VM_SHARED | VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID |
189 _PAGE_USER | _PAGE_PRESENT |
190 _PAGE_WRITE),
191 [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID |
192 _PAGE_USER | _PAGE_PRESENT |
193 _PAGE_WRITE)
194 };
195 DECLARE_VM_GET_PAGE_PROT
196