1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2017 SiFive
4 */
5
6 #include <linux/acpi.h>
7 #include <linux/of.h>
8 #include <linux/prctl.h>
9 #include <asm/acpi.h>
10 #include <asm/cacheflush.h>
11
12 #ifdef CONFIG_SMP
13
14 #include <asm/sbi.h>
15
ipi_remote_fence_i(void * info)16 static void ipi_remote_fence_i(void *info)
17 {
18 return local_flush_icache_all();
19 }
20
flush_icache_all(void)21 void flush_icache_all(void)
22 {
23 local_flush_icache_all();
24
25 if (num_online_cpus() < 2)
26 return;
27
28 /*
29 * Make sure all previous writes to the D$ are ordered before making
30 * the IPI. The RISC-V spec states that a hart must execute a data fence
31 * before triggering a remote fence.i in order to make the modification
32 * visable for remote harts.
33 *
34 * IPIs on RISC-V are triggered by MMIO writes to either CLINT or
35 * S-IMSIC, so the fence ensures previous data writes "happen before"
36 * the MMIO.
37 */
38 RISCV_FENCE(w, o);
39
40 if (riscv_use_sbi_for_rfence())
41 sbi_remote_fence_i(NULL);
42 else
43 on_each_cpu(ipi_remote_fence_i, NULL, 1);
44 }
45 EXPORT_SYMBOL(flush_icache_all);
46
47 /*
48 * Performs an icache flush for the given MM context. RISC-V has no direct
49 * mechanism for instruction cache shoot downs, so instead we send an IPI that
50 * informs the remote harts they need to flush their local instruction caches.
51 * To avoid pathologically slow behavior in a common case (a bunch of
52 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the
53 * IPIs for harts that are not currently executing a MM context and instead
54 * schedule a deferred local instruction cache flush to be performed before
55 * execution resumes on each hart.
56 */
flush_icache_mm(struct mm_struct * mm,bool local)57 void flush_icache_mm(struct mm_struct *mm, bool local)
58 {
59 unsigned int cpu;
60 cpumask_t others, *mask;
61
62 preempt_disable();
63
64 /* Mark every hart's icache as needing a flush for this MM. */
65 mask = &mm->context.icache_stale_mask;
66 cpumask_setall(mask);
67 /* Flush this hart's I$ now, and mark it as flushed. */
68 cpu = smp_processor_id();
69 cpumask_clear_cpu(cpu, mask);
70 local_flush_icache_all();
71
72 /*
73 * Flush the I$ of other harts concurrently executing, and mark them as
74 * flushed.
75 */
76 cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
77 local |= cpumask_empty(&others);
78 if (mm == current->active_mm && local) {
79 /*
80 * It's assumed that at least one strongly ordered operation is
81 * performed on this hart between setting a hart's cpumask bit
82 * and scheduling this MM context on that hart. Sending an SBI
83 * remote message will do this, but in the case where no
84 * messages are sent we still need to order this hart's writes
85 * with flush_icache_deferred().
86 */
87 smp_mb();
88 } else if (riscv_use_sbi_for_rfence()) {
89 sbi_remote_fence_i(&others);
90 } else {
91 on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1);
92 }
93
94 preempt_enable();
95 }
96
97 #endif /* CONFIG_SMP */
98
99 #ifdef CONFIG_MMU
flush_icache_pte(struct mm_struct * mm,pte_t pte)100 void flush_icache_pte(struct mm_struct *mm, pte_t pte)
101 {
102 struct folio *folio = page_folio(pte_page(pte));
103
104 if (!test_bit(PG_dcache_clean, &folio->flags)) {
105 flush_icache_mm(mm, false);
106 set_bit(PG_dcache_clean, &folio->flags);
107 }
108 }
109 #endif /* CONFIG_MMU */
110
111 unsigned int riscv_cbom_block_size;
112 EXPORT_SYMBOL_GPL(riscv_cbom_block_size);
113
114 unsigned int riscv_cboz_block_size;
115 EXPORT_SYMBOL_GPL(riscv_cboz_block_size);
116
117 unsigned int riscv_cbop_block_size;
118 EXPORT_SYMBOL_GPL(riscv_cbop_block_size);
119
cbo_get_block_size(struct device_node * node,const char * name,u32 * block_size,unsigned long * first_hartid)120 static void __init cbo_get_block_size(struct device_node *node,
121 const char *name, u32 *block_size,
122 unsigned long *first_hartid)
123 {
124 unsigned long hartid;
125 u32 val;
126
127 if (riscv_of_processor_hartid(node, &hartid))
128 return;
129
130 if (of_property_read_u32(node, name, &val))
131 return;
132
133 if (!*block_size) {
134 *block_size = val;
135 *first_hartid = hartid;
136 } else if (*block_size != val) {
137 pr_warn("%s mismatched between harts %lu and %lu\n",
138 name, *first_hartid, hartid);
139 }
140 }
141
riscv_init_cbo_blocksizes(void)142 void __init riscv_init_cbo_blocksizes(void)
143 {
144 unsigned long cbom_hartid, cboz_hartid, cbop_hartid;
145 u32 cbom_block_size = 0, cboz_block_size = 0, cbop_block_size = 0;
146 struct device_node *node;
147 struct acpi_table_header *rhct;
148 acpi_status status;
149
150 if (acpi_disabled) {
151 for_each_of_cpu_node(node) {
152 /* set block-size for cbom and/or cboz extension if available */
153 cbo_get_block_size(node, "riscv,cbom-block-size",
154 &cbom_block_size, &cbom_hartid);
155 cbo_get_block_size(node, "riscv,cboz-block-size",
156 &cboz_block_size, &cboz_hartid);
157 cbo_get_block_size(node, "riscv,cbop-block-size",
158 &cbop_block_size, &cbop_hartid);
159 }
160 } else {
161 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
162 if (ACPI_FAILURE(status))
163 return;
164
165 acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, &cbop_block_size);
166 acpi_put_table((struct acpi_table_header *)rhct);
167 }
168
169 if (cbom_block_size)
170 riscv_cbom_block_size = cbom_block_size;
171
172 if (cboz_block_size)
173 riscv_cboz_block_size = cboz_block_size;
174
175 if (cbop_block_size)
176 riscv_cbop_block_size = cbop_block_size;
177 }
178
179 #ifdef CONFIG_SMP
set_icache_stale_mask(void)180 static void set_icache_stale_mask(void)
181 {
182 int cpu = get_cpu();
183 cpumask_t *mask;
184 bool stale_cpu;
185
186 /*
187 * Mark every other hart's icache as needing a flush for
188 * this MM. Maintain the previous value of the current
189 * cpu to handle the case when this function is called
190 * concurrently on different harts.
191 */
192 mask = ¤t->mm->context.icache_stale_mask;
193 stale_cpu = cpumask_test_cpu(cpu, mask);
194
195 cpumask_setall(mask);
196 __assign_cpu(cpu, mask, stale_cpu);
197 put_cpu();
198 }
199 #endif
200
201 /**
202 * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in
203 * userspace.
204 * @ctx: Set the type of icache flushing instructions permitted/prohibited in
205 * userspace. Supported values described below.
206 *
207 * Supported values for ctx:
208 *
209 * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space.
210 *
211 * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in
212 * a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``.
213 * Therefore, caution must be taken; use this flag only when you can guarantee
214 * that no thread in the process will emit fence.i from this point onward.
215 *
216 * @scope: Set scope of where icache flushing instructions are allowed to be
217 * emitted. Supported values described below.
218 *
219 * Supported values for scope:
220 *
221 * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process
222 * is coherent with instruction storage upon
223 * migration.
224 *
225 * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is
226 * coherent with instruction storage upon
227 * migration.
228 *
229 * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are
230 * permitted to emit icache flushing instructions. Whenever any thread in the
231 * process is migrated, the corresponding hart's icache will be guaranteed to be
232 * consistent with instruction storage. This does not enforce any guarantees
233 * outside of migration. If a thread modifies an instruction that another thread
234 * may attempt to execute, the other thread must still emit an icache flushing
235 * instruction before attempting to execute the potentially modified
236 * instruction. This must be performed by the user-space program.
237 *
238 * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the
239 * thread calling this function is permitted to emit icache flushing
240 * instructions. When the thread is migrated, the corresponding hart's icache
241 * will be guaranteed to be consistent with instruction storage.
242 *
243 * On kernels configured without SMP, this function is a nop as migrations
244 * across harts will not occur.
245 */
riscv_set_icache_flush_ctx(unsigned long ctx,unsigned long scope)246 int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope)
247 {
248 #ifdef CONFIG_SMP
249 switch (ctx) {
250 case PR_RISCV_CTX_SW_FENCEI_ON:
251 switch (scope) {
252 case PR_RISCV_SCOPE_PER_PROCESS:
253 current->mm->context.force_icache_flush = true;
254 break;
255 case PR_RISCV_SCOPE_PER_THREAD:
256 current->thread.force_icache_flush = true;
257 break;
258 default:
259 return -EINVAL;
260 }
261 break;
262 case PR_RISCV_CTX_SW_FENCEI_OFF:
263 switch (scope) {
264 case PR_RISCV_SCOPE_PER_PROCESS:
265 set_icache_stale_mask();
266 current->mm->context.force_icache_flush = false;
267 break;
268 case PR_RISCV_SCOPE_PER_THREAD:
269 set_icache_stale_mask();
270 current->thread.force_icache_flush = false;
271 break;
272 default:
273 return -EINVAL;
274 }
275 break;
276 default:
277 return -EINVAL;
278 }
279 return 0;
280 #else
281 switch (ctx) {
282 case PR_RISCV_CTX_SW_FENCEI_ON:
283 case PR_RISCV_CTX_SW_FENCEI_OFF:
284 return 0;
285 default:
286 return -EINVAL;
287 }
288 #endif
289 }
290