1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2024 Rivos Inc.
4 */
5
6 #include <linux/cpu.h>
7 #include <linux/cpumask.h>
8 #include <linux/jump_label.h>
9 #include <linux/mm.h>
10 #include <linux/smp.h>
11 #include <linux/types.h>
12 #include <asm/cpufeature.h>
13 #include <asm/hwprobe.h>
14 #include <asm/vector.h>
15
16 #include "copy-unaligned.h"
17
18 #define MISALIGNED_ACCESS_NS 8000000
19 #define MISALIGNED_BUFFER_SIZE 0x4000
20 #define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE)
21 #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80)
22
23 DEFINE_PER_CPU(long, misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
24 DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
25
26 static long unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
27 static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
28
29 static u64 __maybe_unused
measure_cycles(void (* func)(void * dst,const void * src,size_t len),void * dst,void * src,size_t len)30 measure_cycles(void (*func)(void *dst, const void *src, size_t len),
31 void *dst, void *src, size_t len)
32 {
33 u64 start_cycles, end_cycles, cycles = -1ULL;
34 u64 start_ns;
35
36 /* Do a warmup. */
37 func(dst, src, len);
38
39 preempt_disable();
40
41 /*
42 * For a fixed amount of time, repeatedly try the function, and take
43 * the best time in cycles as the measurement.
44 */
45 start_ns = ktime_get_mono_fast_ns();
46 while (ktime_get_mono_fast_ns() < start_ns + MISALIGNED_ACCESS_NS) {
47 start_cycles = get_cycles64();
48 /* Ensure the CSR read can't reorder WRT to the copy. */
49 mb();
50 func(dst, src, len);
51 /* Ensure the copy ends before the end time is snapped. */
52 mb();
53 end_cycles = get_cycles64();
54 if ((end_cycles - start_cycles) < cycles)
55 cycles = end_cycles - start_cycles;
56 }
57
58 preempt_enable();
59
60 return cycles;
61 }
62
63 /*
64 * Return:
65 * 1 if unaligned accesses are fast
66 * 0 if unaligned accesses are slow
67 * -1 if check cannot be done
68 */
69 static int __maybe_unused
compare_unaligned_access(void (* word_copy)(void * dst,const void * src,size_t len),void (* byte_copy)(void * dst,const void * src,size_t len),void * buf,const char * type)70 compare_unaligned_access(void (*word_copy)(void *dst, const void *src, size_t len),
71 void (*byte_copy)(void *dst, const void *src, size_t len),
72 void *buf, const char *type)
73 {
74 int cpu = smp_processor_id();
75 u64 word_cycles;
76 u64 byte_cycles;
77 void *dst, *src;
78 bool fast;
79 int ratio;
80
81 /* Make an unaligned destination buffer. */
82 dst = (void *)((unsigned long)buf | 0x1);
83 /* Unalign src as well, but differently (off by 1 + 2 = 3). */
84 src = dst + (MISALIGNED_BUFFER_SIZE / 2);
85 src += 2;
86
87 word_cycles = measure_cycles(word_copy, dst, src, MISALIGNED_COPY_SIZE);
88 byte_cycles = measure_cycles(byte_copy, dst, src, MISALIGNED_COPY_SIZE);
89
90 /* Don't divide by zero. */
91 if (!word_cycles || !byte_cycles) {
92 pr_warn("cpu%d: rdtime lacks granularity needed to measure %s unaligned access speed\n",
93 cpu, type);
94
95 return -1;
96 }
97
98 fast = word_cycles < byte_cycles;
99
100 ratio = div_u64((byte_cycles * 100), word_cycles);
101 pr_info("cpu%d: %s unaligned word access speed is %d.%02dx byte access speed (%s)\n",
102 cpu,
103 type,
104 ratio / 100,
105 ratio % 100,
106 fast ? "fast" : "slow");
107
108 return fast;
109 }
110
111 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
check_unaligned_access(struct page * page)112 static int check_unaligned_access(struct page *page)
113 {
114 void *buf = page_address(page);
115 int cpu = smp_processor_id();
116 int ret;
117
118 if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN)
119 return 0;
120
121 ret = compare_unaligned_access(__riscv_copy_words_unaligned,
122 __riscv_copy_bytes_unaligned,
123 buf, "scalar");
124 if (ret < 0)
125 return 0;
126
127 /*
128 * Set the value of fast_misaligned_access of a CPU. These operations
129 * are atomic to avoid race conditions.
130 */
131 if (ret)
132 per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
133 else
134 per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
135
136 return 0;
137 }
138
_check_unaligned_access(void * param)139 static void __init _check_unaligned_access(void *param)
140 {
141 unsigned int cpu = smp_processor_id();
142 struct page **pages = param;
143
144 check_unaligned_access(pages[cpu]);
145 }
146
147 /* Measure unaligned access speed on all CPUs present at boot in parallel. */
check_unaligned_access_speed_all_cpus(void)148 static void __init check_unaligned_access_speed_all_cpus(void)
149 {
150 unsigned int cpu;
151 unsigned int cpu_count = num_possible_cpus();
152 struct page **bufs = kzalloc_objs(*bufs, cpu_count);
153
154 if (!bufs) {
155 pr_warn("Allocation failure, not measuring misaligned performance\n");
156 return;
157 }
158
159 /*
160 * Allocate separate buffers for each CPU so there's no fighting over
161 * cache lines.
162 */
163 for_each_cpu(cpu, cpu_online_mask) {
164 bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
165 if (!bufs[cpu]) {
166 pr_warn("Allocation failure, not measuring misaligned performance\n");
167 goto out;
168 }
169 }
170
171 on_each_cpu(_check_unaligned_access, bufs, 1);
172
173 out:
174 for_each_cpu(cpu, cpu_online_mask) {
175 if (bufs[cpu])
176 __free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER);
177 }
178
179 kfree(bufs);
180 }
181 #else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */
check_unaligned_access_speed_all_cpus(void)182 static void __init check_unaligned_access_speed_all_cpus(void)
183 {
184 }
185 #endif
186
187 DEFINE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);
188
modify_unaligned_access_branches(const cpumask_t * mask)189 static void modify_unaligned_access_branches(const cpumask_t *mask)
190 {
191 bool fast = true;
192 int cpu;
193
194 for_each_cpu(cpu, mask) {
195 if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) {
196 fast = false;
197 break;
198 }
199 }
200
201 if (fast)
202 static_branch_enable_cpuslocked(&fast_unaligned_access_speed_key);
203 else
204 static_branch_disable_cpuslocked(&fast_unaligned_access_speed_key);
205 }
206
riscv_online_cpu(unsigned int cpu)207 static int riscv_online_cpu(unsigned int cpu)
208 {
209 int ret = cpu_online_unaligned_access_init(cpu);
210
211 if (ret)
212 return ret;
213
214 /* We are already set since the last check */
215 if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) {
216 goto exit;
217 } else if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) {
218 per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param;
219 goto exit;
220 }
221
222 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
223 {
224 static struct page *buf;
225
226 buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
227 if (!buf) {
228 pr_warn("Allocation failure, not measuring misaligned performance\n");
229 return -ENOMEM;
230 }
231
232 check_unaligned_access(buf);
233 __free_pages(buf, MISALIGNED_BUFFER_ORDER);
234 }
235 #endif
236
237 exit:
238 modify_unaligned_access_branches(cpu_online_mask);
239
240 return 0;
241 }
242
riscv_offline_cpu(unsigned int cpu)243 static int riscv_offline_cpu(unsigned int cpu)
244 {
245 cpumask_t mask;
246
247 cpumask_copy(&mask, cpu_online_mask);
248 cpumask_clear_cpu(cpu, &mask);
249
250 modify_unaligned_access_branches(&mask);
251
252 return 0;
253 }
254
255 #ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
check_vector_unaligned_access(struct work_struct * work __always_unused)256 static void check_vector_unaligned_access(struct work_struct *work __always_unused)
257 {
258 int cpu = smp_processor_id();
259 struct page *page;
260 int ret;
261
262 if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
263 return;
264
265 page = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER);
266 if (!page) {
267 pr_warn("Allocation failure, not measuring vector misaligned performance\n");
268 return;
269 }
270
271 kernel_vector_begin();
272
273 ret = compare_unaligned_access(__riscv_copy_vec_words_unaligned,
274 __riscv_copy_vec_bytes_unaligned,
275 page_address(page), "vector");
276 kernel_vector_end();
277
278 if (ret < 0)
279 goto free;
280
281 if (ret)
282 per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
283 else
284 per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
285
286 free:
287 __free_pages(page, MISALIGNED_BUFFER_ORDER);
288 }
289
290 #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */
check_vector_unaligned_access(struct work_struct * work __always_unused)291 static void check_vector_unaligned_access(struct work_struct *work __always_unused)
292 {
293 }
294 #endif
295
riscv_online_cpu_vec(unsigned int cpu)296 static int riscv_online_cpu_vec(unsigned int cpu)
297 {
298 if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
299 per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param;
300 return 0;
301 }
302
303 #ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS
304 if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
305 return 0;
306
307 check_vector_unaligned_access_emulated(NULL);
308 check_vector_unaligned_access(NULL);
309 #endif
310
311 return 0;
312 }
313
314 static const char * const speed_str[] __initconst = { NULL, NULL, "slow", "fast", "unsupported" };
315
set_unaligned_scalar_speed_param(char * str)316 static int __init set_unaligned_scalar_speed_param(char *str)
317 {
318 if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW]))
319 unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW;
320 else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_FAST]))
321 unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST;
322 else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED]))
323 unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED;
324 else
325 return -EINVAL;
326
327 return 1;
328 }
329 __setup("unaligned_scalar_speed=", set_unaligned_scalar_speed_param);
330
set_unaligned_vector_speed_param(char * str)331 static int __init set_unaligned_vector_speed_param(char *str)
332 {
333 if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW]))
334 unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW;
335 else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_FAST]))
336 unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST;
337 else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED]))
338 unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
339 else
340 return -EINVAL;
341
342 return 1;
343 }
344 __setup("unaligned_vector_speed=", set_unaligned_vector_speed_param);
345
check_unaligned_access_all_cpus(void)346 static int __init check_unaligned_access_all_cpus(void)
347 {
348 int cpu;
349
350 unaligned_access_init();
351
352 if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) {
353 pr_info("scalar unaligned access speed set to '%s' (%lu) by command line\n",
354 speed_str[unaligned_scalar_speed_param], unaligned_scalar_speed_param);
355 for_each_online_cpu(cpu)
356 per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param;
357 } else if (!check_unaligned_access_emulated_all_cpus()) {
358 check_unaligned_access_speed_all_cpus();
359 }
360
361 if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
362 if (!has_vector() &&
363 unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED) {
364 pr_warn("vector support is not available, ignoring unaligned_vector_speed=%s\n",
365 speed_str[unaligned_vector_speed_param]);
366 } else {
367 pr_info("vector unaligned access speed set to '%s' (%lu) by command line\n",
368 speed_str[unaligned_vector_speed_param], unaligned_vector_speed_param);
369 }
370 }
371
372 if (!has_vector())
373 unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
374
375 if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
376 for_each_online_cpu(cpu)
377 per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param;
378 } else if (!check_vector_unaligned_access_emulated_all_cpus() &&
379 IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) {
380 schedule_on_each_cpu(check_vector_unaligned_access);
381 }
382
383 /*
384 * Setup hotplug callbacks for any new CPUs that come online or go
385 * offline.
386 */
387 cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
388 riscv_online_cpu, riscv_offline_cpu);
389 cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online",
390 riscv_online_cpu_vec, NULL);
391
392 cpus_read_lock();
393 modify_unaligned_access_branches(cpu_online_mask);
394 cpus_read_unlock();
395
396 return 0;
397 }
398
399 /*
400 * Run after clocksource_done_booting() so measure_cycles() uses a stable
401 * clocksource, but before rootfs_initcall() enables usermode helpers. Those
402 * helpers can reach hwprobe and populate the vDSO cache, so async hwprobe
403 * probes must be registered first.
404 */
405 fs_initcall_sync(check_unaligned_access_all_cpus);
406