1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2024 Rivos Inc. 4 */ 5 6 #include <linux/cpu.h> 7 #include <linux/cpumask.h> 8 #include <linux/jump_label.h> 9 #include <linux/mm.h> 10 #include <linux/smp.h> 11 #include <linux/types.h> 12 #include <asm/cpufeature.h> 13 #include <asm/hwprobe.h> 14 #include <asm/vector.h> 15 16 #include "copy-unaligned.h" 17 18 #define MISALIGNED_ACCESS_NS 8000000 19 #define MISALIGNED_BUFFER_SIZE 0x4000 20 #define MISALIGNED_BUFFER_ORDER get_order(MISALIGNED_BUFFER_SIZE) 21 #define MISALIGNED_COPY_SIZE ((MISALIGNED_BUFFER_SIZE / 2) - 0x80) 22 23 DEFINE_PER_CPU(long, misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 24 DEFINE_PER_CPU(long, vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; 25 26 static long unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 27 static long unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 28 29 static u64 __maybe_unused 30 measure_cycles(void (*func)(void *dst, const void *src, size_t len), 31 void *dst, void *src, size_t len) 32 { 33 u64 start_cycles, end_cycles, cycles = -1ULL; 34 u64 start_ns; 35 36 /* Do a warmup. */ 37 func(dst, src, len); 38 39 preempt_disable(); 40 41 /* 42 * For a fixed amount of time, repeatedly try the function, and take 43 * the best time in cycles as the measurement. 44 */ 45 start_ns = ktime_get_mono_fast_ns(); 46 while (ktime_get_mono_fast_ns() < start_ns + MISALIGNED_ACCESS_NS) { 47 start_cycles = get_cycles64(); 48 /* Ensure the CSR read can't reorder WRT to the copy. */ 49 mb(); 50 func(dst, src, len); 51 /* Ensure the copy ends before the end time is snapped. */ 52 mb(); 53 end_cycles = get_cycles64(); 54 if ((end_cycles - start_cycles) < cycles) 55 cycles = end_cycles - start_cycles; 56 } 57 58 preempt_enable(); 59 60 return cycles; 61 } 62 63 /* 64 * Return: 65 * 1 if unaligned accesses are fast 66 * 0 if unaligned accesses are slow 67 * -1 if check cannot be done 68 */ 69 static int __maybe_unused 70 compare_unaligned_access(void (*word_copy)(void *dst, const void *src, size_t len), 71 void (*byte_copy)(void *dst, const void *src, size_t len), 72 void *buf, const char *type) 73 { 74 int cpu = smp_processor_id(); 75 u64 word_cycles; 76 u64 byte_cycles; 77 void *dst, *src; 78 bool fast; 79 int ratio; 80 81 /* Make an unaligned destination buffer. */ 82 dst = (void *)((unsigned long)buf | 0x1); 83 /* Unalign src as well, but differently (off by 1 + 2 = 3). */ 84 src = dst + (MISALIGNED_BUFFER_SIZE / 2); 85 src += 2; 86 87 word_cycles = measure_cycles(word_copy, dst, src, MISALIGNED_COPY_SIZE); 88 byte_cycles = measure_cycles(byte_copy, dst, src, MISALIGNED_COPY_SIZE); 89 90 /* Don't divide by zero. */ 91 if (!word_cycles || !byte_cycles) { 92 pr_warn("cpu%d: rdtime lacks granularity needed to measure %s unaligned access speed\n", 93 cpu, type); 94 95 return -1; 96 } 97 98 fast = word_cycles < byte_cycles; 99 100 ratio = div_u64((byte_cycles * 100), word_cycles); 101 pr_info("cpu%d: %s unaligned word access speed is %d.%02dx byte access speed (%s)\n", 102 cpu, 103 type, 104 ratio / 100, 105 ratio % 100, 106 fast ? "fast" : "slow"); 107 108 return fast; 109 } 110 111 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS 112 static int check_unaligned_access(struct page *page) 113 { 114 void *buf = page_address(page); 115 int cpu = smp_processor_id(); 116 int ret; 117 118 if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) 119 return 0; 120 121 ret = compare_unaligned_access(__riscv_copy_words_unaligned, 122 __riscv_copy_bytes_unaligned, 123 buf, "scalar"); 124 if (ret < 0) 125 return 0; 126 127 /* 128 * Set the value of fast_misaligned_access of a CPU. These operations 129 * are atomic to avoid race conditions. 130 */ 131 if (ret) 132 per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; 133 else 134 per_cpu(misaligned_access_speed, cpu) = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; 135 136 return 0; 137 } 138 139 static void __init _check_unaligned_access(void *param) 140 { 141 unsigned int cpu = smp_processor_id(); 142 struct page **pages = param; 143 144 check_unaligned_access(pages[cpu]); 145 } 146 147 /* Measure unaligned access speed on all CPUs present at boot in parallel. */ 148 static void __init check_unaligned_access_speed_all_cpus(void) 149 { 150 unsigned int cpu; 151 unsigned int cpu_count = num_possible_cpus(); 152 struct page **bufs = kzalloc_objs(*bufs, cpu_count); 153 154 if (!bufs) { 155 pr_warn("Allocation failure, not measuring misaligned performance\n"); 156 return; 157 } 158 159 /* 160 * Allocate separate buffers for each CPU so there's no fighting over 161 * cache lines. 162 */ 163 for_each_cpu(cpu, cpu_online_mask) { 164 bufs[cpu] = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); 165 if (!bufs[cpu]) { 166 pr_warn("Allocation failure, not measuring misaligned performance\n"); 167 goto out; 168 } 169 } 170 171 on_each_cpu(_check_unaligned_access, bufs, 1); 172 173 out: 174 for_each_cpu(cpu, cpu_online_mask) { 175 if (bufs[cpu]) 176 __free_pages(bufs[cpu], MISALIGNED_BUFFER_ORDER); 177 } 178 179 kfree(bufs); 180 } 181 #else /* CONFIG_RISCV_PROBE_UNALIGNED_ACCESS */ 182 static void __init check_unaligned_access_speed_all_cpus(void) 183 { 184 } 185 #endif 186 187 DEFINE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key); 188 189 static void modify_unaligned_access_branches(const cpumask_t *mask) 190 { 191 bool fast = true; 192 int cpu; 193 194 for_each_cpu(cpu, mask) { 195 if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_FAST) { 196 fast = false; 197 break; 198 } 199 } 200 201 if (fast) 202 static_branch_enable_cpuslocked(&fast_unaligned_access_speed_key); 203 else 204 static_branch_disable_cpuslocked(&fast_unaligned_access_speed_key); 205 } 206 207 static int riscv_online_cpu(unsigned int cpu) 208 { 209 int ret = cpu_online_unaligned_access_init(cpu); 210 211 if (ret) 212 return ret; 213 214 /* We are already set since the last check */ 215 if (per_cpu(misaligned_access_speed, cpu) != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) { 216 goto exit; 217 } else if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) { 218 per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param; 219 goto exit; 220 } 221 222 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS 223 { 224 static struct page *buf; 225 226 buf = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); 227 if (!buf) { 228 pr_warn("Allocation failure, not measuring misaligned performance\n"); 229 return -ENOMEM; 230 } 231 232 check_unaligned_access(buf); 233 __free_pages(buf, MISALIGNED_BUFFER_ORDER); 234 } 235 #endif 236 237 exit: 238 modify_unaligned_access_branches(cpu_online_mask); 239 240 return 0; 241 } 242 243 static int riscv_offline_cpu(unsigned int cpu) 244 { 245 cpumask_t mask; 246 247 cpumask_copy(&mask, cpu_online_mask); 248 cpumask_clear_cpu(cpu, &mask); 249 250 modify_unaligned_access_branches(&mask); 251 252 return 0; 253 } 254 255 #ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS 256 static void check_vector_unaligned_access(struct work_struct *work __always_unused) 257 { 258 int cpu = smp_processor_id(); 259 struct page *page; 260 int ret; 261 262 if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) 263 return; 264 265 page = alloc_pages(GFP_KERNEL, MISALIGNED_BUFFER_ORDER); 266 if (!page) { 267 pr_warn("Allocation failure, not measuring vector misaligned performance\n"); 268 return; 269 } 270 271 kernel_vector_begin(); 272 273 ret = compare_unaligned_access(__riscv_copy_vec_words_unaligned, 274 __riscv_copy_vec_bytes_unaligned, 275 page_address(page), "vector"); 276 kernel_vector_end(); 277 278 if (ret < 0) 279 goto free; 280 281 if (ret) 282 per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; 283 else 284 per_cpu(vector_misaligned_access, cpu) = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; 285 286 free: 287 __free_pages(page, MISALIGNED_BUFFER_ORDER); 288 } 289 290 #else /* CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS */ 291 static void check_vector_unaligned_access(struct work_struct *work __always_unused) 292 { 293 } 294 #endif 295 296 static int riscv_online_cpu_vec(unsigned int cpu) 297 { 298 if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { 299 per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; 300 return 0; 301 } 302 303 #ifdef CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS 304 if (per_cpu(vector_misaligned_access, cpu) != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) 305 return 0; 306 307 check_vector_unaligned_access_emulated(NULL); 308 check_vector_unaligned_access(NULL); 309 #endif 310 311 return 0; 312 } 313 314 static const char * const speed_str[] __initconst = { NULL, NULL, "slow", "fast", "unsupported" }; 315 316 static int __init set_unaligned_scalar_speed_param(char *str) 317 { 318 if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW])) 319 unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_SLOW; 320 else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_FAST])) 321 unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_FAST; 322 else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED])) 323 unaligned_scalar_speed_param = RISCV_HWPROBE_MISALIGNED_SCALAR_UNSUPPORTED; 324 else 325 return -EINVAL; 326 327 return 1; 328 } 329 __setup("unaligned_scalar_speed=", set_unaligned_scalar_speed_param); 330 331 static int __init set_unaligned_vector_speed_param(char *str) 332 { 333 if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW])) 334 unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_SLOW; 335 else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_FAST])) 336 unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_FAST; 337 else if (!strcmp(str, speed_str[RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED])) 338 unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; 339 else 340 return -EINVAL; 341 342 return 1; 343 } 344 __setup("unaligned_vector_speed=", set_unaligned_vector_speed_param); 345 346 static int __init check_unaligned_access_all_cpus(void) 347 { 348 int cpu; 349 350 unaligned_access_init(); 351 352 if (unaligned_scalar_speed_param != RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN) { 353 pr_info("scalar unaligned access speed set to '%s' (%lu) by command line\n", 354 speed_str[unaligned_scalar_speed_param], unaligned_scalar_speed_param); 355 for_each_online_cpu(cpu) 356 per_cpu(misaligned_access_speed, cpu) = unaligned_scalar_speed_param; 357 } else if (!check_unaligned_access_emulated_all_cpus()) { 358 check_unaligned_access_speed_all_cpus(); 359 } 360 361 if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { 362 if (!has_vector() && 363 unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED) { 364 pr_warn("vector support is not available, ignoring unaligned_vector_speed=%s\n", 365 speed_str[unaligned_vector_speed_param]); 366 } else { 367 pr_info("vector unaligned access speed set to '%s' (%lu) by command line\n", 368 speed_str[unaligned_vector_speed_param], unaligned_vector_speed_param); 369 } 370 } 371 372 if (!has_vector()) 373 unaligned_vector_speed_param = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; 374 375 if (unaligned_vector_speed_param != RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { 376 for_each_online_cpu(cpu) 377 per_cpu(vector_misaligned_access, cpu) = unaligned_vector_speed_param; 378 } else if (!check_vector_unaligned_access_emulated_all_cpus() && 379 IS_ENABLED(CONFIG_RISCV_PROBE_VECTOR_UNALIGNED_ACCESS)) { 380 schedule_on_each_cpu(check_vector_unaligned_access); 381 } 382 383 /* 384 * Setup hotplug callbacks for any new CPUs that come online or go 385 * offline. 386 */ 387 cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", 388 riscv_online_cpu, riscv_offline_cpu); 389 cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "riscv:online", 390 riscv_online_cpu_vec, NULL); 391 392 cpus_read_lock(); 393 modify_unaligned_access_branches(cpu_online_mask); 394 cpus_read_unlock(); 395 396 return 0; 397 } 398 399 /* 400 * Run after clocksource_done_booting() so measure_cycles() uses a stable 401 * clocksource, but before rootfs_initcall() enables usermode helpers. Those 402 * helpers can reach hwprobe and populate the vDSO cache, so async hwprobe 403 * probes must be registered first. 404 */ 405 fs_initcall_sync(check_unaligned_access_all_cpus); 406