1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copied from arch/arm64/kernel/cpufeature.c
4 *
5 * Copyright (C) 2015 ARM Ltd.
6 * Copyright (C) 2017 SiFive
7 */
8
9 #include <linux/acpi.h>
10 #include <linux/bitmap.h>
11 #include <linux/cpu.h>
12 #include <linux/cpuhotplug.h>
13 #include <linux/ctype.h>
14 #include <linux/log2.h>
15 #include <linux/memory.h>
16 #include <linux/module.h>
17 #include <linux/of.h>
18 #include <asm/acpi.h>
19 #include <asm/alternative.h>
20 #include <asm/bugs.h>
21 #include <asm/cacheflush.h>
22 #include <asm/cpufeature.h>
23 #include <asm/hwcap.h>
24 #include <asm/text-patching.h>
25 #include <asm/hwprobe.h>
26 #include <asm/processor.h>
27 #include <asm/sbi.h>
28 #include <asm/vector.h>
29 #include <asm/vendor_extensions.h>
30 #include <asm/vendor_extensions/thead.h>
31 #include <asm/usercfi.h>
32
33 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
34
35 static bool any_cpu_has_zicboz;
36 static bool any_cpu_has_zicbop;
37 static bool any_cpu_has_zicbom;
38
39 unsigned long elf_hwcap __read_mostly;
40
41 /* Host ISA bitmap */
42 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
43
44 /* Per-cpu ISA extensions. */
45 struct riscv_isainfo hart_isa[NR_CPUS];
46
47 u32 thead_vlenb_of;
48
49 /**
50 * riscv_isa_extension_base() - Get base extension word
51 *
52 * @isa_bitmap: ISA bitmap to use
53 * Return: base extension word as unsigned long value
54 *
55 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
56 */
riscv_isa_extension_base(const unsigned long * isa_bitmap)57 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap)
58 {
59 return !isa_bitmap ? riscv_isa[0] : isa_bitmap[0];
60 }
61 EXPORT_SYMBOL_GPL(riscv_isa_extension_base);
62
63 /**
64 * __riscv_isa_extension_available() - Check whether given extension
65 * is available or not
66 *
67 * @isa_bitmap: ISA bitmap to use
68 * @bit: bit position of the desired extension
69 * Return: true or false
70 *
71 * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
72 */
__riscv_isa_extension_available(const unsigned long * isa_bitmap,unsigned int bit)73 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit)
74 {
75 const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa;
76
77 if (bit >= RISCV_ISA_EXT_MAX)
78 return false;
79
80 return test_bit(bit, bmap);
81 }
82 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
83
riscv_ext_f_depends(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)84 static int riscv_ext_f_depends(const struct riscv_isa_ext_data *data,
85 const unsigned long *isa_bitmap)
86 {
87 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F))
88 return 0;
89
90 return -EPROBE_DEFER;
91 }
92
riscv_ext_zicbom_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)93 static int riscv_ext_zicbom_validate(const struct riscv_isa_ext_data *data,
94 const unsigned long *isa_bitmap)
95 {
96 if (!riscv_cbom_block_size) {
97 pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n");
98 return -EINVAL;
99 }
100 if (!is_power_of_2(riscv_cbom_block_size)) {
101 pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n");
102 return -EINVAL;
103 }
104
105 any_cpu_has_zicbom = true;
106 return 0;
107 }
108
riscv_ext_zicboz_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)109 static int riscv_ext_zicboz_validate(const struct riscv_isa_ext_data *data,
110 const unsigned long *isa_bitmap)
111 {
112 if (!riscv_cboz_block_size) {
113 pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n");
114 return -EINVAL;
115 }
116 if (!is_power_of_2(riscv_cboz_block_size)) {
117 pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");
118 return -EINVAL;
119 }
120 any_cpu_has_zicboz = true;
121 return 0;
122 }
123
riscv_ext_zicbop_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)124 static int riscv_ext_zicbop_validate(const struct riscv_isa_ext_data *data,
125 const unsigned long *isa_bitmap)
126 {
127 if (!riscv_cbop_block_size) {
128 pr_err("Zicbop detected in ISA string, disabling as no cbop-block-size found\n");
129 return -EINVAL;
130 }
131 if (!is_power_of_2(riscv_cbop_block_size)) {
132 pr_err("Zicbop disabled as cbop-block-size present, but is not a power-of-2\n");
133 return -EINVAL;
134 }
135 any_cpu_has_zicbop = true;
136 return 0;
137 }
138
riscv_ext_f_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)139 static int riscv_ext_f_validate(const struct riscv_isa_ext_data *data,
140 const unsigned long *isa_bitmap)
141 {
142 if (!IS_ENABLED(CONFIG_FPU))
143 return -EINVAL;
144
145 /*
146 * Due to extension ordering, d is checked before f, so no deferral
147 * is required.
148 */
149 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D)) {
150 pr_warn_once("This kernel does not support systems with F but not D\n");
151 return -EINVAL;
152 }
153
154 return 0;
155 }
156
riscv_ext_d_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)157 static int riscv_ext_d_validate(const struct riscv_isa_ext_data *data,
158 const unsigned long *isa_bitmap)
159 {
160 if (!IS_ENABLED(CONFIG_FPU))
161 return -EINVAL;
162
163 return 0;
164 }
165
riscv_ext_vector_x_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)166 static int riscv_ext_vector_x_validate(const struct riscv_isa_ext_data *data,
167 const unsigned long *isa_bitmap)
168 {
169 if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
170 return -EINVAL;
171
172 return 0;
173 }
174
riscv_ext_vector_float_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)175 static int riscv_ext_vector_float_validate(const struct riscv_isa_ext_data *data,
176 const unsigned long *isa_bitmap)
177 {
178 if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
179 return -EINVAL;
180
181 if (!IS_ENABLED(CONFIG_FPU))
182 return -EINVAL;
183
184 /*
185 * The kernel doesn't support systems that don't implement both of
186 * F and D, so if any of the vector extensions that do floating point
187 * are to be usable, both floating point extensions need to be usable.
188 *
189 * Since this function validates vector only, and v/Zve* are probed
190 * after f/d, there's no need for a deferral here.
191 */
192 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D))
193 return -EINVAL;
194
195 return 0;
196 }
197
riscv_ext_vector_crypto_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)198 static int riscv_ext_vector_crypto_validate(const struct riscv_isa_ext_data *data,
199 const unsigned long *isa_bitmap)
200 {
201 if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
202 return -EINVAL;
203
204 /*
205 * It isn't the kernel's job to check that the binding is correct, so
206 * it should be enough to check that any of the vector extensions are
207 * enabled, which in-turn means that vector is usable in this kernel
208 */
209 if (!__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32X))
210 return -EPROBE_DEFER;
211
212 return 0;
213 }
214
riscv_ext_zca_depends(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)215 static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data,
216 const unsigned long *isa_bitmap)
217 {
218 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA))
219 return 0;
220
221 return -EPROBE_DEFER;
222 }
riscv_ext_zcd_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)223 static int riscv_ext_zcd_validate(const struct riscv_isa_ext_data *data,
224 const unsigned long *isa_bitmap)
225 {
226 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) &&
227 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_D))
228 return 0;
229
230 return -EPROBE_DEFER;
231 }
232
riscv_ext_zcf_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)233 static int riscv_ext_zcf_validate(const struct riscv_isa_ext_data *data,
234 const unsigned long *isa_bitmap)
235 {
236 if (IS_ENABLED(CONFIG_64BIT))
237 return -EINVAL;
238
239 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA) &&
240 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_F))
241 return 0;
242
243 return -EPROBE_DEFER;
244 }
245
riscv_ext_zilsd_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)246 static int riscv_ext_zilsd_validate(const struct riscv_isa_ext_data *data,
247 const unsigned long *isa_bitmap)
248 {
249 if (IS_ENABLED(CONFIG_64BIT))
250 return -EINVAL;
251
252 return 0;
253 }
254
riscv_ext_zclsd_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)255 static int riscv_ext_zclsd_validate(const struct riscv_isa_ext_data *data,
256 const unsigned long *isa_bitmap)
257 {
258 if (IS_ENABLED(CONFIG_64BIT))
259 return -EINVAL;
260
261 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZILSD) &&
262 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA))
263 return 0;
264
265 return -EPROBE_DEFER;
266 }
267
riscv_vector_f_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)268 static int riscv_vector_f_validate(const struct riscv_isa_ext_data *data,
269 const unsigned long *isa_bitmap)
270 {
271 if (!IS_ENABLED(CONFIG_RISCV_ISA_V))
272 return -EINVAL;
273
274 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVE32F))
275 return 0;
276
277 return -EPROBE_DEFER;
278 }
279
riscv_ext_zvfbfwma_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)280 static int riscv_ext_zvfbfwma_validate(const struct riscv_isa_ext_data *data,
281 const unsigned long *isa_bitmap)
282 {
283 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZFBFMIN) &&
284 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZVFBFMIN))
285 return 0;
286
287 return -EPROBE_DEFER;
288 }
289
riscv_ext_svadu_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)290 static int riscv_ext_svadu_validate(const struct riscv_isa_ext_data *data,
291 const unsigned long *isa_bitmap)
292 {
293 /* SVADE has already been detected, use SVADE only */
294 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SVADE))
295 return -EOPNOTSUPP;
296
297 return 0;
298 }
299
riscv_cfilp_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)300 static int riscv_cfilp_validate(const struct riscv_isa_ext_data *data,
301 const unsigned long *isa_bitmap)
302 {
303 if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) ||
304 (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_FCFI))
305 return -EINVAL;
306
307 return 0;
308 }
309
riscv_cfiss_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)310 static int riscv_cfiss_validate(const struct riscv_isa_ext_data *data,
311 const unsigned long *isa_bitmap)
312 {
313 if (!IS_ENABLED(CONFIG_RISCV_USER_CFI) ||
314 (riscv_nousercfi & CMDLINE_DISABLE_RISCV_USERCFI_BCFI))
315 return -EINVAL;
316
317 return 0;
318 }
319
320 static const unsigned int riscv_a_exts[] = {
321 RISCV_ISA_EXT_ZAAMO,
322 RISCV_ISA_EXT_ZALRSC,
323 };
324
325 #define RISCV_ISA_EXT_ZKN \
326 RISCV_ISA_EXT_ZBKB, \
327 RISCV_ISA_EXT_ZBKC, \
328 RISCV_ISA_EXT_ZBKX, \
329 RISCV_ISA_EXT_ZKND, \
330 RISCV_ISA_EXT_ZKNE, \
331 RISCV_ISA_EXT_ZKNH
332
riscv_ext_smcdeleg_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)333 static int riscv_ext_smcdeleg_validate(const struct riscv_isa_ext_data *data,
334 const unsigned long *isa_bitmap)
335 {
336 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SSCSRIND) &&
337 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZIHPM) &&
338 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZICNTR))
339 return 0;
340
341 return -EPROBE_DEFER;
342 }
343
riscv_ext_ssccfg_validate(const struct riscv_isa_ext_data * data,const unsigned long * isa_bitmap)344 static int riscv_ext_ssccfg_validate(const struct riscv_isa_ext_data *data,
345 const unsigned long *isa_bitmap)
346 {
347 if (!riscv_ext_smcdeleg_validate(data, isa_bitmap) &&
348 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_SMCDELEG))
349 return 0;
350
351 return -EPROBE_DEFER;
352 }
353
354 static const unsigned int riscv_zk_bundled_exts[] = {
355 RISCV_ISA_EXT_ZKN,
356 RISCV_ISA_EXT_ZKR,
357 RISCV_ISA_EXT_ZKT
358 };
359
360 static const unsigned int riscv_zkn_bundled_exts[] = {
361 RISCV_ISA_EXT_ZKN
362 };
363
364 static const unsigned int riscv_zks_bundled_exts[] = {
365 RISCV_ISA_EXT_ZBKB,
366 RISCV_ISA_EXT_ZBKC,
367 RISCV_ISA_EXT_ZKSED,
368 RISCV_ISA_EXT_ZKSH
369 };
370
371 #define RISCV_ISA_EXT_ZVKN \
372 RISCV_ISA_EXT_ZVKNED, \
373 RISCV_ISA_EXT_ZVKNHB, \
374 RISCV_ISA_EXT_ZVKB, \
375 RISCV_ISA_EXT_ZVKT
376
377 static const unsigned int riscv_zvkn_bundled_exts[] = {
378 RISCV_ISA_EXT_ZVKN
379 };
380
381 static const unsigned int riscv_zvknc_bundled_exts[] = {
382 RISCV_ISA_EXT_ZVKN,
383 RISCV_ISA_EXT_ZVBC
384 };
385
386 static const unsigned int riscv_zvkng_bundled_exts[] = {
387 RISCV_ISA_EXT_ZVKN,
388 RISCV_ISA_EXT_ZVKG
389 };
390
391 #define RISCV_ISA_EXT_ZVKS \
392 RISCV_ISA_EXT_ZVKSED, \
393 RISCV_ISA_EXT_ZVKSH, \
394 RISCV_ISA_EXT_ZVKB, \
395 RISCV_ISA_EXT_ZVKT
396
397 static const unsigned int riscv_zvks_bundled_exts[] = {
398 RISCV_ISA_EXT_ZVKS
399 };
400
401 static const unsigned int riscv_zvksc_bundled_exts[] = {
402 RISCV_ISA_EXT_ZVKS,
403 RISCV_ISA_EXT_ZVBC
404 };
405
406 static const unsigned int riscv_zvksg_bundled_exts[] = {
407 RISCV_ISA_EXT_ZVKS,
408 RISCV_ISA_EXT_ZVKG
409 };
410
411 static const unsigned int riscv_zvbb_exts[] = {
412 RISCV_ISA_EXT_ZVKB
413 };
414
415 /*
416 * The RISC-V ISA manual specifies that Zfh implies Zfhmin and Zvfh implies
417 * Zvfhmin. Report the implied subset extensions whenever the supersets are
418 * detected (see https://github.com/riscv/riscv-isa-manual/pull/3070).
419 */
420 static const unsigned int riscv_zfh_exts[] = {
421 RISCV_ISA_EXT_ZFHMIN
422 };
423
424 static const unsigned int riscv_zvfh_exts[] = {
425 RISCV_ISA_EXT_ZVFHMIN
426 };
427
428 #define RISCV_ISA_EXT_ZVE64F_IMPLY_LIST \
429 RISCV_ISA_EXT_ZVE64X, \
430 RISCV_ISA_EXT_ZVE32F, \
431 RISCV_ISA_EXT_ZVE32X
432
433 #define RISCV_ISA_EXT_ZVE64D_IMPLY_LIST \
434 RISCV_ISA_EXT_ZVE64F, \
435 RISCV_ISA_EXT_ZVE64F_IMPLY_LIST
436
437 #define RISCV_ISA_EXT_V_IMPLY_LIST \
438 RISCV_ISA_EXT_ZVE64D, \
439 RISCV_ISA_EXT_ZVE64D_IMPLY_LIST
440
441 static const unsigned int riscv_zve32f_exts[] = {
442 RISCV_ISA_EXT_ZVE32X
443 };
444
445 static const unsigned int riscv_zve64f_exts[] = {
446 RISCV_ISA_EXT_ZVE64F_IMPLY_LIST
447 };
448
449 static const unsigned int riscv_zve64d_exts[] = {
450 RISCV_ISA_EXT_ZVE64D_IMPLY_LIST
451 };
452
453 static const unsigned int riscv_v_exts[] = {
454 RISCV_ISA_EXT_V_IMPLY_LIST
455 };
456
457 static const unsigned int riscv_zve64x_exts[] = {
458 RISCV_ISA_EXT_ZVE32X,
459 RISCV_ISA_EXT_ZVE64X
460 };
461
462 /*
463 * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V
464 * privileged ISA, the existence of the CSRs is implied by any extension which
465 * specifies [ms]envcfg bit(s). Hence, we define a custom ISA extension for the
466 * existence of the CSR, and treat it as a subset of those other extensions.
467 */
468 static const unsigned int riscv_xlinuxenvcfg_exts[] = {
469 RISCV_ISA_EXT_XLINUXENVCFG
470 };
471
472 /*
473 * Zc* spec states that:
474 * - C always implies Zca
475 * - C+F implies Zcf (RV32 only)
476 * - C+D implies Zcd
477 *
478 * These extensions will be enabled and then validated depending on the
479 * availability of F/D RV32.
480 */
481 static const unsigned int riscv_c_exts[] = {
482 RISCV_ISA_EXT_ZCA,
483 RISCV_ISA_EXT_ZCF,
484 RISCV_ISA_EXT_ZCD,
485 };
486
487 /*
488 * The canonical order of ISA extension names in the ISA string is defined in
489 * Chapter 27 of the RISC-V Instruction Set Manual Volume I Unprivileged ISA
490 * (Document Version 20191213).
491 *
492 * Ordinarily, for in-kernel data structures, this order is unimportant but
493 * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo.
494 *
495 * The specification uses vague wording, such as should, when it comes to
496 * ordering, so for our purposes the following rules apply:
497 *
498 * 1. All multi-letter extensions must be separated from other extensions by an
499 * underscore.
500 *
501 * 2. Additional standard extensions (starting with 'Z') must be sorted after
502 * single-letter extensions and before any higher-privileged extensions.
503 *
504 * 3. The first letter following the 'Z' conventionally indicates the most
505 * closely related alphabetical extension category, IMAFDQLCBKJTPVH.
506 * If multiple 'Z' extensions are named, they must be ordered first by
507 * category, then alphabetically within a category.
508 *
509 * 3. Standard supervisor-level extensions (starting with 'S') must be listed
510 * after standard unprivileged extensions. If multiple supervisor-level
511 * extensions are listed, they must be ordered alphabetically.
512 *
513 * 4. Standard machine-level extensions (starting with 'Zxm') must be listed
514 * after any lower-privileged, standard extensions. If multiple
515 * machine-level extensions are listed, they must be ordered
516 * alphabetically.
517 *
518 * 5. Non-standard extensions (starting with 'X') must be listed after all
519 * standard extensions. If multiple non-standard extensions are listed, they
520 * must be ordered alphabetically.
521 *
522 * An example string following the order is:
523 * rv64imadc_zifoo_zigoo_zafoo_sbar_scar_zxmbaz_xqux_xrux
524 *
525 * New entries to this struct should follow the ordering rules described above.
526 */
527 const struct riscv_isa_ext_data riscv_isa_ext[] = {
528 __RISCV_ISA_EXT_DATA(i, RISCV_ISA_EXT_I),
529 __RISCV_ISA_EXT_DATA(m, RISCV_ISA_EXT_M),
530 __RISCV_ISA_EXT_SUPERSET(a, RISCV_ISA_EXT_A, riscv_a_exts),
531 __RISCV_ISA_EXT_DATA_VALIDATE(f, RISCV_ISA_EXT_F, riscv_ext_f_validate),
532 __RISCV_ISA_EXT_DATA_VALIDATE(d, RISCV_ISA_EXT_D, riscv_ext_d_validate),
533 __RISCV_ISA_EXT_DATA(q, RISCV_ISA_EXT_Q),
534 __RISCV_ISA_EXT_SUPERSET(c, RISCV_ISA_EXT_C, riscv_c_exts),
535 __RISCV_ISA_EXT_SUPERSET_VALIDATE(v, RISCV_ISA_EXT_V, riscv_v_exts,
536 riscv_ext_vector_float_validate),
537 __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_H),
538 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts, riscv_ext_zicbom_validate),
539 __RISCV_ISA_EXT_DATA_VALIDATE(zicbop, RISCV_ISA_EXT_ZICBOP, riscv_ext_zicbop_validate),
540 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts, riscv_ext_zicboz_validate),
541 __RISCV_ISA_EXT_DATA(ziccamoa, RISCV_ISA_EXT_ZICCAMOA),
542 __RISCV_ISA_EXT_DATA(ziccif, RISCV_ISA_EXT_ZICCIF),
543 __RISCV_ISA_EXT_DATA(zicclsm, RISCV_ISA_EXT_ZICCLSM),
544 __RISCV_ISA_EXT_DATA(ziccrse, RISCV_ISA_EXT_ZICCRSE),
545 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfilp, RISCV_ISA_EXT_ZICFILP, riscv_xlinuxenvcfg_exts,
546 riscv_cfilp_validate),
547 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zicfiss, RISCV_ISA_EXT_ZICFISS, riscv_xlinuxenvcfg_exts,
548 riscv_cfiss_validate),
549 __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR),
550 __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND),
551 __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR),
552 __RISCV_ISA_EXT_DATA(zifencei, RISCV_ISA_EXT_ZIFENCEI),
553 __RISCV_ISA_EXT_DATA(zihintntl, RISCV_ISA_EXT_ZIHINTNTL),
554 __RISCV_ISA_EXT_DATA(zihintpause, RISCV_ISA_EXT_ZIHINTPAUSE),
555 __RISCV_ISA_EXT_DATA(zihpm, RISCV_ISA_EXT_ZIHPM),
556 __RISCV_ISA_EXT_DATA(zimop, RISCV_ISA_EXT_ZIMOP),
557 __RISCV_ISA_EXT_DATA(za64rs, RISCV_ISA_EXT_ZA64RS),
558 __RISCV_ISA_EXT_DATA(zaamo, RISCV_ISA_EXT_ZAAMO),
559 __RISCV_ISA_EXT_DATA(zabha, RISCV_ISA_EXT_ZABHA),
560 __RISCV_ISA_EXT_DATA(zacas, RISCV_ISA_EXT_ZACAS),
561 __RISCV_ISA_EXT_DATA(zalasr, RISCV_ISA_EXT_ZALASR),
562 __RISCV_ISA_EXT_DATA(zalrsc, RISCV_ISA_EXT_ZALRSC),
563 __RISCV_ISA_EXT_DATA(zawrs, RISCV_ISA_EXT_ZAWRS),
564 __RISCV_ISA_EXT_DATA_VALIDATE(zfa, RISCV_ISA_EXT_ZFA, riscv_ext_f_depends),
565 __RISCV_ISA_EXT_DATA_VALIDATE(zfbfmin, RISCV_ISA_EXT_ZFBFMIN, riscv_ext_f_depends),
566 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zfh, RISCV_ISA_EXT_ZFH,
567 riscv_zfh_exts, riscv_ext_f_depends),
568 __RISCV_ISA_EXT_DATA_VALIDATE(zfhmin, RISCV_ISA_EXT_ZFHMIN, riscv_ext_f_depends),
569 __RISCV_ISA_EXT_DATA(zca, RISCV_ISA_EXT_ZCA),
570 __RISCV_ISA_EXT_DATA_VALIDATE(zcb, RISCV_ISA_EXT_ZCB, riscv_ext_zca_depends),
571 __RISCV_ISA_EXT_DATA_VALIDATE(zcd, RISCV_ISA_EXT_ZCD, riscv_ext_zcd_validate),
572 __RISCV_ISA_EXT_DATA_VALIDATE(zcf, RISCV_ISA_EXT_ZCF, riscv_ext_zcf_validate),
573 __RISCV_ISA_EXT_DATA_VALIDATE(zcmop, RISCV_ISA_EXT_ZCMOP, riscv_ext_zca_depends),
574 __RISCV_ISA_EXT_DATA_VALIDATE(zclsd, RISCV_ISA_EXT_ZCLSD, riscv_ext_zclsd_validate),
575 __RISCV_ISA_EXT_DATA_VALIDATE(zilsd, RISCV_ISA_EXT_ZILSD, riscv_ext_zilsd_validate),
576 __RISCV_ISA_EXT_DATA(zba, RISCV_ISA_EXT_ZBA),
577 __RISCV_ISA_EXT_DATA(zbb, RISCV_ISA_EXT_ZBB),
578 __RISCV_ISA_EXT_DATA(zbc, RISCV_ISA_EXT_ZBC),
579 __RISCV_ISA_EXT_DATA(zbkb, RISCV_ISA_EXT_ZBKB),
580 __RISCV_ISA_EXT_DATA(zbkc, RISCV_ISA_EXT_ZBKC),
581 __RISCV_ISA_EXT_DATA(zbkx, RISCV_ISA_EXT_ZBKX),
582 __RISCV_ISA_EXT_DATA(zbs, RISCV_ISA_EXT_ZBS),
583 __RISCV_ISA_EXT_BUNDLE(zk, riscv_zk_bundled_exts),
584 __RISCV_ISA_EXT_BUNDLE(zkn, riscv_zkn_bundled_exts),
585 __RISCV_ISA_EXT_DATA(zknd, RISCV_ISA_EXT_ZKND),
586 __RISCV_ISA_EXT_DATA(zkne, RISCV_ISA_EXT_ZKNE),
587 __RISCV_ISA_EXT_DATA(zknh, RISCV_ISA_EXT_ZKNH),
588 __RISCV_ISA_EXT_DATA(zkr, RISCV_ISA_EXT_ZKR),
589 __RISCV_ISA_EXT_BUNDLE(zks, riscv_zks_bundled_exts),
590 __RISCV_ISA_EXT_DATA(zkt, RISCV_ISA_EXT_ZKT),
591 __RISCV_ISA_EXT_DATA(zksed, RISCV_ISA_EXT_ZKSED),
592 __RISCV_ISA_EXT_DATA(zksh, RISCV_ISA_EXT_ZKSH),
593 __RISCV_ISA_EXT_DATA(ztso, RISCV_ISA_EXT_ZTSO),
594 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvbb, RISCV_ISA_EXT_ZVBB, riscv_zvbb_exts, riscv_ext_vector_crypto_validate),
595 __RISCV_ISA_EXT_DATA_VALIDATE(zvbc, RISCV_ISA_EXT_ZVBC, riscv_ext_vector_crypto_validate),
596 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve32f, RISCV_ISA_EXT_ZVE32F, riscv_zve32f_exts, riscv_ext_vector_float_validate),
597 __RISCV_ISA_EXT_DATA_VALIDATE(zve32x, RISCV_ISA_EXT_ZVE32X, riscv_ext_vector_x_validate),
598 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64d, RISCV_ISA_EXT_ZVE64D, riscv_zve64d_exts, riscv_ext_vector_float_validate),
599 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64f, RISCV_ISA_EXT_ZVE64F, riscv_zve64f_exts, riscv_ext_vector_float_validate),
600 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zve64x, RISCV_ISA_EXT_ZVE64X, riscv_zve64x_exts, riscv_ext_vector_x_validate),
601 __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfmin, RISCV_ISA_EXT_ZVFBFMIN, riscv_vector_f_validate),
602 __RISCV_ISA_EXT_DATA_VALIDATE(zvfbfwma, RISCV_ISA_EXT_ZVFBFWMA, riscv_ext_zvfbfwma_validate),
603 __RISCV_ISA_EXT_SUPERSET_VALIDATE(zvfh, RISCV_ISA_EXT_ZVFH,
604 riscv_zvfh_exts,
605 riscv_ext_vector_float_validate),
606 __RISCV_ISA_EXT_DATA(zvfhmin, RISCV_ISA_EXT_ZVFHMIN),
607 __RISCV_ISA_EXT_DATA_VALIDATE(zvkb, RISCV_ISA_EXT_ZVKB, riscv_ext_vector_crypto_validate),
608 __RISCV_ISA_EXT_DATA_VALIDATE(zvkg, RISCV_ISA_EXT_ZVKG, riscv_ext_vector_crypto_validate),
609 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkn, riscv_zvkn_bundled_exts, riscv_ext_vector_crypto_validate),
610 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvknc, riscv_zvknc_bundled_exts, riscv_ext_vector_crypto_validate),
611 __RISCV_ISA_EXT_DATA_VALIDATE(zvkned, RISCV_ISA_EXT_ZVKNED, riscv_ext_vector_crypto_validate),
612 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvkng, riscv_zvkng_bundled_exts, riscv_ext_vector_crypto_validate),
613 __RISCV_ISA_EXT_DATA_VALIDATE(zvknha, RISCV_ISA_EXT_ZVKNHA, riscv_ext_vector_crypto_validate),
614 __RISCV_ISA_EXT_DATA_VALIDATE(zvknhb, RISCV_ISA_EXT_ZVKNHB, riscv_ext_vector_crypto_validate),
615 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvks, riscv_zvks_bundled_exts, riscv_ext_vector_crypto_validate),
616 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksc, riscv_zvksc_bundled_exts, riscv_ext_vector_crypto_validate),
617 __RISCV_ISA_EXT_DATA_VALIDATE(zvksed, RISCV_ISA_EXT_ZVKSED, riscv_ext_vector_crypto_validate),
618 __RISCV_ISA_EXT_DATA_VALIDATE(zvksh, RISCV_ISA_EXT_ZVKSH, riscv_ext_vector_crypto_validate),
619 __RISCV_ISA_EXT_BUNDLE_VALIDATE(zvksg, riscv_zvksg_bundled_exts, riscv_ext_vector_crypto_validate),
620 __RISCV_ISA_EXT_DATA_VALIDATE(zvkt, RISCV_ISA_EXT_ZVKT, riscv_ext_vector_crypto_validate),
621 __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA),
622 __RISCV_ISA_EXT_DATA_VALIDATE(smcdeleg, RISCV_ISA_EXT_SMCDELEG,
623 riscv_ext_smcdeleg_validate),
624 __RISCV_ISA_EXT_DATA(smcntrpmf, RISCV_ISA_EXT_SMCNTRPMF),
625 __RISCV_ISA_EXT_DATA(smcsrind, RISCV_ISA_EXT_SMCSRIND),
626 __RISCV_ISA_EXT_DATA(smmpm, RISCV_ISA_EXT_SMMPM),
627 __RISCV_ISA_EXT_SUPERSET(smnpm, RISCV_ISA_EXT_SMNPM, riscv_xlinuxenvcfg_exts),
628 __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN),
629 __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA),
630 __RISCV_ISA_EXT_DATA_VALIDATE(ssccfg, RISCV_ISA_EXT_SSCCFG, riscv_ext_ssccfg_validate),
631 __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
632 __RISCV_ISA_EXT_DATA(sscsrind, RISCV_ISA_EXT_SSCSRIND),
633 __RISCV_ISA_EXT_SUPERSET(ssnpm, RISCV_ISA_EXT_SSNPM, riscv_xlinuxenvcfg_exts),
634 __RISCV_ISA_EXT_DATA(ssqosid, RISCV_ISA_EXT_SSQOSID),
635 __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),
636 __RISCV_ISA_EXT_DATA(svade, RISCV_ISA_EXT_SVADE),
637 __RISCV_ISA_EXT_DATA_VALIDATE(svadu, RISCV_ISA_EXT_SVADU, riscv_ext_svadu_validate),
638 __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
639 __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),
640 __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT),
641 __RISCV_ISA_EXT_DATA(svrsw60t59b, RISCV_ISA_EXT_SVRSW60T59B),
642 __RISCV_ISA_EXT_DATA(svvptc, RISCV_ISA_EXT_SVVPTC),
643 };
644
645 const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext);
646
riscv_isa_set_ext(const struct riscv_isa_ext_data * ext,unsigned long * bitmap)647 static void riscv_isa_set_ext(const struct riscv_isa_ext_data *ext, unsigned long *bitmap)
648 {
649 if (ext->id != RISCV_ISA_EXT_INVALID)
650 set_bit(ext->id, bitmap);
651
652 for (int i = 0; i < ext->subset_ext_size; i++) {
653 if (ext->subset_ext_ids[i] != RISCV_ISA_EXT_INVALID)
654 set_bit(ext->subset_ext_ids[i], bitmap);
655 }
656 }
657
riscv_get_isa_ext_data(unsigned int ext_id)658 static const struct riscv_isa_ext_data *riscv_get_isa_ext_data(unsigned int ext_id)
659 {
660 for (int i = 0; i < riscv_isa_ext_count; i++) {
661 if (riscv_isa_ext[i].id == ext_id)
662 return &riscv_isa_ext[i];
663 }
664
665 return NULL;
666 }
667
668 /*
669 * "Resolve" a source ISA bitmap into one that matches kernel configuration as
670 * well as correct extension dependencies. Some extensions depends on specific
671 * kernel configuration to be usable (V needs CONFIG_RISCV_ISA_V for instance)
672 * and this function will actually validate all the extensions provided in
673 * source_isa into the resolved_isa based on extensions validate() callbacks.
674 */
riscv_resolve_isa(unsigned long * source_isa,unsigned long * resolved_isa,unsigned long * this_hwcap,unsigned long * isa2hwcap)675 static void __init riscv_resolve_isa(unsigned long *source_isa,
676 unsigned long *resolved_isa, unsigned long *this_hwcap,
677 unsigned long *isa2hwcap)
678 {
679 bool loop;
680 const struct riscv_isa_ext_data *ext;
681 DECLARE_BITMAP(prev_resolved_isa, RISCV_ISA_EXT_MAX);
682 int max_loop_count = riscv_isa_ext_count, ret;
683 unsigned int bit;
684
685 do {
686 loop = false;
687 if (max_loop_count-- < 0) {
688 pr_err("Failed to reach a stable ISA state\n");
689 return;
690 }
691 bitmap_copy(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX);
692 for_each_set_bit(bit, source_isa, RISCV_ISA_EXT_MAX) {
693 ext = riscv_get_isa_ext_data(bit);
694
695 if (ext && ext->validate) {
696 ret = ext->validate(ext, resolved_isa);
697 if (ret == -EPROBE_DEFER) {
698 loop = true;
699 continue;
700 } else if (ret) {
701 /* Disable the extension entirely */
702 clear_bit(bit, source_isa);
703 continue;
704 }
705 }
706
707 set_bit(bit, resolved_isa);
708 /* No need to keep it in source isa now that it is enabled */
709 clear_bit(bit, source_isa);
710
711 /* Single letter extensions get set in hwcap */
712 if (bit < RISCV_ISA_EXT_BASE)
713 *this_hwcap |= isa2hwcap[bit];
714 }
715 } while (loop && !bitmap_equal(prev_resolved_isa, resolved_isa, RISCV_ISA_EXT_MAX));
716 }
717
match_isa_ext(const char * name,const char * name_end,unsigned long * bitmap)718 static void __init match_isa_ext(const char *name, const char *name_end, unsigned long *bitmap)
719 {
720 for (int i = 0; i < riscv_isa_ext_count; i++) {
721 const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i];
722
723 if ((name_end - name == strlen(ext->name)) &&
724 !strncasecmp(name, ext->name, name_end - name)) {
725 riscv_isa_set_ext(ext, bitmap);
726 break;
727 }
728 }
729 }
730
riscv_parse_isa_string(const char * isa,unsigned long * bitmap)731 static void __init riscv_parse_isa_string(const char *isa, unsigned long *bitmap)
732 {
733 /*
734 * For all possible cpus, we have already validated in
735 * the boot process that they at least contain "rv" and
736 * whichever of "32"/"64" this kernel supports, and so this
737 * section can be skipped.
738 */
739 isa += 4;
740
741 while (*isa) {
742 const char *ext = isa++;
743 const char *ext_end = isa;
744 bool ext_err = false;
745
746 switch (*ext) {
747 case 'x':
748 case 'X':
749 if (acpi_disabled)
750 pr_warn_once("Vendor extensions are ignored in riscv,isa. Use riscv,isa-extensions instead.");
751 /*
752 * To skip an extension, we find its end.
753 * As multi-letter extensions must be split from other multi-letter
754 * extensions with an "_", the end of a multi-letter extension will
755 * either be the null character or the "_" at the start of the next
756 * multi-letter extension.
757 */
758 for (; *isa && *isa != '_'; ++isa)
759 ;
760 ext_err = true;
761 break;
762 case 's':
763 /*
764 * Workaround for invalid single-letter 's' & 'u' (QEMU).
765 * No need to set the bit in riscv_isa as 's' & 'u' are
766 * not valid ISA extensions. It works unless the first
767 * multi-letter extension in the ISA string begins with
768 * "Su" and is not prefixed with an underscore.
769 */
770 if (ext[-1] != '_' && ext[1] == 'u') {
771 ++isa;
772 ext_err = true;
773 break;
774 }
775 fallthrough;
776 case 'S':
777 case 'z':
778 case 'Z':
779 /*
780 * Before attempting to parse the extension itself, we find its end.
781 * As multi-letter extensions must be split from other multi-letter
782 * extensions with an "_", the end of a multi-letter extension will
783 * either be the null character or the "_" at the start of the next
784 * multi-letter extension.
785 *
786 * Next, as the extensions version is currently ignored, we
787 * eliminate that portion. This is done by parsing backwards from
788 * the end of the extension, removing any numbers. This may be a
789 * major or minor number however, so the process is repeated if a
790 * minor number was found.
791 *
792 * ext_end is intended to represent the first character *after* the
793 * name portion of an extension, but will be decremented to the last
794 * character itself while eliminating the extensions version number.
795 * A simple re-increment solves this problem.
796 */
797 for (; *isa && *isa != '_'; ++isa)
798 if (unlikely(!isalnum(*isa)))
799 ext_err = true;
800
801 ext_end = isa;
802 if (unlikely(ext_err))
803 break;
804
805 if (!isdigit(ext_end[-1]))
806 break;
807
808 while (isdigit(*--ext_end))
809 ;
810
811 if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) {
812 ++ext_end;
813 break;
814 }
815
816 while (isdigit(*--ext_end))
817 ;
818
819 ++ext_end;
820 break;
821 default:
822 /*
823 * Things are a little easier for single-letter extensions, as they
824 * are parsed forwards.
825 *
826 * After checking that our starting position is valid, we need to
827 * ensure that, when isa was incremented at the start of the loop,
828 * that it arrived at the start of the next extension.
829 *
830 * If we are already on a non-digit, there is nothing to do. Either
831 * we have a multi-letter extension's _, or the start of an
832 * extension.
833 *
834 * Otherwise we have found the current extension's major version
835 * number. Parse past it, and a subsequent p/minor version number
836 * if present. The `p` extension must not appear immediately after
837 * a number, so there is no fear of missing it.
838 *
839 */
840 if (unlikely(!isalpha(*ext))) {
841 ext_err = true;
842 break;
843 }
844
845 if (!isdigit(*isa))
846 break;
847
848 while (isdigit(*++isa))
849 ;
850
851 if (tolower(*isa) != 'p')
852 break;
853
854 if (!isdigit(*++isa)) {
855 --isa;
856 break;
857 }
858
859 while (isdigit(*++isa))
860 ;
861
862 break;
863 }
864
865 /*
866 * The parser expects that at the start of an iteration isa points to the
867 * first character of the next extension. As we stop parsing an extension
868 * on meeting a non-alphanumeric character, an extra increment is needed
869 * where the succeeding extension is a multi-letter prefixed with an "_".
870 */
871 if (*isa == '_')
872 ++isa;
873
874 if (unlikely(ext_err))
875 continue;
876
877 match_isa_ext(ext, ext_end, bitmap);
878 }
879 }
880
riscv_fill_hwcap_from_isa_string(unsigned long * isa2hwcap)881 static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap)
882 {
883 struct device_node *node;
884 const char *isa;
885 int rc;
886 struct acpi_table_header *rhct;
887 acpi_status status;
888 unsigned int cpu;
889 u64 boot_vendorid;
890 u64 boot_archid;
891
892 if (!acpi_disabled) {
893 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
894 if (ACPI_FAILURE(status))
895 return;
896 }
897
898 boot_vendorid = riscv_get_mvendorid();
899 boot_archid = riscv_get_marchid();
900
901 for_each_possible_cpu(cpu) {
902 struct riscv_isainfo *isainfo = &hart_isa[cpu];
903 unsigned long this_hwcap = 0;
904 DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 };
905
906 if (acpi_disabled) {
907 node = of_cpu_device_node_get(cpu);
908 if (!node) {
909 pr_warn("Unable to find cpu node\n");
910 continue;
911 }
912
913 rc = of_property_read_string(node, "riscv,isa", &isa);
914 of_node_put(node);
915 if (rc) {
916 pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");
917 continue;
918 }
919 } else {
920 rc = acpi_get_riscv_isa(rhct, cpu, &isa);
921 if (rc < 0) {
922 pr_warn("Unable to get ISA for the hart - %d\n", cpu);
923 continue;
924 }
925 }
926
927 riscv_parse_isa_string(isa, source_isa);
928
929 /*
930 * These ones were as they were part of the base ISA when the
931 * port & dt-bindings were upstreamed, and so can be set
932 * unconditionally where `i` is in riscv,isa on DT systems.
933 */
934 if (acpi_disabled) {
935 set_bit(RISCV_ISA_EXT_ZICSR, source_isa);
936 set_bit(RISCV_ISA_EXT_ZIFENCEI, source_isa);
937 set_bit(RISCV_ISA_EXT_ZICNTR, source_isa);
938 set_bit(RISCV_ISA_EXT_ZIHPM, source_isa);
939 }
940
941 /*
942 * "V" in ISA strings is ambiguous in practice: it should mean
943 * just the standard V-1.0 but vendors aren't well behaved.
944 * Many vendors with T-Head CPU cores which implement the 0.7.1
945 * version of the vector specification put "v" into their DTs.
946 * CPU cores with the ratified spec will contain non-zero
947 * marchid.
948 */
949 if (acpi_disabled && boot_vendorid == THEAD_VENDOR_ID && boot_archid == 0x0)
950 clear_bit(RISCV_ISA_EXT_V, source_isa);
951
952 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);
953
954 /*
955 * All "okay" hart should have same isa. Set HWCAP based on
956 * common capabilities of every "okay" hart, in case they don't
957 * have.
958 */
959 if (elf_hwcap)
960 elf_hwcap &= this_hwcap;
961 else
962 elf_hwcap = this_hwcap;
963
964 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
965 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
966 else
967 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
968 }
969
970 if (!acpi_disabled && rhct)
971 acpi_put_table((struct acpi_table_header *)rhct);
972 }
973
riscv_fill_cpu_vendor_ext(struct device_node * cpu_node,int cpu)974 static void __init riscv_fill_cpu_vendor_ext(struct device_node *cpu_node, int cpu)
975 {
976 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT))
977 return;
978
979 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) {
980 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i];
981
982 for (int j = 0; j < ext_list->ext_data_count; j++) {
983 const struct riscv_isa_ext_data ext = ext_list->ext_data[j];
984 struct riscv_isavendorinfo *isavendorinfo = &ext_list->per_hart_isa_bitmap[cpu];
985
986 if (of_property_match_string(cpu_node, "riscv,isa-extensions",
987 ext.property) < 0)
988 continue;
989
990 /*
991 * Assume that subset extensions are all members of the
992 * same vendor.
993 */
994 if (ext.subset_ext_size)
995 for (int k = 0; k < ext.subset_ext_size; k++)
996 set_bit(ext.subset_ext_ids[k], isavendorinfo->isa);
997
998 set_bit(ext.id, isavendorinfo->isa);
999 }
1000 }
1001 }
1002
1003 /*
1004 * Populate all_harts_isa_bitmap for each vendor with all of the extensions that
1005 * are shared across CPUs for that vendor.
1006 */
riscv_fill_vendor_ext_list(int cpu)1007 static void __init riscv_fill_vendor_ext_list(int cpu)
1008 {
1009 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT))
1010 return;
1011
1012 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) {
1013 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i];
1014
1015 if (!ext_list->is_initialized) {
1016 bitmap_copy(ext_list->all_harts_isa_bitmap.isa,
1017 ext_list->per_hart_isa_bitmap[cpu].isa,
1018 RISCV_ISA_VENDOR_EXT_MAX);
1019 ext_list->is_initialized = true;
1020 } else {
1021 bitmap_and(ext_list->all_harts_isa_bitmap.isa,
1022 ext_list->all_harts_isa_bitmap.isa,
1023 ext_list->per_hart_isa_bitmap[cpu].isa,
1024 RISCV_ISA_VENDOR_EXT_MAX);
1025 }
1026 }
1027 }
1028
has_thead_homogeneous_vlenb(void)1029 static int has_thead_homogeneous_vlenb(void)
1030 {
1031 int cpu;
1032 u32 prev_vlenb = 0;
1033 u32 vlenb = 0;
1034
1035 /* Ignore thead,vlenb property if xtheadvector is not enabled in the kernel */
1036 if (!IS_ENABLED(CONFIG_RISCV_ISA_XTHEADVECTOR))
1037 return 0;
1038
1039 for_each_possible_cpu(cpu) {
1040 struct device_node *cpu_node;
1041
1042 cpu_node = of_cpu_device_node_get(cpu);
1043 if (!cpu_node) {
1044 pr_warn("Unable to find cpu node\n");
1045 return -ENOENT;
1046 }
1047
1048 if (of_property_read_u32(cpu_node, "thead,vlenb", &vlenb)) {
1049 of_node_put(cpu_node);
1050
1051 if (prev_vlenb)
1052 return -ENOENT;
1053 continue;
1054 }
1055
1056 if (prev_vlenb && vlenb != prev_vlenb) {
1057 of_node_put(cpu_node);
1058 return -ENOENT;
1059 }
1060
1061 prev_vlenb = vlenb;
1062 of_node_put(cpu_node);
1063 }
1064
1065 thead_vlenb_of = vlenb;
1066 return 0;
1067 }
1068
riscv_fill_hwcap_from_ext_list(unsigned long * isa2hwcap)1069 static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap)
1070 {
1071 unsigned int cpu;
1072 bool mitigated;
1073
1074 for_each_possible_cpu(cpu) {
1075 unsigned long this_hwcap = 0;
1076 struct device_node *cpu_node;
1077 struct riscv_isainfo *isainfo = &hart_isa[cpu];
1078 DECLARE_BITMAP(source_isa, RISCV_ISA_EXT_MAX) = { 0 };
1079
1080 cpu_node = of_cpu_device_node_get(cpu);
1081 if (!cpu_node) {
1082 pr_warn("Unable to find cpu node\n");
1083 continue;
1084 }
1085
1086 if (!of_property_present(cpu_node, "riscv,isa-extensions")) {
1087 of_node_put(cpu_node);
1088 continue;
1089 }
1090
1091 for (int i = 0; i < riscv_isa_ext_count; i++) {
1092 const struct riscv_isa_ext_data *ext = &riscv_isa_ext[i];
1093
1094 if (of_property_match_string(cpu_node, "riscv,isa-extensions",
1095 ext->property) < 0)
1096 continue;
1097
1098 riscv_isa_set_ext(ext, source_isa);
1099 }
1100
1101 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);
1102 riscv_fill_cpu_vendor_ext(cpu_node, cpu);
1103
1104 of_node_put(cpu_node);
1105
1106 /*
1107 * All "okay" harts should have same isa. Set HWCAP based on
1108 * common capabilities of every "okay" hart, in case they don't.
1109 */
1110 if (elf_hwcap)
1111 elf_hwcap &= this_hwcap;
1112 else
1113 elf_hwcap = this_hwcap;
1114
1115 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
1116 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
1117 else
1118 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
1119
1120 riscv_fill_vendor_ext_list(cpu);
1121 }
1122
1123 /*
1124 * Execute ghostwrite mitigation immediately after detecting extensions
1125 * to disable xtheadvector if necessary.
1126 */
1127 mitigated = ghostwrite_enable_mitigation();
1128
1129 if (!mitigated && has_xtheadvector_no_alternatives() && has_thead_homogeneous_vlenb() < 0) {
1130 pr_warn("Unsupported heterogeneous vlenb detected, vector extension disabled.\n");
1131 disable_xtheadvector();
1132 }
1133
1134 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
1135 return -ENOENT;
1136
1137 return 0;
1138 }
1139
1140 #ifdef CONFIG_RISCV_ISA_FALLBACK
1141 bool __initdata riscv_isa_fallback = true;
1142 #else
1143 bool __initdata riscv_isa_fallback;
riscv_isa_fallback_setup(char * __unused)1144 static int __init riscv_isa_fallback_setup(char *__unused)
1145 {
1146 riscv_isa_fallback = true;
1147 return 1;
1148 }
1149 early_param("riscv_isa_fallback", riscv_isa_fallback_setup);
1150 #endif
1151
riscv_fill_hwcap(void)1152 void __init riscv_fill_hwcap(void)
1153 {
1154 char print_str[NUM_ALPHA_EXTS + 1];
1155 unsigned long isa2hwcap[RISCV_ISA_EXT_BASE] = {0};
1156 int i, j;
1157
1158 isa2hwcap[RISCV_ISA_EXT_I] = COMPAT_HWCAP_ISA_I;
1159 isa2hwcap[RISCV_ISA_EXT_M] = COMPAT_HWCAP_ISA_M;
1160 isa2hwcap[RISCV_ISA_EXT_A] = COMPAT_HWCAP_ISA_A;
1161 isa2hwcap[RISCV_ISA_EXT_F] = COMPAT_HWCAP_ISA_F;
1162 isa2hwcap[RISCV_ISA_EXT_D] = COMPAT_HWCAP_ISA_D;
1163 isa2hwcap[RISCV_ISA_EXT_C] = COMPAT_HWCAP_ISA_C;
1164 isa2hwcap[RISCV_ISA_EXT_V] = COMPAT_HWCAP_ISA_V;
1165
1166 if (!acpi_disabled) {
1167 riscv_fill_hwcap_from_isa_string(isa2hwcap);
1168 } else {
1169 int ret = riscv_fill_hwcap_from_ext_list(isa2hwcap);
1170
1171 if (ret && riscv_isa_fallback) {
1172 pr_info("Falling back to deprecated \"riscv,isa\"\n");
1173 riscv_fill_hwcap_from_isa_string(isa2hwcap);
1174 }
1175 }
1176
1177 /*
1178 * We don't support systems with F but without D, so mask those out
1179 * here.
1180 */
1181 if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) {
1182 pr_info("This kernel does not support systems with F but not D\n");
1183 elf_hwcap &= ~COMPAT_HWCAP_ISA_F;
1184 }
1185
1186 if (__riscv_isa_extension_available(NULL, RISCV_ISA_EXT_ZVE32X) ||
1187 has_xtheadvector_no_alternatives()) {
1188 /*
1189 * This cannot fail when called on the boot hart
1190 */
1191 riscv_v_setup_vsize();
1192 }
1193
1194 memset(print_str, 0, sizeof(print_str));
1195 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
1196 if (riscv_isa[0] & BIT_MASK(i))
1197 print_str[j++] = (char)('a' + i);
1198 pr_info("riscv: base ISA extensions %s\n", print_str);
1199
1200 memset(print_str, 0, sizeof(print_str));
1201 for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
1202 if (elf_hwcap & BIT_MASK(i))
1203 print_str[j++] = (char)('a' + i);
1204 pr_info("riscv: ELF capabilities %s\n", print_str);
1205 }
1206
riscv_get_elf_hwcap(void)1207 unsigned long riscv_get_elf_hwcap(void)
1208 {
1209 unsigned long hwcap;
1210
1211 hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1));
1212
1213 if (!riscv_v_vstate_ctrl_user_allowed())
1214 hwcap &= ~COMPAT_HWCAP_ISA_V;
1215
1216 return hwcap;
1217 }
1218
riscv_user_isa_enable(void)1219 void __init riscv_user_isa_enable(void)
1220 {
1221 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOZ))
1222 current->thread.envcfg |= ENVCFG_CBZE;
1223 else if (any_cpu_has_zicboz)
1224 pr_warn("Zicboz disabled as it is unavailable on some harts\n");
1225
1226 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOM))
1227 current->thread.envcfg |= ENVCFG_CBCFE;
1228 else if (any_cpu_has_zicbom)
1229 pr_warn("Zicbom disabled as it is unavailable on some harts\n");
1230
1231 if (!riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICBOP) &&
1232 any_cpu_has_zicbop)
1233 pr_warn("Zicbop disabled as it is unavailable on some harts\n");
1234 }
1235
1236 #ifdef CONFIG_RISCV_ALTERNATIVE
1237 /*
1238 * Alternative patch sites consider 48 bits when determining when to patch
1239 * the old instruction sequence with the new. These bits are broken into a
1240 * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the
1241 * patch site is for an erratum, identified by the 32-bit patch ID. When
1242 * the vendor ID is zero, the patch site is for a cpufeature. cpufeatures
1243 * further break down patch ID into two 16-bit numbers. The lower 16 bits
1244 * are the cpufeature ID and the upper 16 bits are used for a value specific
1245 * to the cpufeature and patch site. If the upper 16 bits are zero, then it
1246 * implies no specific value is specified. cpufeatures that want to control
1247 * patching on a per-site basis will provide non-zero values and implement
1248 * checks here. The checks return true when patching should be done, and
1249 * false otherwise.
1250 */
riscv_cpufeature_patch_check(u16 id,u16 value)1251 static bool riscv_cpufeature_patch_check(u16 id, u16 value)
1252 {
1253 if (!value)
1254 return true;
1255
1256 switch (id) {
1257 case RISCV_ISA_EXT_ZICBOZ:
1258 /*
1259 * Zicboz alternative applications provide the maximum
1260 * supported block size order, or zero when it doesn't
1261 * matter. If the current block size exceeds the maximum,
1262 * then the alternative cannot be applied.
1263 */
1264 return riscv_cboz_block_size <= (1U << value);
1265 }
1266
1267 return false;
1268 }
1269
riscv_cpufeature_patch_func(struct alt_entry * begin,struct alt_entry * end,unsigned int stage)1270 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
1271 struct alt_entry *end,
1272 unsigned int stage)
1273 {
1274 struct alt_entry *alt;
1275 void *oldptr, *altptr;
1276 u16 id, value, vendor;
1277
1278 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
1279 return;
1280
1281 for (alt = begin; alt < end; alt++) {
1282 id = PATCH_ID_CPUFEATURE_ID(alt->patch_id);
1283 vendor = PATCH_ID_CPUFEATURE_ID(alt->vendor_id);
1284
1285 /*
1286 * Any alternative with a patch_id that is less than
1287 * RISCV_ISA_EXT_MAX is interpreted as a standard extension.
1288 *
1289 * Any alternative with patch_id that is greater than or equal
1290 * to RISCV_VENDOR_EXT_ALTERNATIVES_BASE is interpreted as a
1291 * vendor extension.
1292 */
1293 if (id < RISCV_ISA_EXT_MAX) {
1294 /*
1295 * This patch should be treated as errata so skip
1296 * processing here.
1297 */
1298 if (alt->vendor_id != 0)
1299 continue;
1300
1301 if (!__riscv_isa_extension_available(NULL, id))
1302 continue;
1303
1304 value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id);
1305 if (!riscv_cpufeature_patch_check(id, value))
1306 continue;
1307 } else if (id >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE) {
1308 if (!__riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor,
1309 id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE))
1310 continue;
1311 } else {
1312 WARN(1, "This extension id:%d is not in ISA extension list", id);
1313 continue;
1314 }
1315
1316 oldptr = ALT_OLD_PTR(alt);
1317 altptr = ALT_ALT_PTR(alt);
1318
1319 mutex_lock(&text_mutex);
1320 patch_text_nosync(oldptr, altptr, alt->alt_len);
1321 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);
1322 mutex_unlock(&text_mutex);
1323 }
1324 }
1325 #endif
1326