1 // SPDX-License-Identifier: GPL-2.0-only
29 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
33 /* Host ISA bitmap */
36 /* Per-cpu ISA extensions. */
40  * riscv_isa_extension_base() - Get base extension word
42  * @isa_bitmap: ISA bitmap to use
45  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
56  * __riscv_isa_extension_available() - Check whether given extension
59  * @isa_bitmap: ISA bitmap to use
63  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
80 		pr_err("Zicbom detected in ISA string, disabling as no cbom-block-size found\n");  in riscv_ext_zicbom_validate()
81 		return -EINVAL;  in riscv_ext_zicbom_validate()
84 		pr_err("Zicbom disabled as cbom-block-size present, but is not a power-of-2\n");  in riscv_ext_zicbom_validate()
85 		return -EINVAL;  in riscv_ext_zicbom_validate()
94 		pr_err("Zicboz detected in ISA string, disabling as no cboz-block-size found\n");  in riscv_ext_zicboz_validate()
95 		return -EINVAL;  in riscv_ext_zicboz_validate()
98 		pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");  in riscv_ext_zicboz_validate()
99 		return -EINVAL;  in riscv_ext_zicboz_validate()
110 	return -EPROBE_DEFER;  in riscv_ext_zca_depends()
119 	return -EPROBE_DEFER;  in riscv_ext_zcd_validate()
126 		return -EINVAL;  in riscv_ext_zcf_validate()
132 	return -EPROBE_DEFER;  in riscv_ext_zcf_validate()
240  * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V
241  * privileged ISA, the existence of the CSRs is implied by any extension which
242  * specifies [ms]envcfg bit(s). Hence, we define a custom ISA extension for the
243  * existence of the CSR, and treat it as a subset of those other extensions.
251  * - C always implies Zca
252  * - C+F implies Zcf (RV32 only)
253  * - C+D implies Zcd
255  * These extensions will be enabled and then validated depending on the
265  * The canonical order of ISA extension names in the ISA string is defined in
268  * Ordinarily, for in-kernel data structures, this order is unimportant but
269  * isa_ext_arr defines the order of the ISA string in /proc/cpuinfo.
274  * 1. All multi-letter extensions must be separated from other extensions by an
277  * 2. Additional standard extensions (starting with 'Z') must be sorted after
278  *    single-letter extensions and before any higher-privileged extensions.
282  *    If multiple 'Z' extensions are named, they must be ordered first by
285  * 3. Standard supervisor-level extensions (starting with 'S') must be listed
286  *    after standard unprivileged extensions.  If multiple supervisor-level
287  *    extensions are listed, they must be ordered alphabetically.
289  * 4. Standard machine-level extensions (starting with 'Zxm') must be listed
290  *    after any lower-privileged, standard extensions.  If multiple
291  *    machine-level extensions are listed, they must be ordered
294  * 5. Non-standard extensions (starting with 'X') must be listed after all
295  *    standard extensions. If multiple non-standard extensions are listed, they
391 	if (ext->id != RISCV_ISA_EXT_INVALID)  in riscv_isa_set_ext()
392 		set_bit(ext->id, bitmap);  in riscv_isa_set_ext()
394 	for (int i = 0; i < ext->subset_ext_size; i++) {  in riscv_isa_set_ext()
395 		if (ext->subset_ext_ids[i] != RISCV_ISA_EXT_INVALID)  in riscv_isa_set_ext()
396 			set_bit(ext->subset_ext_ids[i], bitmap);  in riscv_isa_set_ext()
411  * "Resolve" a source ISA bitmap into one that matches kernel configuration as
412  * well as correct extension dependencies. Some extensions depends on specific
414  * and this function will actually validate all the extensions provided in
415  * source_isa into the resolved_isa based on extensions validate() callbacks.
429 		if (max_loop_count-- < 0) {  in riscv_resolve_isa()
430 			pr_err("Failed to reach a stable ISA state\n");  in riscv_resolve_isa()
437 			if (ext && ext->validate) {  in riscv_resolve_isa()
438 				ret = ext->validate(ext, resolved_isa);  in riscv_resolve_isa()
439 				if (ret == -EPROBE_DEFER) {  in riscv_resolve_isa()
450 			/* No need to keep it in source isa now that it is enabled */  in riscv_resolve_isa()
453 			/* Single letter extensions get set in hwcap */  in riscv_resolve_isa()
465 		if ((name_end - name == strlen(ext->name)) &&  in match_isa_ext()
466 		    !strncasecmp(name, ext->name, name_end - name)) {  in match_isa_ext()
473 static void __init riscv_parse_isa_string(const char *isa, unsigned long *bitmap)  in riscv_parse_isa_string()  argument
481 	isa += 4;  in riscv_parse_isa_string()
483 	while (*isa) {  in riscv_parse_isa_string()
484 		const char *ext = isa++;  in riscv_parse_isa_string()
485 		const char *ext_end = isa;  in riscv_parse_isa_string()
492 				pr_warn_once("Vendor extensions are ignored in riscv,isa. Use riscv,isa-extensions instead.");  in riscv_parse_isa_string()
495 			 * As multi-letter extensions must be split from other multi-letter  in riscv_parse_isa_string()
496 			 * extensions with an "_", the end of a multi-letter extension will  in riscv_parse_isa_string()
498 			 * multi-letter extension.  in riscv_parse_isa_string()
500 			for (; *isa && *isa != '_'; ++isa)  in riscv_parse_isa_string()
506 			 * Workaround for invalid single-letter 's' & 'u' (QEMU).  in riscv_parse_isa_string()
508 			 * not valid ISA extensions. It works unless the first  in riscv_parse_isa_string()
509 			 * multi-letter extension in the ISA string begins with  in riscv_parse_isa_string()
512 			if (ext[-1] != '_' && ext[1] == 'u') {  in riscv_parse_isa_string()
513 				++isa;  in riscv_parse_isa_string()
523 			 * As multi-letter extensions must be split from other multi-letter  in riscv_parse_isa_string()
524 			 * extensions with an "_", the end of a multi-letter extension will  in riscv_parse_isa_string()
526 			 * multi-letter extension.  in riscv_parse_isa_string()
528 			 * Next, as the extensions version is currently ignored, we  in riscv_parse_isa_string()
536 			 * character itself while eliminating the extensions version number.  in riscv_parse_isa_string()
537 			 * A simple re-increment solves this problem.  in riscv_parse_isa_string()
539 			for (; *isa && *isa != '_'; ++isa)  in riscv_parse_isa_string()
540 				if (unlikely(!isalnum(*isa)))  in riscv_parse_isa_string()
543 			ext_end = isa;  in riscv_parse_isa_string()
547 			if (!isdigit(ext_end[-1]))  in riscv_parse_isa_string()
550 			while (isdigit(*--ext_end))  in riscv_parse_isa_string()
553 			if (tolower(ext_end[0]) != 'p' || !isdigit(ext_end[-1])) {  in riscv_parse_isa_string()
558 			while (isdigit(*--ext_end))  in riscv_parse_isa_string()
565 			 * Things are a little easier for single-letter extensions, as they  in riscv_parse_isa_string()
569 			 * ensure that, when isa was incremented at the start of the loop,  in riscv_parse_isa_string()
572 			 * If we are already on a non-digit, there is nothing to do. Either  in riscv_parse_isa_string()
573 			 * we have a multi-letter extension's _, or the start of an  in riscv_parse_isa_string()
587 			if (!isdigit(*isa))  in riscv_parse_isa_string()
590 			while (isdigit(*++isa))  in riscv_parse_isa_string()
593 			if (tolower(*isa) != 'p')  in riscv_parse_isa_string()
596 			if (!isdigit(*++isa)) {  in riscv_parse_isa_string()
597 				--isa;  in riscv_parse_isa_string()
601 			while (isdigit(*++isa))  in riscv_parse_isa_string()
608 		 * The parser expects that at the start of an iteration isa points to the  in riscv_parse_isa_string()
610 		 * on meeting a non-alphanumeric character, an extra increment is needed  in riscv_parse_isa_string()
611 		 * where the succeeding extension is a multi-letter prefixed with an "_".  in riscv_parse_isa_string()
613 		if (*isa == '_')  in riscv_parse_isa_string()
614 			++isa;  in riscv_parse_isa_string()
626 	const char *isa;  in riscv_fill_hwcap_from_isa_string()  local
655 			rc = of_property_read_string(node, "riscv,isa", &isa);  in riscv_fill_hwcap_from_isa_string()
658 				pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");  in riscv_fill_hwcap_from_isa_string()
662 			rc = acpi_get_riscv_isa(rhct, cpu, &isa);  in riscv_fill_hwcap_from_isa_string()
664 				pr_warn("Unable to get ISA for the hart - %d\n", cpu);  in riscv_fill_hwcap_from_isa_string()
669 		riscv_parse_isa_string(isa, source_isa);  in riscv_fill_hwcap_from_isa_string()
672 		 * These ones were as they were part of the base ISA when the  in riscv_fill_hwcap_from_isa_string()
673 		 * port & dt-bindings were upstreamed, and so can be set  in riscv_fill_hwcap_from_isa_string()
674 		 * unconditionally where `i` is in riscv,isa on DT systems.  in riscv_fill_hwcap_from_isa_string()
684 		 * "V" in ISA strings is ambiguous in practice: it should mean  in riscv_fill_hwcap_from_isa_string()
685 		 * just the standard V-1.0 but vendors aren't well behaved.  in riscv_fill_hwcap_from_isa_string()
686 		 * Many vendors with T-Head CPU cores which implement the 0.7.1  in riscv_fill_hwcap_from_isa_string()
688 		 * CPU cores with the ratified spec will contain non-zero  in riscv_fill_hwcap_from_isa_string()
696 		riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);  in riscv_fill_hwcap_from_isa_string()
699 		 * All "okay" hart should have same isa. Set HWCAP based on  in riscv_fill_hwcap_from_isa_string()
709 			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_isa_string()
711 			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_isa_string()
726 		for (int j = 0; j < ext_list->ext_data_count; j++) {  in riscv_fill_cpu_vendor_ext()
727 			const struct riscv_isa_ext_data ext = ext_list->ext_data[j];  in riscv_fill_cpu_vendor_ext()
728 			struct riscv_isavendorinfo *isavendorinfo = &ext_list->per_hart_isa_bitmap[cpu];  in riscv_fill_cpu_vendor_ext()
730 			if (of_property_match_string(cpu_node, "riscv,isa-extensions",  in riscv_fill_cpu_vendor_ext()
735 			 * Assume that subset extensions are all members of the  in riscv_fill_cpu_vendor_ext()
740 					set_bit(ext.subset_ext_ids[k], isavendorinfo->isa);  in riscv_fill_cpu_vendor_ext()
742 			set_bit(ext.id, isavendorinfo->isa);  in riscv_fill_cpu_vendor_ext()
748  * Populate all_harts_isa_bitmap for each vendor with all of the extensions that
759 		if (!ext_list->is_initialized) {  in riscv_fill_vendor_ext_list()
760 			bitmap_copy(ext_list->all_harts_isa_bitmap.isa,  in riscv_fill_vendor_ext_list()
761 				    ext_list->per_hart_isa_bitmap[cpu].isa,  in riscv_fill_vendor_ext_list()
763 			ext_list->is_initialized = true;  in riscv_fill_vendor_ext_list()
765 			bitmap_and(ext_list->all_harts_isa_bitmap.isa,  in riscv_fill_vendor_ext_list()
766 				   ext_list->all_harts_isa_bitmap.isa,  in riscv_fill_vendor_ext_list()
767 				   ext_list->per_hart_isa_bitmap[cpu].isa,  in riscv_fill_vendor_ext_list()
789 		if (!of_property_present(cpu_node, "riscv,isa-extensions")) {  in riscv_fill_hwcap_from_ext_list()
797 			if (of_property_match_string(cpu_node, "riscv,isa-extensions",  in riscv_fill_hwcap_from_ext_list()
798 						     ext->property) < 0)  in riscv_fill_hwcap_from_ext_list()
804 		riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);  in riscv_fill_hwcap_from_ext_list()
810 		 * All "okay" harts should have same isa. Set HWCAP based on  in riscv_fill_hwcap_from_ext_list()
819 			bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_ext_list()
821 			bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);  in riscv_fill_hwcap_from_ext_list()
827 		return -ENOENT;  in riscv_fill_hwcap_from_ext_list()
850 	isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I;  in riscv_fill_hwcap()
851 	isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M;  in riscv_fill_hwcap()
852 	isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A;  in riscv_fill_hwcap()
853 	isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F;  in riscv_fill_hwcap()
854 	isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D;  in riscv_fill_hwcap()
855 	isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C;  in riscv_fill_hwcap()
856 	isa2hwcap['v' - 'a'] = COMPAT_HWCAP_ISA_V;  in riscv_fill_hwcap()
864 			pr_info("Falling back to deprecated \"riscv,isa\"\n");  in riscv_fill_hwcap()
887 		 * ISA string in device tree might have 'v' flag, but  in riscv_fill_hwcap()
899 	pr_info("riscv: base ISA extensions %s\n", print_str);  in riscv_fill_hwcap()
912 	hwcap = (elf_hwcap & ((1UL << RISCV_ISA_EXT_BASE) - 1));  in riscv_get_elf_hwcap()
930  * 16-bit vendor ID and a 32-bit patch ID. A non-zero vendor ID means the
931  * patch site is for an erratum, identified by the 32-bit patch ID. When
933  * further break down patch ID into two 16-bit numbers. The lower 16 bits
937  * patching on a per-site basis will provide non-zero values and implement
972 		id = PATCH_ID_CPUFEATURE_ID(alt->patch_id);  in riscv_cpufeature_patch_func()
973 		vendor = PATCH_ID_CPUFEATURE_ID(alt->vendor_id);  in riscv_cpufeature_patch_func()
988 			if (alt->vendor_id != 0)  in riscv_cpufeature_patch_func()
994 			value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id);  in riscv_cpufeature_patch_func()
999 								    id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE))  in riscv_cpufeature_patch_func()
1002 			WARN(1, "This extension id:%d is not in ISA extension list", id);  in riscv_cpufeature_patch_func()
1010 		patch_text_nosync(oldptr, altptr, alt->alt_len);  in riscv_cpufeature_patch_func()
1011 		riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);  in riscv_cpufeature_patch_func()