cpufeature.c (f557af081de6b45a25e27d633b4d8d2dbc2f428e) cpufeature.c (23c996fc2bc1978a02c64eddb90b4ab5d309c8df)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copied from arch/arm64/kernel/cpufeature.c
4 *
5 * Copyright (C) 2015 ARM Ltd.
6 * Copyright (C) 2017 SiFive
7 */
8

--- 10 unchanged lines hidden (view full) ---

19#include <asm/alternative.h>
20#include <asm/cacheflush.h>
21#include <asm/cpufeature.h>
22#include <asm/hwcap.h>
23#include <asm/patch.h>
24#include <asm/processor.h>
25#include <asm/sbi.h>
26#include <asm/vector.h>
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copied from arch/arm64/kernel/cpufeature.c
4 *
5 * Copyright (C) 2015 ARM Ltd.
6 * Copyright (C) 2017 SiFive
7 */
8

--- 10 unchanged lines hidden (view full) ---

19#include <asm/alternative.h>
20#include <asm/cacheflush.h>
21#include <asm/cpufeature.h>
22#include <asm/hwcap.h>
23#include <asm/patch.h>
24#include <asm/processor.h>
25#include <asm/sbi.h>
26#include <asm/vector.h>
27#include <asm/vendor_extensions.h>
27
28#define NUM_ALPHA_EXTS ('z' - 'a' + 1)
29
30unsigned long elf_hwcap __read_mostly;
31
32/* Host ISA bitmap */
33static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
34

--- 60 unchanged lines hidden (view full) ---

95 }
96 if (!is_power_of_2(riscv_cboz_block_size)) {
97 pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");
98 return -EINVAL;
99 }
100 return 0;
101}
102
28
29#define NUM_ALPHA_EXTS ('z' - 'a' + 1)
30
31unsigned long elf_hwcap __read_mostly;
32
33/* Host ISA bitmap */
34static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
35

--- 60 unchanged lines hidden (view full) ---

96 }
97 if (!is_power_of_2(riscv_cboz_block_size)) {
98 pr_err("Zicboz disabled as cboz-block-size present, but is not a power-of-2\n");
99 return -EINVAL;
100 }
101 return 0;
102}
103
103#define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \
104 .name = #_name, \
105 .property = #_name, \
106 .id = _id, \
107 .subset_ext_ids = _subset_exts, \
108 .subset_ext_size = _subset_exts_size, \
109 .validate = _validate \
110}
111
112#define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, NULL)
113
114#define __RISCV_ISA_EXT_DATA_VALIDATE(_name, _id, _validate) \
115 _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, _validate)
116
117/* Used to declare pure "lasso" extension (Zk for instance) */
118#define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \
119 _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \
120 ARRAY_SIZE(_bundled_exts), NULL)
121
122/* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */
123#define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \
124 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), NULL)
125#define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \
126 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate)
127
128static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data,
129 const unsigned long *isa_bitmap)
130{
131 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA))
132 return 0;
133
134 return -EPROBE_DEFER;
135}

--- 264 unchanged lines hidden (view full) ---

400 __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA),
401 __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN),
402 __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA),
403 __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
404 __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),
405 __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
406 __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),
407 __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT),
104static int riscv_ext_zca_depends(const struct riscv_isa_ext_data *data,
105 const unsigned long *isa_bitmap)
106{
107 if (__riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_ZCA))
108 return 0;
109
110 return -EPROBE_DEFER;
111}

--- 264 unchanged lines hidden (view full) ---

376 __RISCV_ISA_EXT_DATA(smaia, RISCV_ISA_EXT_SMAIA),
377 __RISCV_ISA_EXT_DATA(smstateen, RISCV_ISA_EXT_SMSTATEEN),
378 __RISCV_ISA_EXT_DATA(ssaia, RISCV_ISA_EXT_SSAIA),
379 __RISCV_ISA_EXT_DATA(sscofpmf, RISCV_ISA_EXT_SSCOFPMF),
380 __RISCV_ISA_EXT_DATA(sstc, RISCV_ISA_EXT_SSTC),
381 __RISCV_ISA_EXT_DATA(svinval, RISCV_ISA_EXT_SVINVAL),
382 __RISCV_ISA_EXT_DATA(svnapot, RISCV_ISA_EXT_SVNAPOT),
383 __RISCV_ISA_EXT_DATA(svpbmt, RISCV_ISA_EXT_SVPBMT),
408 __RISCV_ISA_EXT_DATA(xandespmu, RISCV_ISA_EXT_XANDESPMU),
409};
410
411const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext);
412
413static void riscv_isa_set_ext(const struct riscv_isa_ext_data *ext, unsigned long *bitmap)
414{
415 if (ext->id != RISCV_ISA_EXT_INVALID)
416 set_bit(ext->id, bitmap);

--- 90 unchanged lines hidden (view full) ---

507 isa += 4;
508
509 while (*isa) {
510 const char *ext = isa++;
511 const char *ext_end = isa;
512 bool ext_err = false;
513
514 switch (*ext) {
384};
385
386const size_t riscv_isa_ext_count = ARRAY_SIZE(riscv_isa_ext);
387
388static void riscv_isa_set_ext(const struct riscv_isa_ext_data *ext, unsigned long *bitmap)
389{
390 if (ext->id != RISCV_ISA_EXT_INVALID)
391 set_bit(ext->id, bitmap);

--- 90 unchanged lines hidden (view full) ---

482 isa += 4;
483
484 while (*isa) {
485 const char *ext = isa++;
486 const char *ext_end = isa;
487 bool ext_err = false;
488
489 switch (*ext) {
490 case 'x':
491 case 'X':
492 if (acpi_disabled)
493 pr_warn_once("Vendor extensions are ignored in riscv,isa. Use riscv,isa-extensions instead.");
494 /*
495 * To skip an extension, we find its end.
496 * As multi-letter extensions must be split from other multi-letter
497 * extensions with an "_", the end of a multi-letter extension will
498 * either be the null character or the "_" at the start of the next
499 * multi-letter extension.
500 */
501 for (; *isa && *isa != '_'; ++isa)
502 ;
503 ext_err = true;
504 break;
515 case 's':
516 /*
517 * Workaround for invalid single-letter 's' & 'u' (QEMU).
518 * No need to set the bit in riscv_isa as 's' & 'u' are
519 * not valid ISA extensions. It works unless the first
520 * multi-letter extension in the ISA string begins with
521 * "Su" and is not prefixed with an underscore.
522 */
523 if (ext[-1] != '_' && ext[1] == 'u') {
524 ++isa;
525 ext_err = true;
526 break;
527 }
528 fallthrough;
529 case 'S':
505 case 's':
506 /*
507 * Workaround for invalid single-letter 's' & 'u' (QEMU).
508 * No need to set the bit in riscv_isa as 's' & 'u' are
509 * not valid ISA extensions. It works unless the first
510 * multi-letter extension in the ISA string begins with
511 * "Su" and is not prefixed with an underscore.
512 */
513 if (ext[-1] != '_' && ext[1] == 'u') {
514 ++isa;
515 ext_err = true;
516 break;
517 }
518 fallthrough;
519 case 'S':
530 case 'x':
531 case 'X':
532 case 'z':
533 case 'Z':
534 /*
535 * Before attempting to parse the extension itself, we find its end.
536 * As multi-letter extensions must be split from other multi-letter
537 * extensions with an "_", the end of a multi-letter extension will
538 * either be the null character or the "_" at the start of the next
539 * multi-letter extension.

--- 183 unchanged lines hidden (view full) ---

723 else
724 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
725 }
726
727 if (!acpi_disabled && rhct)
728 acpi_put_table((struct acpi_table_header *)rhct);
729}
730
520 case 'z':
521 case 'Z':
522 /*
523 * Before attempting to parse the extension itself, we find its end.
524 * As multi-letter extensions must be split from other multi-letter
525 * extensions with an "_", the end of a multi-letter extension will
526 * either be the null character or the "_" at the start of the next
527 * multi-letter extension.

--- 183 unchanged lines hidden (view full) ---

711 else
712 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
713 }
714
715 if (!acpi_disabled && rhct)
716 acpi_put_table((struct acpi_table_header *)rhct);
717}
718
719static void __init riscv_fill_cpu_vendor_ext(struct device_node *cpu_node, int cpu)
720{
721 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT))
722 return;
723
724 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) {
725 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i];
726
727 for (int j = 0; j < ext_list->ext_data_count; j++) {
728 const struct riscv_isa_ext_data ext = ext_list->ext_data[j];
729 struct riscv_isavendorinfo *isavendorinfo = &ext_list->per_hart_isa_bitmap[cpu];
730
731 if (of_property_match_string(cpu_node, "riscv,isa-extensions",
732 ext.property) < 0)
733 continue;
734
735 /*
736 * Assume that subset extensions are all members of the
737 * same vendor.
738 */
739 if (ext.subset_ext_size)
740 for (int k = 0; k < ext.subset_ext_size; k++)
741 set_bit(ext.subset_ext_ids[k], isavendorinfo->isa);
742
743 set_bit(ext.id, isavendorinfo->isa);
744 }
745 }
746}
747
748/*
749 * Populate all_harts_isa_bitmap for each vendor with all of the extensions that
750 * are shared across CPUs for that vendor.
751 */
752static void __init riscv_fill_vendor_ext_list(int cpu)
753{
754 if (!IS_ENABLED(CONFIG_RISCV_ISA_VENDOR_EXT))
755 return;
756
757 for (int i = 0; i < riscv_isa_vendor_ext_list_size; i++) {
758 struct riscv_isa_vendor_ext_data_list *ext_list = riscv_isa_vendor_ext_list[i];
759
760 if (!ext_list->is_initialized) {
761 bitmap_copy(ext_list->all_harts_isa_bitmap.isa,
762 ext_list->per_hart_isa_bitmap[cpu].isa,
763 RISCV_ISA_VENDOR_EXT_MAX);
764 ext_list->is_initialized = true;
765 } else {
766 bitmap_and(ext_list->all_harts_isa_bitmap.isa,
767 ext_list->all_harts_isa_bitmap.isa,
768 ext_list->per_hart_isa_bitmap[cpu].isa,
769 RISCV_ISA_VENDOR_EXT_MAX);
770 }
771 }
772}
773
731static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap)
732{
733 unsigned int cpu;
734
735 for_each_possible_cpu(cpu) {
736 unsigned long this_hwcap = 0;
737 struct device_node *cpu_node;
738 struct riscv_isainfo *isainfo = &hart_isa[cpu];

--- 16 unchanged lines hidden (view full) ---

755 if (of_property_match_string(cpu_node, "riscv,isa-extensions",
756 ext->property) < 0)
757 continue;
758
759 riscv_isa_set_ext(ext, source_isa);
760 }
761
762 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);
774static int __init riscv_fill_hwcap_from_ext_list(unsigned long *isa2hwcap)
775{
776 unsigned int cpu;
777
778 for_each_possible_cpu(cpu) {
779 unsigned long this_hwcap = 0;
780 struct device_node *cpu_node;
781 struct riscv_isainfo *isainfo = &hart_isa[cpu];

--- 16 unchanged lines hidden (view full) ---

798 if (of_property_match_string(cpu_node, "riscv,isa-extensions",
799 ext->property) < 0)
800 continue;
801
802 riscv_isa_set_ext(ext, source_isa);
803 }
804
805 riscv_resolve_isa(source_isa, isainfo->isa, &this_hwcap, isa2hwcap);
806 riscv_fill_cpu_vendor_ext(cpu_node, cpu);
763
764 of_node_put(cpu_node);
765
766 /*
767 * All "okay" harts should have same isa. Set HWCAP based on
768 * common capabilities of every "okay" hart, in case they don't.
769 */
770 if (elf_hwcap)
771 elf_hwcap &= this_hwcap;
772 else
773 elf_hwcap = this_hwcap;
774
775 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
776 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
777 else
778 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
807
808 of_node_put(cpu_node);
809
810 /*
811 * All "okay" harts should have same isa. Set HWCAP based on
812 * common capabilities of every "okay" hart, in case they don't.
813 */
814 if (elf_hwcap)
815 elf_hwcap &= this_hwcap;
816 else
817 elf_hwcap = this_hwcap;
818
819 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
820 bitmap_copy(riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
821 else
822 bitmap_and(riscv_isa, riscv_isa, isainfo->isa, RISCV_ISA_EXT_MAX);
823
824 riscv_fill_vendor_ext_list(cpu);
779 }
780
781 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
782 return -ENOENT;
783
784 return 0;
785}
786

--- 126 unchanged lines hidden (view full) ---

913}
914
915void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
916 struct alt_entry *end,
917 unsigned int stage)
918{
919 struct alt_entry *alt;
920 void *oldptr, *altptr;
825 }
826
827 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
828 return -ENOENT;
829
830 return 0;
831}
832

--- 126 unchanged lines hidden (view full) ---

959}
960
961void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
962 struct alt_entry *end,
963 unsigned int stage)
964{
965 struct alt_entry *alt;
966 void *oldptr, *altptr;
921 u16 id, value;
967 u16 id, value, vendor;
922
923 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
924 return;
925
926 for (alt = begin; alt < end; alt++) {
968
969 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
970 return;
971
972 for (alt = begin; alt < end; alt++) {
927 if (alt->vendor_id != 0)
928 continue;
929
930 id = PATCH_ID_CPUFEATURE_ID(alt->patch_id);
973 id = PATCH_ID_CPUFEATURE_ID(alt->patch_id);
974 vendor = PATCH_ID_CPUFEATURE_ID(alt->vendor_id);
931
975
932 if (id >= RISCV_ISA_EXT_MAX) {
976 /*
977 * Any alternative with a patch_id that is less than
978 * RISCV_ISA_EXT_MAX is interpreted as a standard extension.
979 *
980 * Any alternative with patch_id that is greater than or equal
981 * to RISCV_VENDOR_EXT_ALTERNATIVES_BASE is interpreted as a
982 * vendor extension.
983 */
984 if (id < RISCV_ISA_EXT_MAX) {
985 /*
986 * This patch should be treated as errata so skip
987 * processing here.
988 */
989 if (alt->vendor_id != 0)
990 continue;
991
992 if (!__riscv_isa_extension_available(NULL, id))
993 continue;
994
995 value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id);
996 if (!riscv_cpufeature_patch_check(id, value))
997 continue;
998 } else if (id >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE) {
999 if (!__riscv_isa_vendor_extension_available(VENDOR_EXT_ALL_CPUS, vendor,
1000 id - RISCV_VENDOR_EXT_ALTERNATIVES_BASE))
1001 continue;
1002 } else {
933 WARN(1, "This extension id:%d is not in ISA extension list", id);
934 continue;
935 }
936
1003 WARN(1, "This extension id:%d is not in ISA extension list", id);
1004 continue;
1005 }
1006
937 if (!__riscv_isa_extension_available(NULL, id))
938 continue;
939
940 value = PATCH_ID_CPUFEATURE_VALUE(alt->patch_id);
941 if (!riscv_cpufeature_patch_check(id, value))
942 continue;
943
944 oldptr = ALT_OLD_PTR(alt);
945 altptr = ALT_ALT_PTR(alt);
946
947 mutex_lock(&text_mutex);
948 patch_text_nosync(oldptr, altptr, alt->alt_len);
949 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);
950 mutex_unlock(&text_mutex);
951 }
952}
953#endif
1007 oldptr = ALT_OLD_PTR(alt);
1008 altptr = ALT_ALT_PTR(alt);
1009
1010 mutex_lock(&text_mutex);
1011 patch_text_nosync(oldptr, altptr, alt->alt_len);
1012 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);
1013 mutex_unlock(&text_mutex);
1014 }
1015}
1016#endif