1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 Heiko Stuebner <heiko@sntech.de> 4 */ 5 6 #include <linux/bug.h> 7 #include <linux/kernel.h> 8 #include <linux/memory.h> 9 #include <linux/module.h> 10 #include <linux/string.h> 11 #include <linux/uaccess.h> 12 #include <asm/alternative.h> 13 #include <asm/bugs.h> 14 #include <asm/cacheflush.h> 15 #include <asm/cpufeature.h> 16 #include <asm/dma-noncoherent.h> 17 #include <asm/errata_list.h> 18 #include <asm/hwprobe.h> 19 #include <asm/io.h> 20 #include <asm/text-patching.h> 21 #include <asm/vendorid_list.h> 22 #include <asm/vendor_extensions.h> 23 24 #define CSR_TH_SXSTATUS 0x5c0 25 #define SXSTATUS_MAEE _AC(0x200000, UL) 26 27 static bool errata_probe_mae(unsigned int stage, 28 unsigned long arch_id, unsigned long impid) 29 { 30 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_MAE)) 31 return false; 32 33 if (arch_id != 0 || impid != 0) 34 return false; 35 36 if (stage != RISCV_ALTERNATIVES_EARLY_BOOT && 37 stage != RISCV_ALTERNATIVES_MODULE) 38 return false; 39 40 if (!(csr_read(CSR_TH_SXSTATUS) & SXSTATUS_MAEE)) 41 return false; 42 43 return true; 44 } 45 46 /* 47 * th.dcache.ipa rs1 (invalidate, physical address) 48 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 | 49 * 0000001 01010 rs1 000 00000 0001011 50 * th.dcache.iva rs1 (invalidate, virtual address) 51 * 0000001 00110 rs1 000 00000 0001011 52 * 53 * th.dcache.cpa rs1 (clean, physical address) 54 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 | 55 * 0000001 01001 rs1 000 00000 0001011 56 * th.dcache.cva rs1 (clean, virtual address) 57 * 0000001 00101 rs1 000 00000 0001011 58 * 59 * th.dcache.cipa rs1 (clean then invalidate, physical address) 60 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 | 61 * 0000001 01011 rs1 000 00000 0001011 62 * th.dcache.civa rs1 (clean then invalidate, virtual address) 63 * 0000001 00111 rs1 000 00000 0001011 64 * 65 * th.sync.s (make sure all cache operations finished) 66 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 | 67 * 0000000 11001 00000 000 00000 0001011 68 */ 69 #define THEAD_INVAL_A0 ".long 0x02a5000b" 70 #define THEAD_CLEAN_A0 ".long 0x0295000b" 71 #define THEAD_FLUSH_A0 ".long 0x02b5000b" 72 #define THEAD_SYNC_S ".long 0x0190000b" 73 74 #define THEAD_CMO_OP(_op, _start, _size, _cachesize) \ 75 asm volatile("mv a0, %1\n\t" \ 76 "j 2f\n\t" \ 77 "3:\n\t" \ 78 THEAD_##_op##_A0 "\n\t" \ 79 "add a0, a0, %0\n\t" \ 80 "2:\n\t" \ 81 "bltu a0, %2, 3b\n\t" \ 82 THEAD_SYNC_S \ 83 : : "r"(_cachesize), \ 84 "r"((unsigned long)(_start) & ~((_cachesize) - 1UL)), \ 85 "r"((unsigned long)(_start) + (_size)) \ 86 : "a0") 87 88 static void thead_errata_cache_inv(phys_addr_t paddr, size_t size) 89 { 90 THEAD_CMO_OP(INVAL, paddr, size, riscv_cbom_block_size); 91 } 92 93 static void thead_errata_cache_wback(phys_addr_t paddr, size_t size) 94 { 95 THEAD_CMO_OP(CLEAN, paddr, size, riscv_cbom_block_size); 96 } 97 98 static void thead_errata_cache_wback_inv(phys_addr_t paddr, size_t size) 99 { 100 THEAD_CMO_OP(FLUSH, paddr, size, riscv_cbom_block_size); 101 } 102 103 static const struct riscv_nonstd_cache_ops thead_errata_cmo_ops = { 104 .wback = &thead_errata_cache_wback, 105 .inv = &thead_errata_cache_inv, 106 .wback_inv = &thead_errata_cache_wback_inv, 107 }; 108 109 static bool errata_probe_cmo(unsigned int stage, 110 unsigned long arch_id, unsigned long impid) 111 { 112 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_CMO)) 113 return false; 114 115 if (arch_id != 0 || impid != 0) 116 return false; 117 118 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 119 return false; 120 121 if (stage == RISCV_ALTERNATIVES_BOOT) { 122 riscv_cbom_block_size = L1_CACHE_BYTES; 123 riscv_noncoherent_supported(); 124 riscv_noncoherent_register_cache_ops(&thead_errata_cmo_ops); 125 } 126 127 return true; 128 } 129 130 static bool errata_probe_pmu(unsigned int stage, 131 unsigned long arch_id, unsigned long impid) 132 { 133 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PMU)) 134 return false; 135 136 /* target-c9xx cores report arch_id and impid as 0 */ 137 if (arch_id != 0 || impid != 0) 138 return false; 139 140 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 141 return false; 142 143 return true; 144 } 145 146 static bool errata_probe_ghostwrite(unsigned int stage, 147 unsigned long arch_id, unsigned long impid) 148 { 149 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_GHOSTWRITE)) 150 return false; 151 152 /* 153 * target-c9xx cores report arch_id and impid as 0 154 * 155 * While ghostwrite may not affect all c9xx cores that implement 156 * xtheadvector, there is no futher granularity than c9xx. Assume 157 * vulnerable for this entire class of processors when xtheadvector is 158 * enabled. 159 */ 160 if (arch_id != 0 || impid != 0) 161 return false; 162 163 if (stage != RISCV_ALTERNATIVES_EARLY_BOOT) 164 return false; 165 166 ghostwrite_set_vulnerable(); 167 168 return true; 169 } 170 171 static u32 thead_errata_probe(unsigned int stage, 172 unsigned long archid, unsigned long impid) 173 { 174 u32 cpu_req_errata = 0; 175 176 if (errata_probe_mae(stage, archid, impid)) 177 cpu_req_errata |= BIT(ERRATA_THEAD_MAE); 178 179 errata_probe_cmo(stage, archid, impid); 180 181 if (errata_probe_pmu(stage, archid, impid)) 182 cpu_req_errata |= BIT(ERRATA_THEAD_PMU); 183 184 errata_probe_ghostwrite(stage, archid, impid); 185 186 return cpu_req_errata; 187 } 188 189 void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, 190 unsigned long archid, unsigned long impid, 191 unsigned int stage) 192 { 193 struct alt_entry *alt; 194 u32 cpu_req_errata = thead_errata_probe(stage, archid, impid); 195 u32 tmp; 196 void *oldptr, *altptr; 197 198 BUILD_BUG_ON(ERRATA_THEAD_NUMBER >= RISCV_VENDOR_EXT_ALTERNATIVES_BASE); 199 200 for (alt = begin; alt < end; alt++) { 201 if (alt->vendor_id != THEAD_VENDOR_ID) 202 continue; 203 if (alt->patch_id >= ERRATA_THEAD_NUMBER) 204 continue; 205 206 tmp = (1U << alt->patch_id); 207 if (cpu_req_errata & tmp) { 208 oldptr = ALT_OLD_PTR(alt); 209 altptr = ALT_ALT_PTR(alt); 210 211 /* On vm-alternatives, the mmu isn't running yet */ 212 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) { 213 memcpy(oldptr, altptr, alt->alt_len); 214 } else { 215 mutex_lock(&text_mutex); 216 patch_text_nosync(oldptr, altptr, alt->alt_len); 217 mutex_unlock(&text_mutex); 218 } 219 } 220 } 221 222 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 223 local_flush_icache_all(); 224 } 225