1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2021 Heiko Stuebner <heiko@sntech.de> 4 */ 5 6 #include <linux/bug.h> 7 #include <linux/kernel.h> 8 #include <linux/memory.h> 9 #include <linux/module.h> 10 #include <linux/string.h> 11 #include <linux/uaccess.h> 12 #include <asm/alternative.h> 13 #include <asm/cacheflush.h> 14 #include <asm/cpufeature.h> 15 #include <asm/dma-noncoherent.h> 16 #include <asm/errata_list.h> 17 #include <asm/hwprobe.h> 18 #include <asm/io.h> 19 #include <asm/patch.h> 20 #include <asm/vendorid_list.h> 21 22 static bool errata_probe_pbmt(unsigned int stage, 23 unsigned long arch_id, unsigned long impid) 24 { 25 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PBMT)) 26 return false; 27 28 if (arch_id != 0 || impid != 0) 29 return false; 30 31 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT || 32 stage == RISCV_ALTERNATIVES_MODULE) 33 return true; 34 35 return false; 36 } 37 38 /* 39 * th.dcache.ipa rs1 (invalidate, physical address) 40 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 | 41 * 0000001 01010 rs1 000 00000 0001011 42 * th.dcache.iva rs1 (invalidate, virtual address) 43 * 0000001 00110 rs1 000 00000 0001011 44 * 45 * th.dcache.cpa rs1 (clean, physical address) 46 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 | 47 * 0000001 01001 rs1 000 00000 0001011 48 * th.dcache.cva rs1 (clean, virtual address) 49 * 0000001 00101 rs1 000 00000 0001011 50 * 51 * th.dcache.cipa rs1 (clean then invalidate, physical address) 52 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 | 53 * 0000001 01011 rs1 000 00000 0001011 54 * th.dcache.civa rs1 (clean then invalidate, virtual address) 55 * 0000001 00111 rs1 000 00000 0001011 56 * 57 * th.sync.s (make sure all cache operations finished) 58 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 | 59 * 0000000 11001 00000 000 00000 0001011 60 */ 61 #define THEAD_INVAL_A0 ".long 0x02a5000b" 62 #define THEAD_CLEAN_A0 ".long 0x0295000b" 63 #define THEAD_FLUSH_A0 ".long 0x02b5000b" 64 #define THEAD_SYNC_S ".long 0x0190000b" 65 66 #define THEAD_CMO_OP(_op, _start, _size, _cachesize) \ 67 asm volatile("mv a0, %1\n\t" \ 68 "j 2f\n\t" \ 69 "3:\n\t" \ 70 THEAD_##_op##_A0 "\n\t" \ 71 "add a0, a0, %0\n\t" \ 72 "2:\n\t" \ 73 "bltu a0, %2, 3b\n\t" \ 74 THEAD_SYNC_S \ 75 : : "r"(_cachesize), \ 76 "r"((unsigned long)(_start) & ~((_cachesize) - 1UL)), \ 77 "r"((unsigned long)(_start) + (_size)) \ 78 : "a0") 79 80 static void thead_errata_cache_inv(phys_addr_t paddr, size_t size) 81 { 82 THEAD_CMO_OP(INVAL, paddr, size, riscv_cbom_block_size); 83 } 84 85 static void thead_errata_cache_wback(phys_addr_t paddr, size_t size) 86 { 87 THEAD_CMO_OP(CLEAN, paddr, size, riscv_cbom_block_size); 88 } 89 90 static void thead_errata_cache_wback_inv(phys_addr_t paddr, size_t size) 91 { 92 THEAD_CMO_OP(FLUSH, paddr, size, riscv_cbom_block_size); 93 } 94 95 static const struct riscv_nonstd_cache_ops thead_errata_cmo_ops = { 96 .wback = &thead_errata_cache_wback, 97 .inv = &thead_errata_cache_inv, 98 .wback_inv = &thead_errata_cache_wback_inv, 99 }; 100 101 static bool errata_probe_cmo(unsigned int stage, 102 unsigned long arch_id, unsigned long impid) 103 { 104 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_CMO)) 105 return false; 106 107 if (arch_id != 0 || impid != 0) 108 return false; 109 110 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 111 return false; 112 113 if (stage == RISCV_ALTERNATIVES_BOOT) { 114 riscv_cbom_block_size = L1_CACHE_BYTES; 115 riscv_noncoherent_supported(); 116 riscv_noncoherent_register_cache_ops(&thead_errata_cmo_ops); 117 } 118 119 return true; 120 } 121 122 static bool errata_probe_pmu(unsigned int stage, 123 unsigned long arch_id, unsigned long impid) 124 { 125 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PMU)) 126 return false; 127 128 /* target-c9xx cores report arch_id and impid as 0 */ 129 if (arch_id != 0 || impid != 0) 130 return false; 131 132 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 133 return false; 134 135 return true; 136 } 137 138 static u32 thead_errata_probe(unsigned int stage, 139 unsigned long archid, unsigned long impid) 140 { 141 u32 cpu_req_errata = 0; 142 143 if (errata_probe_pbmt(stage, archid, impid)) 144 cpu_req_errata |= BIT(ERRATA_THEAD_PBMT); 145 146 errata_probe_cmo(stage, archid, impid); 147 148 if (errata_probe_pmu(stage, archid, impid)) 149 cpu_req_errata |= BIT(ERRATA_THEAD_PMU); 150 151 return cpu_req_errata; 152 } 153 154 void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end, 155 unsigned long archid, unsigned long impid, 156 unsigned int stage) 157 { 158 struct alt_entry *alt; 159 u32 cpu_req_errata = thead_errata_probe(stage, archid, impid); 160 u32 tmp; 161 void *oldptr, *altptr; 162 163 for (alt = begin; alt < end; alt++) { 164 if (alt->vendor_id != THEAD_VENDOR_ID) 165 continue; 166 if (alt->patch_id >= ERRATA_THEAD_NUMBER) 167 continue; 168 169 tmp = (1U << alt->patch_id); 170 if (cpu_req_errata & tmp) { 171 oldptr = ALT_OLD_PTR(alt); 172 altptr = ALT_ALT_PTR(alt); 173 174 /* On vm-alternatives, the mmu isn't running yet */ 175 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) { 176 memcpy(oldptr, altptr, alt->alt_len); 177 } else { 178 mutex_lock(&text_mutex); 179 patch_text_nosync(oldptr, altptr, alt->alt_len); 180 mutex_unlock(&text_mutex); 181 } 182 } 183 } 184 185 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) 186 local_flush_icache_all(); 187 } 188