1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 * 5 * Derived from MIPS: 6 * Copyright (C) 1994 - 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) 7 * Copyright (C) 2007 MIPS Technologies, Inc. 8 */ 9 #include <linux/cacheinfo.h> 10 #include <linux/export.h> 11 #include <linux/fs.h> 12 #include <linux/highmem.h> 13 #include <linux/kernel.h> 14 #include <linux/linkage.h> 15 #include <linux/mm.h> 16 #include <linux/sched.h> 17 #include <linux/syscalls.h> 18 19 #include <asm/bootinfo.h> 20 #include <asm/cacheflush.h> 21 #include <asm/cpu.h> 22 #include <asm/cpu-features.h> 23 #include <asm/loongarch.h> 24 #include <asm/numa.h> 25 #include <asm/processor.h> 26 #include <asm/setup.h> 27 28 void cache_error_setup(void) 29 { 30 extern char __weak except_vec_cex; 31 set_merr_handler(0x0, &except_vec_cex, 0x80); 32 } 33 34 static void flush_cache_leaf(unsigned int leaf) 35 { 36 int i, j, nr_nodes; 37 uint64_t addr = CSR_DMW0_BASE; 38 struct cache_desc *cdesc = current_cpu_data.cache_leaves + leaf; 39 40 nr_nodes = cache_private(cdesc) ? 1 : loongson_sysconf.nr_nodes; 41 42 do { 43 for (i = 0; i < cdesc->sets; i++) { 44 for (j = 0; j < cdesc->ways; j++) { 45 flush_cache_line(leaf, addr); 46 addr++; 47 } 48 49 addr -= cdesc->ways; 50 addr += cdesc->linesz; 51 } 52 addr += (1ULL << NODE_ADDRSPACE_SHIFT); 53 } while (--nr_nodes > 0); 54 } 55 56 asmlinkage __visible void __flush_cache_all(void) 57 { 58 int leaf; 59 struct cache_desc *cdesc = current_cpu_data.cache_leaves; 60 unsigned int cache_present = current_cpu_data.cache_leaves_present; 61 62 leaf = cache_present - 1; 63 if (cache_inclusive(cdesc + leaf)) { 64 flush_cache_leaf(leaf); 65 return; 66 } 67 68 for (leaf = 0; leaf < cache_present; leaf++) 69 flush_cache_leaf(leaf); 70 } 71 72 #define L1IUPRE (1 << 0) 73 #define L1IUUNIFY (1 << 1) 74 #define L1DPRE (1 << 2) 75 76 #define LXIUPRE (1 << 0) 77 #define LXIUUNIFY (1 << 1) 78 #define LXIUPRIV (1 << 2) 79 #define LXIUINCL (1 << 3) 80 #define LXDPRE (1 << 4) 81 #define LXDPRIV (1 << 5) 82 #define LXDINCL (1 << 6) 83 84 #define populate_cache_properties(cfg0, cdesc, level, leaf) \ 85 do { \ 86 unsigned int cfg1; \ 87 \ 88 cfg1 = read_cpucfg(LOONGARCH_CPUCFG17 + leaf); \ 89 if (level == 1) { \ 90 cdesc->flags |= CACHE_PRIVATE; \ 91 } else { \ 92 if (cfg0 & LXIUPRIV) \ 93 cdesc->flags |= CACHE_PRIVATE; \ 94 if (cfg0 & LXIUINCL) \ 95 cdesc->flags |= CACHE_INCLUSIVE; \ 96 } \ 97 cdesc->level = level; \ 98 cdesc->flags |= CACHE_PRESENT; \ 99 cdesc->ways = ((cfg1 & CPUCFG_CACHE_WAYS_M) >> CPUCFG_CACHE_WAYS) + 1; \ 100 cdesc->sets = 1 << ((cfg1 & CPUCFG_CACHE_SETS_M) >> CPUCFG_CACHE_SETS); \ 101 cdesc->linesz = 1 << ((cfg1 & CPUCFG_CACHE_LSIZE_M) >> CPUCFG_CACHE_LSIZE); \ 102 cdesc++; leaf++; \ 103 } while (0) 104 105 void cpu_cache_init(void) 106 { 107 unsigned int leaf = 0, level = 1; 108 unsigned int config = read_cpucfg(LOONGARCH_CPUCFG16); 109 struct cache_desc *cdesc = current_cpu_data.cache_leaves; 110 111 if (config & L1IUPRE) { 112 if (config & L1IUUNIFY) 113 cdesc->type = CACHE_TYPE_UNIFIED; 114 else 115 cdesc->type = CACHE_TYPE_INST; 116 populate_cache_properties(config, cdesc, level, leaf); 117 } 118 119 if (config & L1DPRE) { 120 cdesc->type = CACHE_TYPE_DATA; 121 populate_cache_properties(config, cdesc, level, leaf); 122 } 123 124 config = config >> 3; 125 for (level = 2; level <= CACHE_LEVEL_MAX; level++) { 126 if (!config) 127 break; 128 129 if (config & LXIUPRE) { 130 if (config & LXIUUNIFY) 131 cdesc->type = CACHE_TYPE_UNIFIED; 132 else 133 cdesc->type = CACHE_TYPE_INST; 134 populate_cache_properties(config, cdesc, level, leaf); 135 } 136 137 if (config & LXDPRE) { 138 cdesc->type = CACHE_TYPE_DATA; 139 populate_cache_properties(config, cdesc, level, leaf); 140 } 141 142 config = config >> 7; 143 } 144 145 BUG_ON(leaf > CACHE_LEAVES_MAX); 146 147 current_cpu_data.cache_leaves_present = leaf; 148 current_cpu_data.options |= LOONGARCH_CPU_PREFETCH; 149 } 150 151 static const pgprot_t protection_map[16] = { 152 [VM_NONE] = __pgprot(_CACHE_CC | _PAGE_USER | 153 _PAGE_NO_EXEC | _PAGE_NO_READ | 154 (_PAGE_PROTNONE ? : _PAGE_PRESENT)), 155 [VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 156 _PAGE_USER | _PAGE_PRESENT | 157 _PAGE_NO_EXEC), 158 [VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 159 _PAGE_USER | _PAGE_PRESENT | 160 _PAGE_NO_EXEC), 161 [VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 162 _PAGE_USER | _PAGE_PRESENT | 163 _PAGE_NO_EXEC), 164 [VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID | 165 _PAGE_USER | _PAGE_PRESENT), 166 [VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 167 _PAGE_USER | _PAGE_PRESENT), 168 [VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 169 _PAGE_USER | _PAGE_PRESENT), 170 [VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 171 _PAGE_USER | _PAGE_PRESENT), 172 [VM_SHARED] = __pgprot(_CACHE_CC | _PAGE_USER | 173 _PAGE_NO_EXEC | _PAGE_NO_READ | 174 (_PAGE_PROTNONE ? : _PAGE_PRESENT)), 175 [VM_SHARED | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 176 _PAGE_USER | _PAGE_PRESENT | 177 _PAGE_NO_EXEC), 178 [VM_SHARED | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 179 _PAGE_USER | _PAGE_PRESENT | 180 _PAGE_NO_EXEC | _PAGE_WRITE), 181 [VM_SHARED | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 182 _PAGE_USER | _PAGE_PRESENT | 183 _PAGE_NO_EXEC | _PAGE_WRITE), 184 [VM_SHARED | VM_EXEC] = __pgprot(_CACHE_CC | _PAGE_VALID | 185 _PAGE_USER | _PAGE_PRESENT), 186 [VM_SHARED | VM_EXEC | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 187 _PAGE_USER | _PAGE_PRESENT), 188 [VM_SHARED | VM_EXEC | VM_WRITE] = __pgprot(_CACHE_CC | _PAGE_VALID | 189 _PAGE_USER | _PAGE_PRESENT | 190 _PAGE_WRITE), 191 [VM_SHARED | VM_EXEC | VM_WRITE | VM_READ] = __pgprot(_CACHE_CC | _PAGE_VALID | 192 _PAGE_USER | _PAGE_PRESENT | 193 _PAGE_WRITE) 194 }; 195 DECLARE_VM_GET_PAGE_PROT 196