1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 SiFive 4 */ 5 6 #include <linux/acpi.h> 7 #include <linux/of.h> 8 #include <linux/prctl.h> 9 #include <asm/acpi.h> 10 #include <asm/cacheflush.h> 11 12 #ifdef CONFIG_SMP 13 14 #include <asm/sbi.h> 15 16 static void ipi_remote_fence_i(void *info) 17 { 18 return local_flush_icache_all(); 19 } 20 21 void flush_icache_all(void) 22 { 23 local_flush_icache_all(); 24 25 if (num_online_cpus() < 2) 26 return; 27 28 /* 29 * Make sure all previous writes to the D$ are ordered before making 30 * the IPI. The RISC-V spec states that a hart must execute a data fence 31 * before triggering a remote fence.i in order to make the modification 32 * visable for remote harts. 33 * 34 * IPIs on RISC-V are triggered by MMIO writes to either CLINT or 35 * S-IMSIC, so the fence ensures previous data writes "happen before" 36 * the MMIO. 37 */ 38 RISCV_FENCE(w, o); 39 40 if (riscv_use_sbi_for_rfence()) 41 sbi_remote_fence_i(NULL); 42 else 43 on_each_cpu(ipi_remote_fence_i, NULL, 1); 44 } 45 EXPORT_SYMBOL(flush_icache_all); 46 47 /* 48 * Performs an icache flush for the given MM context. RISC-V has no direct 49 * mechanism for instruction cache shoot downs, so instead we send an IPI that 50 * informs the remote harts they need to flush their local instruction caches. 51 * To avoid pathologically slow behavior in a common case (a bunch of 52 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the 53 * IPIs for harts that are not currently executing a MM context and instead 54 * schedule a deferred local instruction cache flush to be performed before 55 * execution resumes on each hart. 56 */ 57 void flush_icache_mm(struct mm_struct *mm, bool local) 58 { 59 unsigned int cpu; 60 cpumask_t others, *mask; 61 62 preempt_disable(); 63 64 /* Mark every hart's icache as needing a flush for this MM. */ 65 mask = &mm->context.icache_stale_mask; 66 cpumask_setall(mask); 67 /* Flush this hart's I$ now, and mark it as flushed. */ 68 cpu = smp_processor_id(); 69 cpumask_clear_cpu(cpu, mask); 70 local_flush_icache_all(); 71 72 /* 73 * Flush the I$ of other harts concurrently executing, and mark them as 74 * flushed. 75 */ 76 cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); 77 local |= cpumask_empty(&others); 78 if (mm == current->active_mm && local) { 79 /* 80 * It's assumed that at least one strongly ordered operation is 81 * performed on this hart between setting a hart's cpumask bit 82 * and scheduling this MM context on that hart. Sending an SBI 83 * remote message will do this, but in the case where no 84 * messages are sent we still need to order this hart's writes 85 * with flush_icache_deferred(). 86 */ 87 smp_mb(); 88 } else if (riscv_use_sbi_for_rfence()) { 89 sbi_remote_fence_i(&others); 90 } else { 91 on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); 92 } 93 94 preempt_enable(); 95 } 96 97 #endif /* CONFIG_SMP */ 98 99 #ifdef CONFIG_MMU 100 void flush_icache_pte(struct mm_struct *mm, pte_t pte) 101 { 102 struct folio *folio = page_folio(pte_page(pte)); 103 104 if (!test_bit(PG_dcache_clean, &folio->flags)) { 105 flush_icache_mm(mm, false); 106 set_bit(PG_dcache_clean, &folio->flags); 107 } 108 } 109 #endif /* CONFIG_MMU */ 110 111 unsigned int riscv_cbom_block_size; 112 EXPORT_SYMBOL_GPL(riscv_cbom_block_size); 113 114 unsigned int riscv_cboz_block_size; 115 EXPORT_SYMBOL_GPL(riscv_cboz_block_size); 116 117 unsigned int riscv_cbop_block_size; 118 EXPORT_SYMBOL_GPL(riscv_cbop_block_size); 119 120 static void __init cbo_get_block_size(struct device_node *node, 121 const char *name, u32 *block_size, 122 unsigned long *first_hartid) 123 { 124 unsigned long hartid; 125 u32 val; 126 127 if (riscv_of_processor_hartid(node, &hartid)) 128 return; 129 130 if (of_property_read_u32(node, name, &val)) 131 return; 132 133 if (!*block_size) { 134 *block_size = val; 135 *first_hartid = hartid; 136 } else if (*block_size != val) { 137 pr_warn("%s mismatched between harts %lu and %lu\n", 138 name, *first_hartid, hartid); 139 } 140 } 141 142 void __init riscv_init_cbo_blocksizes(void) 143 { 144 unsigned long cbom_hartid, cboz_hartid, cbop_hartid; 145 u32 cbom_block_size = 0, cboz_block_size = 0, cbop_block_size = 0; 146 struct device_node *node; 147 struct acpi_table_header *rhct; 148 acpi_status status; 149 150 if (acpi_disabled) { 151 for_each_of_cpu_node(node) { 152 /* set block-size for cbom and/or cboz extension if available */ 153 cbo_get_block_size(node, "riscv,cbom-block-size", 154 &cbom_block_size, &cbom_hartid); 155 cbo_get_block_size(node, "riscv,cboz-block-size", 156 &cboz_block_size, &cboz_hartid); 157 cbo_get_block_size(node, "riscv,cbop-block-size", 158 &cbop_block_size, &cbop_hartid); 159 } 160 } else { 161 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); 162 if (ACPI_FAILURE(status)) 163 return; 164 165 acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, &cbop_block_size); 166 acpi_put_table((struct acpi_table_header *)rhct); 167 } 168 169 if (cbom_block_size) 170 riscv_cbom_block_size = cbom_block_size; 171 172 if (cboz_block_size) 173 riscv_cboz_block_size = cboz_block_size; 174 175 if (cbop_block_size) 176 riscv_cbop_block_size = cbop_block_size; 177 } 178 179 #ifdef CONFIG_SMP 180 static void set_icache_stale_mask(void) 181 { 182 int cpu = get_cpu(); 183 cpumask_t *mask; 184 bool stale_cpu; 185 186 /* 187 * Mark every other hart's icache as needing a flush for 188 * this MM. Maintain the previous value of the current 189 * cpu to handle the case when this function is called 190 * concurrently on different harts. 191 */ 192 mask = ¤t->mm->context.icache_stale_mask; 193 stale_cpu = cpumask_test_cpu(cpu, mask); 194 195 cpumask_setall(mask); 196 __assign_cpu(cpu, mask, stale_cpu); 197 put_cpu(); 198 } 199 #endif 200 201 /** 202 * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in 203 * userspace. 204 * @ctx: Set the type of icache flushing instructions permitted/prohibited in 205 * userspace. Supported values described below. 206 * 207 * Supported values for ctx: 208 * 209 * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space. 210 * 211 * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in 212 * a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``. 213 * Therefore, caution must be taken; use this flag only when you can guarantee 214 * that no thread in the process will emit fence.i from this point onward. 215 * 216 * @scope: Set scope of where icache flushing instructions are allowed to be 217 * emitted. Supported values described below. 218 * 219 * Supported values for scope: 220 * 221 * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process 222 * is coherent with instruction storage upon 223 * migration. 224 * 225 * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is 226 * coherent with instruction storage upon 227 * migration. 228 * 229 * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are 230 * permitted to emit icache flushing instructions. Whenever any thread in the 231 * process is migrated, the corresponding hart's icache will be guaranteed to be 232 * consistent with instruction storage. This does not enforce any guarantees 233 * outside of migration. If a thread modifies an instruction that another thread 234 * may attempt to execute, the other thread must still emit an icache flushing 235 * instruction before attempting to execute the potentially modified 236 * instruction. This must be performed by the user-space program. 237 * 238 * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the 239 * thread calling this function is permitted to emit icache flushing 240 * instructions. When the thread is migrated, the corresponding hart's icache 241 * will be guaranteed to be consistent with instruction storage. 242 * 243 * On kernels configured without SMP, this function is a nop as migrations 244 * across harts will not occur. 245 */ 246 int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope) 247 { 248 #ifdef CONFIG_SMP 249 switch (ctx) { 250 case PR_RISCV_CTX_SW_FENCEI_ON: 251 switch (scope) { 252 case PR_RISCV_SCOPE_PER_PROCESS: 253 current->mm->context.force_icache_flush = true; 254 break; 255 case PR_RISCV_SCOPE_PER_THREAD: 256 current->thread.force_icache_flush = true; 257 break; 258 default: 259 return -EINVAL; 260 } 261 break; 262 case PR_RISCV_CTX_SW_FENCEI_OFF: 263 switch (scope) { 264 case PR_RISCV_SCOPE_PER_PROCESS: 265 set_icache_stale_mask(); 266 current->mm->context.force_icache_flush = false; 267 break; 268 case PR_RISCV_SCOPE_PER_THREAD: 269 set_icache_stale_mask(); 270 current->thread.force_icache_flush = false; 271 break; 272 default: 273 return -EINVAL; 274 } 275 break; 276 default: 277 return -EINVAL; 278 } 279 return 0; 280 #else 281 switch (ctx) { 282 case PR_RISCV_CTX_SW_FENCEI_ON: 283 case PR_RISCV_CTX_SW_FENCEI_OFF: 284 return 0; 285 default: 286 return -EINVAL; 287 } 288 #endif 289 } 290