1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 SiFive 4 */ 5 6 #include <linux/acpi.h> 7 #include <linux/of.h> 8 #include <linux/prctl.h> 9 #include <asm/acpi.h> 10 #include <asm/cacheflush.h> 11 12 #ifdef CONFIG_SMP 13 14 #include <asm/sbi.h> 15 16 static void ipi_remote_fence_i(void *info) 17 { 18 return local_flush_icache_all(); 19 } 20 21 void flush_icache_all(void) 22 { 23 local_flush_icache_all(); 24 25 if (num_online_cpus() < 2) 26 return; 27 28 /* 29 * Make sure all previous writes to the D$ are ordered before making 30 * the IPI. The RISC-V spec states that a hart must execute a data fence 31 * before triggering a remote fence.i in order to make the modification 32 * visable for remote harts. 33 * 34 * IPIs on RISC-V are triggered by MMIO writes to either CLINT or 35 * S-IMSIC, so the fence ensures previous data writes "happen before" 36 * the MMIO. 37 */ 38 RISCV_FENCE(w, o); 39 40 if (riscv_use_sbi_for_rfence()) 41 sbi_remote_fence_i(NULL); 42 else 43 on_each_cpu(ipi_remote_fence_i, NULL, 1); 44 } 45 EXPORT_SYMBOL(flush_icache_all); 46 47 /* 48 * Performs an icache flush for the given MM context. RISC-V has no direct 49 * mechanism for instruction cache shoot downs, so instead we send an IPI that 50 * informs the remote harts they need to flush their local instruction caches. 51 * To avoid pathologically slow behavior in a common case (a bunch of 52 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the 53 * IPIs for harts that are not currently executing a MM context and instead 54 * schedule a deferred local instruction cache flush to be performed before 55 * execution resumes on each hart. 56 */ 57 void flush_icache_mm(struct mm_struct *mm, bool local) 58 { 59 unsigned int cpu; 60 cpumask_t others, *mask; 61 62 preempt_disable(); 63 64 /* Mark every hart's icache as needing a flush for this MM. */ 65 mask = &mm->context.icache_stale_mask; 66 cpumask_setall(mask); 67 /* Flush this hart's I$ now, and mark it as flushed. */ 68 cpu = smp_processor_id(); 69 cpumask_clear_cpu(cpu, mask); 70 local_flush_icache_all(); 71 72 /* 73 * Flush the I$ of other harts concurrently executing, and mark them as 74 * flushed. 75 */ 76 cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); 77 local |= cpumask_empty(&others); 78 if (mm == current->active_mm && local) { 79 /* 80 * It's assumed that at least one strongly ordered operation is 81 * performed on this hart between setting a hart's cpumask bit 82 * and scheduling this MM context on that hart. Sending an SBI 83 * remote message will do this, but in the case where no 84 * messages are sent we still need to order this hart's writes 85 * with flush_icache_deferred(). 86 */ 87 smp_mb(); 88 } else if (riscv_use_sbi_for_rfence()) { 89 sbi_remote_fence_i(&others); 90 } else { 91 on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); 92 } 93 94 preempt_enable(); 95 } 96 97 #endif /* CONFIG_SMP */ 98 99 #ifdef CONFIG_MMU 100 void flush_icache_pte(struct mm_struct *mm, pte_t pte) 101 { 102 struct folio *folio = page_folio(pte_page(pte)); 103 104 if (!test_bit(PG_dcache_clean, &folio->flags)) { 105 flush_icache_mm(mm, false); 106 set_bit(PG_dcache_clean, &folio->flags); 107 } 108 } 109 #endif /* CONFIG_MMU */ 110 111 unsigned int riscv_cbom_block_size; 112 EXPORT_SYMBOL_GPL(riscv_cbom_block_size); 113 114 unsigned int riscv_cboz_block_size; 115 EXPORT_SYMBOL_GPL(riscv_cboz_block_size); 116 117 static void __init cbo_get_block_size(struct device_node *node, 118 const char *name, u32 *block_size, 119 unsigned long *first_hartid) 120 { 121 unsigned long hartid; 122 u32 val; 123 124 if (riscv_of_processor_hartid(node, &hartid)) 125 return; 126 127 if (of_property_read_u32(node, name, &val)) 128 return; 129 130 if (!*block_size) { 131 *block_size = val; 132 *first_hartid = hartid; 133 } else if (*block_size != val) { 134 pr_warn("%s mismatched between harts %lu and %lu\n", 135 name, *first_hartid, hartid); 136 } 137 } 138 139 void __init riscv_init_cbo_blocksizes(void) 140 { 141 unsigned long cbom_hartid, cboz_hartid; 142 u32 cbom_block_size = 0, cboz_block_size = 0; 143 struct device_node *node; 144 struct acpi_table_header *rhct; 145 acpi_status status; 146 147 if (acpi_disabled) { 148 for_each_of_cpu_node(node) { 149 /* set block-size for cbom and/or cboz extension if available */ 150 cbo_get_block_size(node, "riscv,cbom-block-size", 151 &cbom_block_size, &cbom_hartid); 152 cbo_get_block_size(node, "riscv,cboz-block-size", 153 &cboz_block_size, &cboz_hartid); 154 } 155 } else { 156 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); 157 if (ACPI_FAILURE(status)) 158 return; 159 160 acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL); 161 acpi_put_table((struct acpi_table_header *)rhct); 162 } 163 164 if (cbom_block_size) 165 riscv_cbom_block_size = cbom_block_size; 166 167 if (cboz_block_size) 168 riscv_cboz_block_size = cboz_block_size; 169 } 170 171 #ifdef CONFIG_SMP 172 static void set_icache_stale_mask(void) 173 { 174 int cpu = get_cpu(); 175 cpumask_t *mask; 176 bool stale_cpu; 177 178 /* 179 * Mark every other hart's icache as needing a flush for 180 * this MM. Maintain the previous value of the current 181 * cpu to handle the case when this function is called 182 * concurrently on different harts. 183 */ 184 mask = ¤t->mm->context.icache_stale_mask; 185 stale_cpu = cpumask_test_cpu(cpu, mask); 186 187 cpumask_setall(mask); 188 cpumask_assign_cpu(cpu, mask, stale_cpu); 189 put_cpu(); 190 } 191 #endif 192 193 /** 194 * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in 195 * userspace. 196 * @ctx: Set the type of icache flushing instructions permitted/prohibited in 197 * userspace. Supported values described below. 198 * 199 * Supported values for ctx: 200 * 201 * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space. 202 * 203 * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in 204 * a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``. 205 * Therefore, caution must be taken; use this flag only when you can guarantee 206 * that no thread in the process will emit fence.i from this point onward. 207 * 208 * @scope: Set scope of where icache flushing instructions are allowed to be 209 * emitted. Supported values described below. 210 * 211 * Supported values for scope: 212 * 213 * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process 214 * is coherent with instruction storage upon 215 * migration. 216 * 217 * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is 218 * coherent with instruction storage upon 219 * migration. 220 * 221 * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are 222 * permitted to emit icache flushing instructions. Whenever any thread in the 223 * process is migrated, the corresponding hart's icache will be guaranteed to be 224 * consistent with instruction storage. This does not enforce any guarantees 225 * outside of migration. If a thread modifies an instruction that another thread 226 * may attempt to execute, the other thread must still emit an icache flushing 227 * instruction before attempting to execute the potentially modified 228 * instruction. This must be performed by the user-space program. 229 * 230 * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the 231 * thread calling this function is permitted to emit icache flushing 232 * instructions. When the thread is migrated, the corresponding hart's icache 233 * will be guaranteed to be consistent with instruction storage. 234 * 235 * On kernels configured without SMP, this function is a nop as migrations 236 * across harts will not occur. 237 */ 238 int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope) 239 { 240 #ifdef CONFIG_SMP 241 switch (ctx) { 242 case PR_RISCV_CTX_SW_FENCEI_ON: 243 switch (scope) { 244 case PR_RISCV_SCOPE_PER_PROCESS: 245 current->mm->context.force_icache_flush = true; 246 break; 247 case PR_RISCV_SCOPE_PER_THREAD: 248 current->thread.force_icache_flush = true; 249 break; 250 default: 251 return -EINVAL; 252 } 253 break; 254 case PR_RISCV_CTX_SW_FENCEI_OFF: 255 switch (scope) { 256 case PR_RISCV_SCOPE_PER_PROCESS: 257 set_icache_stale_mask(); 258 current->mm->context.force_icache_flush = false; 259 break; 260 case PR_RISCV_SCOPE_PER_THREAD: 261 set_icache_stale_mask(); 262 current->thread.force_icache_flush = false; 263 break; 264 default: 265 return -EINVAL; 266 } 267 break; 268 default: 269 return -EINVAL; 270 } 271 return 0; 272 #else 273 switch (ctx) { 274 case PR_RISCV_CTX_SW_FENCEI_ON: 275 case PR_RISCV_CTX_SW_FENCEI_OFF: 276 return 0; 277 default: 278 return -EINVAL; 279 } 280 #endif 281 } 282