1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 SiFive 4 */ 5 6 #include <linux/acpi.h> 7 #include <linux/of.h> 8 #include <linux/prctl.h> 9 #include <asm/acpi.h> 10 #include <asm/cacheflush.h> 11 12 #ifdef CONFIG_SMP 13 14 #include <asm/sbi.h> 15 16 static void ipi_remote_fence_i(void *info) 17 { 18 return local_flush_icache_all(); 19 } 20 21 void flush_icache_all(void) 22 { 23 local_flush_icache_all(); 24 25 if (num_online_cpus() < 2) 26 return; 27 else if (riscv_use_sbi_for_rfence()) 28 sbi_remote_fence_i(NULL); 29 else 30 on_each_cpu(ipi_remote_fence_i, NULL, 1); 31 } 32 EXPORT_SYMBOL(flush_icache_all); 33 34 /* 35 * Performs an icache flush for the given MM context. RISC-V has no direct 36 * mechanism for instruction cache shoot downs, so instead we send an IPI that 37 * informs the remote harts they need to flush their local instruction caches. 38 * To avoid pathologically slow behavior in a common case (a bunch of 39 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the 40 * IPIs for harts that are not currently executing a MM context and instead 41 * schedule a deferred local instruction cache flush to be performed before 42 * execution resumes on each hart. 43 */ 44 void flush_icache_mm(struct mm_struct *mm, bool local) 45 { 46 unsigned int cpu; 47 cpumask_t others, *mask; 48 49 preempt_disable(); 50 51 /* Mark every hart's icache as needing a flush for this MM. */ 52 mask = &mm->context.icache_stale_mask; 53 cpumask_setall(mask); 54 /* Flush this hart's I$ now, and mark it as flushed. */ 55 cpu = smp_processor_id(); 56 cpumask_clear_cpu(cpu, mask); 57 local_flush_icache_all(); 58 59 /* 60 * Flush the I$ of other harts concurrently executing, and mark them as 61 * flushed. 62 */ 63 cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); 64 local |= cpumask_empty(&others); 65 if (mm == current->active_mm && local) { 66 /* 67 * It's assumed that at least one strongly ordered operation is 68 * performed on this hart between setting a hart's cpumask bit 69 * and scheduling this MM context on that hart. Sending an SBI 70 * remote message will do this, but in the case where no 71 * messages are sent we still need to order this hart's writes 72 * with flush_icache_deferred(). 73 */ 74 smp_mb(); 75 } else if (riscv_use_sbi_for_rfence()) { 76 sbi_remote_fence_i(&others); 77 } else { 78 on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); 79 } 80 81 preempt_enable(); 82 } 83 84 #endif /* CONFIG_SMP */ 85 86 #ifdef CONFIG_MMU 87 void flush_icache_pte(struct mm_struct *mm, pte_t pte) 88 { 89 struct folio *folio = page_folio(pte_page(pte)); 90 91 if (!test_bit(PG_dcache_clean, &folio->flags)) { 92 flush_icache_mm(mm, false); 93 set_bit(PG_dcache_clean, &folio->flags); 94 } 95 } 96 #endif /* CONFIG_MMU */ 97 98 unsigned int riscv_cbom_block_size; 99 EXPORT_SYMBOL_GPL(riscv_cbom_block_size); 100 101 unsigned int riscv_cboz_block_size; 102 EXPORT_SYMBOL_GPL(riscv_cboz_block_size); 103 104 static void __init cbo_get_block_size(struct device_node *node, 105 const char *name, u32 *block_size, 106 unsigned long *first_hartid) 107 { 108 unsigned long hartid; 109 u32 val; 110 111 if (riscv_of_processor_hartid(node, &hartid)) 112 return; 113 114 if (of_property_read_u32(node, name, &val)) 115 return; 116 117 if (!*block_size) { 118 *block_size = val; 119 *first_hartid = hartid; 120 } else if (*block_size != val) { 121 pr_warn("%s mismatched between harts %lu and %lu\n", 122 name, *first_hartid, hartid); 123 } 124 } 125 126 void __init riscv_init_cbo_blocksizes(void) 127 { 128 unsigned long cbom_hartid, cboz_hartid; 129 u32 cbom_block_size = 0, cboz_block_size = 0; 130 struct device_node *node; 131 struct acpi_table_header *rhct; 132 acpi_status status; 133 134 if (acpi_disabled) { 135 for_each_of_cpu_node(node) { 136 /* set block-size for cbom and/or cboz extension if available */ 137 cbo_get_block_size(node, "riscv,cbom-block-size", 138 &cbom_block_size, &cbom_hartid); 139 cbo_get_block_size(node, "riscv,cboz-block-size", 140 &cboz_block_size, &cboz_hartid); 141 } 142 } else { 143 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); 144 if (ACPI_FAILURE(status)) 145 return; 146 147 acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL); 148 acpi_put_table((struct acpi_table_header *)rhct); 149 } 150 151 if (cbom_block_size) 152 riscv_cbom_block_size = cbom_block_size; 153 154 if (cboz_block_size) 155 riscv_cboz_block_size = cboz_block_size; 156 } 157 158 #ifdef CONFIG_SMP 159 static void set_icache_stale_mask(void) 160 { 161 int cpu = get_cpu(); 162 cpumask_t *mask; 163 bool stale_cpu; 164 165 /* 166 * Mark every other hart's icache as needing a flush for 167 * this MM. Maintain the previous value of the current 168 * cpu to handle the case when this function is called 169 * concurrently on different harts. 170 */ 171 mask = ¤t->mm->context.icache_stale_mask; 172 stale_cpu = cpumask_test_cpu(cpu, mask); 173 174 cpumask_setall(mask); 175 cpumask_assign_cpu(cpu, mask, stale_cpu); 176 put_cpu(); 177 } 178 #endif 179 180 /** 181 * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in 182 * userspace. 183 * @ctx: Set the type of icache flushing instructions permitted/prohibited in 184 * userspace. Supported values described below. 185 * 186 * Supported values for ctx: 187 * 188 * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space. 189 * 190 * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in 191 * a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``. 192 * Therefore, caution must be taken; use this flag only when you can guarantee 193 * that no thread in the process will emit fence.i from this point onward. 194 * 195 * @scope: Set scope of where icache flushing instructions are allowed to be 196 * emitted. Supported values described below. 197 * 198 * Supported values for scope: 199 * 200 * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process 201 * is coherent with instruction storage upon 202 * migration. 203 * 204 * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is 205 * coherent with instruction storage upon 206 * migration. 207 * 208 * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are 209 * permitted to emit icache flushing instructions. Whenever any thread in the 210 * process is migrated, the corresponding hart's icache will be guaranteed to be 211 * consistent with instruction storage. This does not enforce any guarantees 212 * outside of migration. If a thread modifies an instruction that another thread 213 * may attempt to execute, the other thread must still emit an icache flushing 214 * instruction before attempting to execute the potentially modified 215 * instruction. This must be performed by the user-space program. 216 * 217 * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the 218 * thread calling this function is permitted to emit icache flushing 219 * instructions. When the thread is migrated, the corresponding hart's icache 220 * will be guaranteed to be consistent with instruction storage. 221 * 222 * On kernels configured without SMP, this function is a nop as migrations 223 * across harts will not occur. 224 */ 225 int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope) 226 { 227 #ifdef CONFIG_SMP 228 switch (ctx) { 229 case PR_RISCV_CTX_SW_FENCEI_ON: 230 switch (scope) { 231 case PR_RISCV_SCOPE_PER_PROCESS: 232 current->mm->context.force_icache_flush = true; 233 break; 234 case PR_RISCV_SCOPE_PER_THREAD: 235 current->thread.force_icache_flush = true; 236 break; 237 default: 238 return -EINVAL; 239 } 240 break; 241 case PR_RISCV_CTX_SW_FENCEI_OFF: 242 switch (scope) { 243 case PR_RISCV_SCOPE_PER_PROCESS: 244 set_icache_stale_mask(); 245 current->mm->context.force_icache_flush = false; 246 break; 247 case PR_RISCV_SCOPE_PER_THREAD: 248 set_icache_stale_mask(); 249 current->thread.force_icache_flush = false; 250 break; 251 default: 252 return -EINVAL; 253 } 254 break; 255 default: 256 return -EINVAL; 257 } 258 return 0; 259 #else 260 switch (ctx) { 261 case PR_RISCV_CTX_SW_FENCEI_ON: 262 case PR_RISCV_CTX_SW_FENCEI_OFF: 263 return 0; 264 default: 265 return -EINVAL; 266 } 267 #endif 268 } 269