1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2017 SiFive 4 */ 5 6 #include <linux/acpi.h> 7 #include <linux/of.h> 8 #include <linux/prctl.h> 9 #include <asm/acpi.h> 10 #include <asm/cacheflush.h> 11 12 #ifdef CONFIG_SMP 13 14 #include <asm/sbi.h> 15 16 static void ipi_remote_fence_i(void *info) 17 { 18 return local_flush_icache_all(); 19 } 20 21 void flush_icache_all(void) 22 { 23 local_flush_icache_all(); 24 25 if (num_online_cpus() < 2) 26 return; 27 else if (riscv_use_sbi_for_rfence()) 28 sbi_remote_fence_i(NULL); 29 else 30 on_each_cpu(ipi_remote_fence_i, NULL, 1); 31 } 32 EXPORT_SYMBOL(flush_icache_all); 33 34 /* 35 * Performs an icache flush for the given MM context. RISC-V has no direct 36 * mechanism for instruction cache shoot downs, so instead we send an IPI that 37 * informs the remote harts they need to flush their local instruction caches. 38 * To avoid pathologically slow behavior in a common case (a bunch of 39 * single-hart processes on a many-hart machine, ie 'make -j') we avoid the 40 * IPIs for harts that are not currently executing a MM context and instead 41 * schedule a deferred local instruction cache flush to be performed before 42 * execution resumes on each hart. 43 */ 44 void flush_icache_mm(struct mm_struct *mm, bool local) 45 { 46 unsigned int cpu; 47 cpumask_t others, *mask; 48 49 preempt_disable(); 50 51 /* Mark every hart's icache as needing a flush for this MM. */ 52 mask = &mm->context.icache_stale_mask; 53 cpumask_setall(mask); 54 /* Flush this hart's I$ now, and mark it as flushed. */ 55 cpu = smp_processor_id(); 56 cpumask_clear_cpu(cpu, mask); 57 local_flush_icache_all(); 58 59 /* 60 * Flush the I$ of other harts concurrently executing, and mark them as 61 * flushed. 62 */ 63 cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); 64 local |= cpumask_empty(&others); 65 if (mm == current->active_mm && local) { 66 /* 67 * It's assumed that at least one strongly ordered operation is 68 * performed on this hart between setting a hart's cpumask bit 69 * and scheduling this MM context on that hart. Sending an SBI 70 * remote message will do this, but in the case where no 71 * messages are sent we still need to order this hart's writes 72 * with flush_icache_deferred(). 73 */ 74 smp_mb(); 75 } else if (riscv_use_sbi_for_rfence()) { 76 sbi_remote_fence_i(&others); 77 } else { 78 on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); 79 } 80 81 preempt_enable(); 82 } 83 84 #endif /* CONFIG_SMP */ 85 86 #ifdef CONFIG_MMU 87 void flush_icache_pte(struct mm_struct *mm, pte_t pte) 88 { 89 struct folio *folio = page_folio(pte_page(pte)); 90 91 if (!test_bit(PG_dcache_clean, &folio->flags)) { 92 flush_icache_mm(mm, false); 93 set_bit(PG_dcache_clean, &folio->flags); 94 } 95 } 96 #endif /* CONFIG_MMU */ 97 98 unsigned int riscv_cbom_block_size; 99 EXPORT_SYMBOL_GPL(riscv_cbom_block_size); 100 101 unsigned int riscv_cboz_block_size; 102 EXPORT_SYMBOL_GPL(riscv_cboz_block_size); 103 104 static void __init cbo_get_block_size(struct device_node *node, 105 const char *name, u32 *block_size, 106 unsigned long *first_hartid) 107 { 108 unsigned long hartid; 109 u32 val; 110 111 if (riscv_of_processor_hartid(node, &hartid)) 112 return; 113 114 if (of_property_read_u32(node, name, &val)) 115 return; 116 117 if (!*block_size) { 118 *block_size = val; 119 *first_hartid = hartid; 120 } else if (*block_size != val) { 121 pr_warn("%s mismatched between harts %lu and %lu\n", 122 name, *first_hartid, hartid); 123 } 124 } 125 126 void __init riscv_init_cbo_blocksizes(void) 127 { 128 unsigned long cbom_hartid, cboz_hartid; 129 u32 cbom_block_size = 0, cboz_block_size = 0; 130 struct device_node *node; 131 struct acpi_table_header *rhct; 132 acpi_status status; 133 134 if (acpi_disabled) { 135 for_each_of_cpu_node(node) { 136 /* set block-size for cbom and/or cboz extension if available */ 137 cbo_get_block_size(node, "riscv,cbom-block-size", 138 &cbom_block_size, &cbom_hartid); 139 cbo_get_block_size(node, "riscv,cboz-block-size", 140 &cboz_block_size, &cboz_hartid); 141 } 142 } else { 143 status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); 144 if (ACPI_FAILURE(status)) 145 return; 146 147 acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL); 148 acpi_put_table((struct acpi_table_header *)rhct); 149 } 150 151 if (cbom_block_size) 152 riscv_cbom_block_size = cbom_block_size; 153 154 if (cboz_block_size) 155 riscv_cboz_block_size = cboz_block_size; 156 } 157 158 #ifdef CONFIG_SMP 159 static void set_icache_stale_mask(void) 160 { 161 cpumask_t *mask; 162 bool stale_cpu; 163 164 /* 165 * Mark every other hart's icache as needing a flush for 166 * this MM. Maintain the previous value of the current 167 * cpu to handle the case when this function is called 168 * concurrently on different harts. 169 */ 170 mask = ¤t->mm->context.icache_stale_mask; 171 stale_cpu = cpumask_test_cpu(smp_processor_id(), mask); 172 173 cpumask_setall(mask); 174 cpumask_assign_cpu(smp_processor_id(), mask, stale_cpu); 175 } 176 #endif 177 178 /** 179 * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in 180 * userspace. 181 * @ctx: Set the type of icache flushing instructions permitted/prohibited in 182 * userspace. Supported values described below. 183 * 184 * Supported values for ctx: 185 * 186 * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space. 187 * 188 * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in 189 * a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``. 190 * Therefore, caution must be taken; use this flag only when you can guarantee 191 * that no thread in the process will emit fence.i from this point onward. 192 * 193 * @scope: Set scope of where icache flushing instructions are allowed to be 194 * emitted. Supported values described below. 195 * 196 * Supported values for scope: 197 * 198 * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process 199 * is coherent with instruction storage upon 200 * migration. 201 * 202 * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is 203 * coherent with instruction storage upon 204 * migration. 205 * 206 * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are 207 * permitted to emit icache flushing instructions. Whenever any thread in the 208 * process is migrated, the corresponding hart's icache will be guaranteed to be 209 * consistent with instruction storage. This does not enforce any guarantees 210 * outside of migration. If a thread modifies an instruction that another thread 211 * may attempt to execute, the other thread must still emit an icache flushing 212 * instruction before attempting to execute the potentially modified 213 * instruction. This must be performed by the user-space program. 214 * 215 * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the 216 * thread calling this function is permitted to emit icache flushing 217 * instructions. When the thread is migrated, the corresponding hart's icache 218 * will be guaranteed to be consistent with instruction storage. 219 * 220 * On kernels configured without SMP, this function is a nop as migrations 221 * across harts will not occur. 222 */ 223 int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope) 224 { 225 #ifdef CONFIG_SMP 226 switch (ctx) { 227 case PR_RISCV_CTX_SW_FENCEI_ON: 228 switch (scope) { 229 case PR_RISCV_SCOPE_PER_PROCESS: 230 current->mm->context.force_icache_flush = true; 231 break; 232 case PR_RISCV_SCOPE_PER_THREAD: 233 current->thread.force_icache_flush = true; 234 break; 235 default: 236 return -EINVAL; 237 } 238 break; 239 case PR_RISCV_CTX_SW_FENCEI_OFF: 240 switch (scope) { 241 case PR_RISCV_SCOPE_PER_PROCESS: 242 current->mm->context.force_icache_flush = false; 243 244 set_icache_stale_mask(); 245 break; 246 case PR_RISCV_SCOPE_PER_THREAD: 247 current->thread.force_icache_flush = false; 248 249 set_icache_stale_mask(); 250 break; 251 default: 252 return -EINVAL; 253 } 254 break; 255 default: 256 return -EINVAL; 257 } 258 return 0; 259 #else 260 switch (ctx) { 261 case PR_RISCV_CTX_SW_FENCEI_ON: 262 case PR_RISCV_CTX_SW_FENCEI_OFF: 263 return 0; 264 default: 265 return -EINVAL; 266 } 267 #endif 268 } 269