150acfb2bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
208f051edSAndrew Waterman /*
308f051edSAndrew Waterman * Copyright (C) 2017 SiFive
408f051edSAndrew Waterman */
508f051edSAndrew Waterman
62960f371SSunil V L #include <linux/acpi.h>
75c20a3a9SAndrew Jones #include <linux/of.h>
86b9391b5SCharlie Jenkins #include <linux/prctl.h>
92960f371SSunil V L #include <asm/acpi.h>
1008f051edSAndrew Waterman #include <asm/cacheflush.h>
1108f051edSAndrew Waterman
1258de7754SGary Guo #ifdef CONFIG_SMP
1358de7754SGary Guo
1458de7754SGary Guo #include <asm/sbi.h>
1558de7754SGary Guo
ipi_remote_fence_i(void * info)168bf90f32SChristoph Hellwig static void ipi_remote_fence_i(void *info)
178bf90f32SChristoph Hellwig {
188bf90f32SChristoph Hellwig return local_flush_icache_all();
198bf90f32SChristoph Hellwig }
208bf90f32SChristoph Hellwig
flush_icache_all(void)2158de7754SGary Guo void flush_icache_all(void)
2258de7754SGary Guo {
23bb8958d5SAlexandre Ghiti local_flush_icache_all();
24bb8958d5SAlexandre Ghiti
259546f004SSamuel Holland if (num_online_cpus() < 2)
269546f004SSamuel Holland return;
279546f004SSamuel Holland else if (riscv_use_sbi_for_rfence())
2858de7754SGary Guo sbi_remote_fence_i(NULL);
298bf90f32SChristoph Hellwig else
308bf90f32SChristoph Hellwig on_each_cpu(ipi_remote_fence_i, NULL, 1);
3158de7754SGary Guo }
321833e327SOlof Johansson EXPORT_SYMBOL(flush_icache_all);
3358de7754SGary Guo
3458de7754SGary Guo /*
3558de7754SGary Guo * Performs an icache flush for the given MM context. RISC-V has no direct
3658de7754SGary Guo * mechanism for instruction cache shoot downs, so instead we send an IPI that
3758de7754SGary Guo * informs the remote harts they need to flush their local instruction caches.
3858de7754SGary Guo * To avoid pathologically slow behavior in a common case (a bunch of
3958de7754SGary Guo * single-hart processes on a many-hart machine, ie 'make -j') we avoid the
4058de7754SGary Guo * IPIs for harts that are not currently executing a MM context and instead
4158de7754SGary Guo * schedule a deferred local instruction cache flush to be performed before
4258de7754SGary Guo * execution resumes on each hart.
4358de7754SGary Guo */
flush_icache_mm(struct mm_struct * mm,bool local)4458de7754SGary Guo void flush_icache_mm(struct mm_struct *mm, bool local)
4558de7754SGary Guo {
4658de7754SGary Guo unsigned int cpu;
478bf90f32SChristoph Hellwig cpumask_t others, *mask;
4858de7754SGary Guo
4958de7754SGary Guo preempt_disable();
5058de7754SGary Guo
5158de7754SGary Guo /* Mark every hart's icache as needing a flush for this MM. */
5258de7754SGary Guo mask = &mm->context.icache_stale_mask;
5358de7754SGary Guo cpumask_setall(mask);
5458de7754SGary Guo /* Flush this hart's I$ now, and mark it as flushed. */
5558de7754SGary Guo cpu = smp_processor_id();
5658de7754SGary Guo cpumask_clear_cpu(cpu, mask);
5758de7754SGary Guo local_flush_icache_all();
5858de7754SGary Guo
5958de7754SGary Guo /*
6058de7754SGary Guo * Flush the I$ of other harts concurrently executing, and mark them as
6158de7754SGary Guo * flushed.
6258de7754SGary Guo */
6358de7754SGary Guo cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu));
6458de7754SGary Guo local |= cpumask_empty(&others);
658bf90f32SChristoph Hellwig if (mm == current->active_mm && local) {
6658de7754SGary Guo /*
6758de7754SGary Guo * It's assumed that at least one strongly ordered operation is
6858de7754SGary Guo * performed on this hart between setting a hart's cpumask bit
6958de7754SGary Guo * and scheduling this MM context on that hart. Sending an SBI
7058de7754SGary Guo * remote message will do this, but in the case where no
7158de7754SGary Guo * messages are sent we still need to order this hart's writes
7258de7754SGary Guo * with flush_icache_deferred().
7358de7754SGary Guo */
7458de7754SGary Guo smp_mb();
75dc892fb4SSamuel Holland } else if (riscv_use_sbi_for_rfence()) {
7626fb751cSAtish Patra sbi_remote_fence_i(&others);
778bf90f32SChristoph Hellwig } else {
788bf90f32SChristoph Hellwig on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1);
7958de7754SGary Guo }
8058de7754SGary Guo
8158de7754SGary Guo preempt_enable();
8258de7754SGary Guo }
8358de7754SGary Guo
8458de7754SGary Guo #endif /* CONFIG_SMP */
8558de7754SGary Guo
866bd33e1eSChristoph Hellwig #ifdef CONFIG_MMU
flush_icache_pte(struct mm_struct * mm,pte_t pte)8701261e24SAlexandre Ghiti void flush_icache_pte(struct mm_struct *mm, pte_t pte)
8808f051edSAndrew Waterman {
89864609c6SMatthew Wilcox (Oracle) struct folio *folio = page_folio(pte_page(pte));
9008f051edSAndrew Waterman
91864609c6SMatthew Wilcox (Oracle) if (!test_bit(PG_dcache_clean, &folio->flags)) {
9201261e24SAlexandre Ghiti flush_icache_mm(mm, false);
93864609c6SMatthew Wilcox (Oracle) set_bit(PG_dcache_clean, &folio->flags);
94950b879bSGuo Ren }
9508f051edSAndrew Waterman }
966bd33e1eSChristoph Hellwig #endif /* CONFIG_MMU */
975c20a3a9SAndrew Jones
985c20a3a9SAndrew Jones unsigned int riscv_cbom_block_size;
995c20a3a9SAndrew Jones EXPORT_SYMBOL_GPL(riscv_cbom_block_size);
1005c20a3a9SAndrew Jones
1017ea5a736SAndrew Jones unsigned int riscv_cboz_block_size;
1027ea5a736SAndrew Jones EXPORT_SYMBOL_GPL(riscv_cboz_block_size);
1037ea5a736SAndrew Jones
cbo_get_block_size(struct device_node * node,const char * name,u32 * block_size,unsigned long * first_hartid)1043b472f86SJisheng Zhang static void __init cbo_get_block_size(struct device_node *node,
1058b05e7d0SAndrew Jones const char *name, u32 *block_size,
1068b05e7d0SAndrew Jones unsigned long *first_hartid)
1075c20a3a9SAndrew Jones {
1085c20a3a9SAndrew Jones unsigned long hartid;
1098b05e7d0SAndrew Jones u32 val;
1105c20a3a9SAndrew Jones
1118b05e7d0SAndrew Jones if (riscv_of_processor_hartid(node, &hartid))
1128b05e7d0SAndrew Jones return;
1135c20a3a9SAndrew Jones
1148b05e7d0SAndrew Jones if (of_property_read_u32(node, name, &val))
1158b05e7d0SAndrew Jones return;
1165c20a3a9SAndrew Jones
1178b05e7d0SAndrew Jones if (!*block_size) {
1188b05e7d0SAndrew Jones *block_size = val;
1198b05e7d0SAndrew Jones *first_hartid = hartid;
1208b05e7d0SAndrew Jones } else if (*block_size != val) {
1218b05e7d0SAndrew Jones pr_warn("%s mismatched between harts %lu and %lu\n",
1228b05e7d0SAndrew Jones name, *first_hartid, hartid);
1235c20a3a9SAndrew Jones }
1245c20a3a9SAndrew Jones }
1255c20a3a9SAndrew Jones
riscv_init_cbo_blocksizes(void)1263b472f86SJisheng Zhang void __init riscv_init_cbo_blocksizes(void)
1275c20a3a9SAndrew Jones {
1287ea5a736SAndrew Jones unsigned long cbom_hartid, cboz_hartid;
1297ea5a736SAndrew Jones u32 cbom_block_size = 0, cboz_block_size = 0;
1305c20a3a9SAndrew Jones struct device_node *node;
1312960f371SSunil V L struct acpi_table_header *rhct;
1322960f371SSunil V L acpi_status status;
1335c20a3a9SAndrew Jones
1342960f371SSunil V L if (acpi_disabled) {
1355c20a3a9SAndrew Jones for_each_of_cpu_node(node) {
1367ea5a736SAndrew Jones /* set block-size for cbom and/or cboz extension if available */
1378b05e7d0SAndrew Jones cbo_get_block_size(node, "riscv,cbom-block-size",
1387ea5a736SAndrew Jones &cbom_block_size, &cbom_hartid);
1397ea5a736SAndrew Jones cbo_get_block_size(node, "riscv,cboz-block-size",
1407ea5a736SAndrew Jones &cboz_block_size, &cboz_hartid);
1415c20a3a9SAndrew Jones }
1422960f371SSunil V L } else {
1432960f371SSunil V L status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct);
1442960f371SSunil V L if (ACPI_FAILURE(status))
1452960f371SSunil V L return;
1462960f371SSunil V L
1472960f371SSunil V L acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL);
1482960f371SSunil V L acpi_put_table((struct acpi_table_header *)rhct);
1492960f371SSunil V L }
1505c20a3a9SAndrew Jones
1517ea5a736SAndrew Jones if (cbom_block_size)
1527ea5a736SAndrew Jones riscv_cbom_block_size = cbom_block_size;
1537ea5a736SAndrew Jones
1547ea5a736SAndrew Jones if (cboz_block_size)
1557ea5a736SAndrew Jones riscv_cboz_block_size = cboz_block_size;
1565c20a3a9SAndrew Jones }
1576b9391b5SCharlie Jenkins
1586b9391b5SCharlie Jenkins #ifdef CONFIG_SMP
set_icache_stale_mask(void)1596b9391b5SCharlie Jenkins static void set_icache_stale_mask(void)
1606b9391b5SCharlie Jenkins {
161*7c1e5b96SCharlie Jenkins int cpu = get_cpu();
1626b9391b5SCharlie Jenkins cpumask_t *mask;
1636b9391b5SCharlie Jenkins bool stale_cpu;
1646b9391b5SCharlie Jenkins
1656b9391b5SCharlie Jenkins /*
1666b9391b5SCharlie Jenkins * Mark every other hart's icache as needing a flush for
1676b9391b5SCharlie Jenkins * this MM. Maintain the previous value of the current
1686b9391b5SCharlie Jenkins * cpu to handle the case when this function is called
1696b9391b5SCharlie Jenkins * concurrently on different harts.
1706b9391b5SCharlie Jenkins */
1716b9391b5SCharlie Jenkins mask = ¤t->mm->context.icache_stale_mask;
172*7c1e5b96SCharlie Jenkins stale_cpu = cpumask_test_cpu(cpu, mask);
1736b9391b5SCharlie Jenkins
1746b9391b5SCharlie Jenkins cpumask_setall(mask);
175*7c1e5b96SCharlie Jenkins cpumask_assign_cpu(cpu, mask, stale_cpu);
176*7c1e5b96SCharlie Jenkins put_cpu();
1776b9391b5SCharlie Jenkins }
1786b9391b5SCharlie Jenkins #endif
1796b9391b5SCharlie Jenkins
1806b9391b5SCharlie Jenkins /**
1816b9391b5SCharlie Jenkins * riscv_set_icache_flush_ctx() - Enable/disable icache flushing instructions in
1826b9391b5SCharlie Jenkins * userspace.
1836b9391b5SCharlie Jenkins * @ctx: Set the type of icache flushing instructions permitted/prohibited in
1846b9391b5SCharlie Jenkins * userspace. Supported values described below.
1856b9391b5SCharlie Jenkins *
1866b9391b5SCharlie Jenkins * Supported values for ctx:
1876b9391b5SCharlie Jenkins *
1886b9391b5SCharlie Jenkins * * %PR_RISCV_CTX_SW_FENCEI_ON: Allow fence.i in user space.
1896b9391b5SCharlie Jenkins *
1906b9391b5SCharlie Jenkins * * %PR_RISCV_CTX_SW_FENCEI_OFF: Disallow fence.i in user space. All threads in
1916b9391b5SCharlie Jenkins * a process will be affected when ``scope == PR_RISCV_SCOPE_PER_PROCESS``.
1926b9391b5SCharlie Jenkins * Therefore, caution must be taken; use this flag only when you can guarantee
1936b9391b5SCharlie Jenkins * that no thread in the process will emit fence.i from this point onward.
1946b9391b5SCharlie Jenkins *
1956b9391b5SCharlie Jenkins * @scope: Set scope of where icache flushing instructions are allowed to be
1966b9391b5SCharlie Jenkins * emitted. Supported values described below.
1976b9391b5SCharlie Jenkins *
1986b9391b5SCharlie Jenkins * Supported values for scope:
1996b9391b5SCharlie Jenkins *
2006b9391b5SCharlie Jenkins * * %PR_RISCV_SCOPE_PER_PROCESS: Ensure the icache of any thread in this process
2016b9391b5SCharlie Jenkins * is coherent with instruction storage upon
2026b9391b5SCharlie Jenkins * migration.
2036b9391b5SCharlie Jenkins *
2046b9391b5SCharlie Jenkins * * %PR_RISCV_SCOPE_PER_THREAD: Ensure the icache of the current thread is
2056b9391b5SCharlie Jenkins * coherent with instruction storage upon
2066b9391b5SCharlie Jenkins * migration.
2076b9391b5SCharlie Jenkins *
2086b9391b5SCharlie Jenkins * When ``scope == PR_RISCV_SCOPE_PER_PROCESS``, all threads in the process are
2096b9391b5SCharlie Jenkins * permitted to emit icache flushing instructions. Whenever any thread in the
2106b9391b5SCharlie Jenkins * process is migrated, the corresponding hart's icache will be guaranteed to be
2116b9391b5SCharlie Jenkins * consistent with instruction storage. This does not enforce any guarantees
2126b9391b5SCharlie Jenkins * outside of migration. If a thread modifies an instruction that another thread
2136b9391b5SCharlie Jenkins * may attempt to execute, the other thread must still emit an icache flushing
2146b9391b5SCharlie Jenkins * instruction before attempting to execute the potentially modified
2156b9391b5SCharlie Jenkins * instruction. This must be performed by the user-space program.
2166b9391b5SCharlie Jenkins *
2176b9391b5SCharlie Jenkins * In per-thread context (eg. ``scope == PR_RISCV_SCOPE_PER_THREAD``) only the
2186b9391b5SCharlie Jenkins * thread calling this function is permitted to emit icache flushing
2196b9391b5SCharlie Jenkins * instructions. When the thread is migrated, the corresponding hart's icache
2206b9391b5SCharlie Jenkins * will be guaranteed to be consistent with instruction storage.
2216b9391b5SCharlie Jenkins *
2226b9391b5SCharlie Jenkins * On kernels configured without SMP, this function is a nop as migrations
2236b9391b5SCharlie Jenkins * across harts will not occur.
2246b9391b5SCharlie Jenkins */
riscv_set_icache_flush_ctx(unsigned long ctx,unsigned long scope)2256b9391b5SCharlie Jenkins int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long scope)
2266b9391b5SCharlie Jenkins {
2276b9391b5SCharlie Jenkins #ifdef CONFIG_SMP
2286b9391b5SCharlie Jenkins switch (ctx) {
2296b9391b5SCharlie Jenkins case PR_RISCV_CTX_SW_FENCEI_ON:
2306b9391b5SCharlie Jenkins switch (scope) {
2316b9391b5SCharlie Jenkins case PR_RISCV_SCOPE_PER_PROCESS:
2326b9391b5SCharlie Jenkins current->mm->context.force_icache_flush = true;
2336b9391b5SCharlie Jenkins break;
2346b9391b5SCharlie Jenkins case PR_RISCV_SCOPE_PER_THREAD:
2356b9391b5SCharlie Jenkins current->thread.force_icache_flush = true;
2366b9391b5SCharlie Jenkins break;
2376b9391b5SCharlie Jenkins default:
2386b9391b5SCharlie Jenkins return -EINVAL;
2396b9391b5SCharlie Jenkins }
2406b9391b5SCharlie Jenkins break;
2416b9391b5SCharlie Jenkins case PR_RISCV_CTX_SW_FENCEI_OFF:
2426b9391b5SCharlie Jenkins switch (scope) {
2436b9391b5SCharlie Jenkins case PR_RISCV_SCOPE_PER_PROCESS:
2446b9391b5SCharlie Jenkins set_icache_stale_mask();
245*7c1e5b96SCharlie Jenkins current->mm->context.force_icache_flush = false;
2466b9391b5SCharlie Jenkins break;
2476b9391b5SCharlie Jenkins case PR_RISCV_SCOPE_PER_THREAD:
2486b9391b5SCharlie Jenkins set_icache_stale_mask();
249*7c1e5b96SCharlie Jenkins current->thread.force_icache_flush = false;
2506b9391b5SCharlie Jenkins break;
2516b9391b5SCharlie Jenkins default:
2526b9391b5SCharlie Jenkins return -EINVAL;
2536b9391b5SCharlie Jenkins }
2546b9391b5SCharlie Jenkins break;
2556b9391b5SCharlie Jenkins default:
2566b9391b5SCharlie Jenkins return -EINVAL;
2576b9391b5SCharlie Jenkins }
2586b9391b5SCharlie Jenkins return 0;
2596b9391b5SCharlie Jenkins #else
2606b9391b5SCharlie Jenkins switch (ctx) {
2616b9391b5SCharlie Jenkins case PR_RISCV_CTX_SW_FENCEI_ON:
2626b9391b5SCharlie Jenkins case PR_RISCV_CTX_SW_FENCEI_OFF:
2636b9391b5SCharlie Jenkins return 0;
2646b9391b5SCharlie Jenkins default:
2656b9391b5SCharlie Jenkins return -EINVAL;
2666b9391b5SCharlie Jenkins }
2676b9391b5SCharlie Jenkins #endif
2686b9391b5SCharlie Jenkins }
269