150acfb2bSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 208f051edSAndrew Waterman /* 308f051edSAndrew Waterman * Copyright (C) 2017 SiFive 408f051edSAndrew Waterman */ 508f051edSAndrew Waterman 62960f371SSunil V L #include <linux/acpi.h> 75c20a3a9SAndrew Jones #include <linux/of.h> 82960f371SSunil V L #include <asm/acpi.h> 908f051edSAndrew Waterman #include <asm/cacheflush.h> 1008f051edSAndrew Waterman 1158de7754SGary Guo #ifdef CONFIG_SMP 1258de7754SGary Guo 1358de7754SGary Guo #include <asm/sbi.h> 1458de7754SGary Guo 158bf90f32SChristoph Hellwig static void ipi_remote_fence_i(void *info) 168bf90f32SChristoph Hellwig { 178bf90f32SChristoph Hellwig return local_flush_icache_all(); 188bf90f32SChristoph Hellwig } 198bf90f32SChristoph Hellwig 2058de7754SGary Guo void flush_icache_all(void) 2158de7754SGary Guo { 22bb8958d5SAlexandre Ghiti local_flush_icache_all(); 23bb8958d5SAlexandre Ghiti 2462792284SAnup Patel if (IS_ENABLED(CONFIG_RISCV_SBI) && !riscv_use_ipi_for_rfence()) 2558de7754SGary Guo sbi_remote_fence_i(NULL); 268bf90f32SChristoph Hellwig else 278bf90f32SChristoph Hellwig on_each_cpu(ipi_remote_fence_i, NULL, 1); 2858de7754SGary Guo } 291833e327SOlof Johansson EXPORT_SYMBOL(flush_icache_all); 3058de7754SGary Guo 3158de7754SGary Guo /* 3258de7754SGary Guo * Performs an icache flush for the given MM context. RISC-V has no direct 3358de7754SGary Guo * mechanism for instruction cache shoot downs, so instead we send an IPI that 3458de7754SGary Guo * informs the remote harts they need to flush their local instruction caches. 3558de7754SGary Guo * To avoid pathologically slow behavior in a common case (a bunch of 3658de7754SGary Guo * single-hart processes on a many-hart machine, ie 'make -j') we avoid the 3758de7754SGary Guo * IPIs for harts that are not currently executing a MM context and instead 3858de7754SGary Guo * schedule a deferred local instruction cache flush to be performed before 3958de7754SGary Guo * execution resumes on each hart. 4058de7754SGary Guo */ 4158de7754SGary Guo void flush_icache_mm(struct mm_struct *mm, bool local) 4258de7754SGary Guo { 4358de7754SGary Guo unsigned int cpu; 448bf90f32SChristoph Hellwig cpumask_t others, *mask; 4558de7754SGary Guo 4658de7754SGary Guo preempt_disable(); 4758de7754SGary Guo 4858de7754SGary Guo /* Mark every hart's icache as needing a flush for this MM. */ 4958de7754SGary Guo mask = &mm->context.icache_stale_mask; 5058de7754SGary Guo cpumask_setall(mask); 5158de7754SGary Guo /* Flush this hart's I$ now, and mark it as flushed. */ 5258de7754SGary Guo cpu = smp_processor_id(); 5358de7754SGary Guo cpumask_clear_cpu(cpu, mask); 5458de7754SGary Guo local_flush_icache_all(); 5558de7754SGary Guo 5658de7754SGary Guo /* 5758de7754SGary Guo * Flush the I$ of other harts concurrently executing, and mark them as 5858de7754SGary Guo * flushed. 5958de7754SGary Guo */ 6058de7754SGary Guo cpumask_andnot(&others, mm_cpumask(mm), cpumask_of(cpu)); 6158de7754SGary Guo local |= cpumask_empty(&others); 628bf90f32SChristoph Hellwig if (mm == current->active_mm && local) { 6358de7754SGary Guo /* 6458de7754SGary Guo * It's assumed that at least one strongly ordered operation is 6558de7754SGary Guo * performed on this hart between setting a hart's cpumask bit 6658de7754SGary Guo * and scheduling this MM context on that hart. Sending an SBI 6758de7754SGary Guo * remote message will do this, but in the case where no 6858de7754SGary Guo * messages are sent we still need to order this hart's writes 6958de7754SGary Guo * with flush_icache_deferred(). 7058de7754SGary Guo */ 7158de7754SGary Guo smp_mb(); 7262792284SAnup Patel } else if (IS_ENABLED(CONFIG_RISCV_SBI) && 7362792284SAnup Patel !riscv_use_ipi_for_rfence()) { 7426fb751cSAtish Patra sbi_remote_fence_i(&others); 758bf90f32SChristoph Hellwig } else { 768bf90f32SChristoph Hellwig on_each_cpu_mask(&others, ipi_remote_fence_i, NULL, 1); 7758de7754SGary Guo } 7858de7754SGary Guo 7958de7754SGary Guo preempt_enable(); 8058de7754SGary Guo } 8158de7754SGary Guo 8258de7754SGary Guo #endif /* CONFIG_SMP */ 8358de7754SGary Guo 846bd33e1eSChristoph Hellwig #ifdef CONFIG_MMU 85*01261e24SAlexandre Ghiti void flush_icache_pte(struct mm_struct *mm, pte_t pte) 8608f051edSAndrew Waterman { 87864609c6SMatthew Wilcox (Oracle) struct folio *folio = page_folio(pte_page(pte)); 8808f051edSAndrew Waterman 89864609c6SMatthew Wilcox (Oracle) if (!test_bit(PG_dcache_clean, &folio->flags)) { 90*01261e24SAlexandre Ghiti flush_icache_mm(mm, false); 91864609c6SMatthew Wilcox (Oracle) set_bit(PG_dcache_clean, &folio->flags); 92950b879bSGuo Ren } 9308f051edSAndrew Waterman } 946bd33e1eSChristoph Hellwig #endif /* CONFIG_MMU */ 955c20a3a9SAndrew Jones 965c20a3a9SAndrew Jones unsigned int riscv_cbom_block_size; 975c20a3a9SAndrew Jones EXPORT_SYMBOL_GPL(riscv_cbom_block_size); 985c20a3a9SAndrew Jones 997ea5a736SAndrew Jones unsigned int riscv_cboz_block_size; 1007ea5a736SAndrew Jones EXPORT_SYMBOL_GPL(riscv_cboz_block_size); 1017ea5a736SAndrew Jones 1023b472f86SJisheng Zhang static void __init cbo_get_block_size(struct device_node *node, 1038b05e7d0SAndrew Jones const char *name, u32 *block_size, 1048b05e7d0SAndrew Jones unsigned long *first_hartid) 1055c20a3a9SAndrew Jones { 1065c20a3a9SAndrew Jones unsigned long hartid; 1078b05e7d0SAndrew Jones u32 val; 1085c20a3a9SAndrew Jones 1098b05e7d0SAndrew Jones if (riscv_of_processor_hartid(node, &hartid)) 1108b05e7d0SAndrew Jones return; 1115c20a3a9SAndrew Jones 1128b05e7d0SAndrew Jones if (of_property_read_u32(node, name, &val)) 1138b05e7d0SAndrew Jones return; 1145c20a3a9SAndrew Jones 1158b05e7d0SAndrew Jones if (!*block_size) { 1168b05e7d0SAndrew Jones *block_size = val; 1178b05e7d0SAndrew Jones *first_hartid = hartid; 1188b05e7d0SAndrew Jones } else if (*block_size != val) { 1198b05e7d0SAndrew Jones pr_warn("%s mismatched between harts %lu and %lu\n", 1208b05e7d0SAndrew Jones name, *first_hartid, hartid); 1215c20a3a9SAndrew Jones } 1225c20a3a9SAndrew Jones } 1235c20a3a9SAndrew Jones 1243b472f86SJisheng Zhang void __init riscv_init_cbo_blocksizes(void) 1255c20a3a9SAndrew Jones { 1267ea5a736SAndrew Jones unsigned long cbom_hartid, cboz_hartid; 1277ea5a736SAndrew Jones u32 cbom_block_size = 0, cboz_block_size = 0; 1285c20a3a9SAndrew Jones struct device_node *node; 1292960f371SSunil V L struct acpi_table_header *rhct; 1302960f371SSunil V L acpi_status status; 1315c20a3a9SAndrew Jones 1322960f371SSunil V L if (acpi_disabled) { 1335c20a3a9SAndrew Jones for_each_of_cpu_node(node) { 1347ea5a736SAndrew Jones /* set block-size for cbom and/or cboz extension if available */ 1358b05e7d0SAndrew Jones cbo_get_block_size(node, "riscv,cbom-block-size", 1367ea5a736SAndrew Jones &cbom_block_size, &cbom_hartid); 1377ea5a736SAndrew Jones cbo_get_block_size(node, "riscv,cboz-block-size", 1387ea5a736SAndrew Jones &cboz_block_size, &cboz_hartid); 1395c20a3a9SAndrew Jones } 1402960f371SSunil V L } else { 1412960f371SSunil V L status = acpi_get_table(ACPI_SIG_RHCT, 0, &rhct); 1422960f371SSunil V L if (ACPI_FAILURE(status)) 1432960f371SSunil V L return; 1442960f371SSunil V L 1452960f371SSunil V L acpi_get_cbo_block_size(rhct, &cbom_block_size, &cboz_block_size, NULL); 1462960f371SSunil V L acpi_put_table((struct acpi_table_header *)rhct); 1472960f371SSunil V L } 1485c20a3a9SAndrew Jones 1497ea5a736SAndrew Jones if (cbom_block_size) 1507ea5a736SAndrew Jones riscv_cbom_block_size = cbom_block_size; 1517ea5a736SAndrew Jones 1527ea5a736SAndrew Jones if (cboz_block_size) 1537ea5a736SAndrew Jones riscv_cboz_block_size = cboz_block_size; 1545c20a3a9SAndrew Jones } 155