1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * SiFive composable cache controller Driver 4 * 5 * Copyright (C) 2018-2022 SiFive, Inc. 6 * 7 */ 8 9 #define pr_fmt(fmt) "CCACHE: " fmt 10 11 #include <linux/align.h> 12 #include <linux/debugfs.h> 13 #include <linux/interrupt.h> 14 #include <linux/of_irq.h> 15 #include <linux/of_address.h> 16 #include <linux/device.h> 17 #include <linux/bitfield.h> 18 #include <asm/cacheflush.h> 19 #include <asm/cacheinfo.h> 20 #include <asm/dma-noncoherent.h> 21 #include <soc/sifive/sifive_ccache.h> 22 23 #define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100 24 #define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104 25 #define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108 26 27 #define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120 28 #define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124 29 #define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128 30 31 #define SIFIVE_CCACHE_DATECCFIX_LOW 0x140 32 #define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144 33 #define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148 34 35 #define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160 36 #define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164 37 #define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168 38 39 #define SIFIVE_CCACHE_CONFIG 0x00 40 #define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0) 41 #define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8) 42 #define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16) 43 #define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24) 44 45 #define SIFIVE_CCACHE_FLUSH64 0x200 46 #define SIFIVE_CCACHE_FLUSH32 0x240 47 48 #define SIFIVE_CCACHE_WAYENABLE 0x08 49 #define SIFIVE_CCACHE_ECCINJECTERR 0x40 50 51 #define SIFIVE_CCACHE_MAX_ECCINTR 4 52 #define SIFIVE_CCACHE_LINE_SIZE 64 53 54 static void __iomem *ccache_base; 55 static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR]; 56 static struct riscv_cacheinfo_ops ccache_cache_ops; 57 static int level; 58 59 enum { 60 DIR_CORR = 0, 61 DATA_CORR, 62 DATA_UNCORR, 63 DIR_UNCORR, 64 }; 65 66 enum { 67 QUIRK_NONSTANDARD_CACHE_OPS = BIT(0), 68 QUIRK_BROKEN_DATA_UNCORR = BIT(1), 69 }; 70 71 #ifdef CONFIG_DEBUG_FS 72 static struct dentry *sifive_test; 73 74 static ssize_t ccache_write(struct file *file, const char __user *data, 75 size_t count, loff_t *ppos) 76 { 77 unsigned int val; 78 79 if (kstrtouint_from_user(data, count, 0, &val)) 80 return -EINVAL; 81 if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) 82 writel(val, ccache_base + SIFIVE_CCACHE_ECCINJECTERR); 83 else 84 return -EINVAL; 85 return count; 86 } 87 88 static const struct file_operations ccache_fops = { 89 .owner = THIS_MODULE, 90 .open = simple_open, 91 .write = ccache_write 92 }; 93 94 static void setup_sifive_debug(void) 95 { 96 sifive_test = debugfs_create_dir("sifive_ccache_cache", NULL); 97 98 debugfs_create_file("sifive_debug_inject_error", 0200, 99 sifive_test, NULL, &ccache_fops); 100 } 101 #endif 102 103 static void ccache_config_read(void) 104 { 105 u32 cfg; 106 107 cfg = readl(ccache_base + SIFIVE_CCACHE_CONFIG); 108 pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n", 109 FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg), 110 FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg), 111 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)), 112 BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg))); 113 114 cfg = readl(ccache_base + SIFIVE_CCACHE_WAYENABLE); 115 pr_info("Index of the largest way enabled: %u\n", cfg); 116 } 117 118 static const struct of_device_id sifive_ccache_ids[] = { 119 { .compatible = "sifive,fu540-c000-ccache" }, 120 { .compatible = "sifive,fu740-c000-ccache" }, 121 { .compatible = "starfive,jh7100-ccache", 122 .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) }, 123 { .compatible = "sifive,ccache0" }, 124 { /* end of table */ } 125 }; 126 127 static ATOMIC_NOTIFIER_HEAD(ccache_err_chain); 128 129 int register_sifive_ccache_error_notifier(struct notifier_block *nb) 130 { 131 return atomic_notifier_chain_register(&ccache_err_chain, nb); 132 } 133 EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier); 134 135 int unregister_sifive_ccache_error_notifier(struct notifier_block *nb) 136 { 137 return atomic_notifier_chain_unregister(&ccache_err_chain, nb); 138 } 139 EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier); 140 141 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS 142 static void ccache_flush_range(phys_addr_t start, size_t len) 143 { 144 phys_addr_t end = start + len; 145 phys_addr_t line; 146 147 if (!len) 148 return; 149 150 mb(); 151 for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end; 152 line += SIFIVE_CCACHE_LINE_SIZE) { 153 #ifdef CONFIG_32BIT 154 writel(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32); 155 #else 156 writeq(line, ccache_base + SIFIVE_CCACHE_FLUSH64); 157 #endif 158 mb(); 159 } 160 } 161 162 static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = { 163 .wback = &ccache_flush_range, 164 .inv = &ccache_flush_range, 165 .wback_inv = &ccache_flush_range, 166 }; 167 #endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */ 168 169 static int ccache_largest_wayenabled(void) 170 { 171 return readl(ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF; 172 } 173 174 static ssize_t number_of_ways_enabled_show(struct device *dev, 175 struct device_attribute *attr, 176 char *buf) 177 { 178 return sprintf(buf, "%u\n", ccache_largest_wayenabled()); 179 } 180 181 static DEVICE_ATTR_RO(number_of_ways_enabled); 182 183 static struct attribute *priv_attrs[] = { 184 &dev_attr_number_of_ways_enabled.attr, 185 NULL, 186 }; 187 188 static const struct attribute_group priv_attr_group = { 189 .attrs = priv_attrs, 190 }; 191 192 static const struct attribute_group *ccache_get_priv_group(struct cacheinfo 193 *this_leaf) 194 { 195 /* We want to use private group for composable cache only */ 196 if (this_leaf->level == level) 197 return &priv_attr_group; 198 else 199 return NULL; 200 } 201 202 static irqreturn_t ccache_int_handler(int irq, void *device) 203 { 204 unsigned int add_h, add_l; 205 206 if (irq == g_irq[DIR_CORR]) { 207 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH); 208 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW); 209 pr_err("DirError @ 0x%08X.%08X\n", add_h, add_l); 210 /* Reading this register clears the DirError interrupt sig */ 211 readl(ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT); 212 atomic_notifier_call_chain(&ccache_err_chain, 213 SIFIVE_CCACHE_ERR_TYPE_CE, 214 "DirECCFix"); 215 } 216 if (irq == g_irq[DIR_UNCORR]) { 217 add_h = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH); 218 add_l = readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW); 219 /* Reading this register clears the DirFail interrupt sig */ 220 readl(ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT); 221 atomic_notifier_call_chain(&ccache_err_chain, 222 SIFIVE_CCACHE_ERR_TYPE_UE, 223 "DirECCFail"); 224 panic("CCACHE: DirFail @ 0x%08X.%08X\n", add_h, add_l); 225 } 226 if (irq == g_irq[DATA_CORR]) { 227 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH); 228 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW); 229 pr_err("DataError @ 0x%08X.%08X\n", add_h, add_l); 230 /* Reading this register clears the DataError interrupt sig */ 231 readl(ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT); 232 atomic_notifier_call_chain(&ccache_err_chain, 233 SIFIVE_CCACHE_ERR_TYPE_CE, 234 "DatECCFix"); 235 } 236 if (irq == g_irq[DATA_UNCORR]) { 237 add_h = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH); 238 add_l = readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW); 239 pr_err("DataFail @ 0x%08X.%08X\n", add_h, add_l); 240 /* Reading this register clears the DataFail interrupt sig */ 241 readl(ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT); 242 atomic_notifier_call_chain(&ccache_err_chain, 243 SIFIVE_CCACHE_ERR_TYPE_UE, 244 "DatECCFail"); 245 } 246 247 return IRQ_HANDLED; 248 } 249 250 static int __init sifive_ccache_init(void) 251 { 252 struct device_node *np; 253 struct resource res; 254 int i, rc, intr_num; 255 const struct of_device_id *match; 256 unsigned long quirks; 257 258 np = of_find_matching_node_and_match(NULL, sifive_ccache_ids, &match); 259 if (!np) 260 return -ENODEV; 261 262 quirks = (uintptr_t)match->data; 263 264 if (of_address_to_resource(np, 0, &res)) { 265 rc = -ENODEV; 266 goto err_node_put; 267 } 268 269 ccache_base = ioremap(res.start, resource_size(&res)); 270 if (!ccache_base) { 271 rc = -ENOMEM; 272 goto err_node_put; 273 } 274 275 if (of_property_read_u32(np, "cache-level", &level)) { 276 rc = -ENOENT; 277 goto err_unmap; 278 } 279 280 intr_num = of_property_count_u32_elems(np, "interrupts"); 281 if (!intr_num) { 282 pr_err("No interrupts property\n"); 283 rc = -ENODEV; 284 goto err_unmap; 285 } 286 287 for (i = 0; i < intr_num; i++) { 288 g_irq[i] = irq_of_parse_and_map(np, i); 289 290 if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR)) 291 continue; 292 293 rc = request_irq(g_irq[i], ccache_int_handler, 0, "ccache_ecc", 294 NULL); 295 if (rc) { 296 pr_err("Could not request IRQ %d\n", g_irq[i]); 297 goto err_free_irq; 298 } 299 } 300 of_node_put(np); 301 302 #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS 303 if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) { 304 riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE; 305 riscv_noncoherent_supported(); 306 riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops); 307 } 308 #endif 309 310 ccache_config_read(); 311 312 ccache_cache_ops.get_priv_group = ccache_get_priv_group; 313 riscv_set_cacheinfo_ops(&ccache_cache_ops); 314 315 #ifdef CONFIG_DEBUG_FS 316 setup_sifive_debug(); 317 #endif 318 return 0; 319 320 err_free_irq: 321 while (--i >= 0) 322 free_irq(g_irq[i], NULL); 323 err_unmap: 324 iounmap(ccache_base); 325 err_node_put: 326 of_node_put(np); 327 return rc; 328 } 329 330 arch_initcall(sifive_ccache_init); 331