1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2015-2016 Socionext Inc. 4 * Author: Masahiro Yamada <yamada.masahiro@socionext.com> 5 */ 6 7 #define pr_fmt(fmt) "uniphier: " fmt 8 9 #include <linux/bitops.h> 10 #include <linux/init.h> 11 #include <linux/io.h> 12 #include <linux/log2.h> 13 #include <linux/of_address.h> 14 #include <linux/slab.h> 15 #include <asm/hardware/cache-uniphier.h> 16 #include <asm/outercache.h> 17 18 /* control registers */ 19 #define UNIPHIER_SSCC 0x0 /* Control Register */ 20 #define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */ 21 #define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */ 22 #define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */ 23 #define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */ 24 #define UNIPHIER_SSCC_ON BIT(0) /* enable cache */ 25 #define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */ 26 #define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */ 27 28 /* revision registers */ 29 #define UNIPHIER_SSCID 0x0 /* ID Register */ 30 31 /* operation registers */ 32 #define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */ 33 #define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */ 34 #define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */ 35 #define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */ 36 #define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */ 37 #define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */ 38 #define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */ 39 #define UNIPHIER_SSCOQM_S_MASK (0x3 << 17) 40 #define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17) 41 #define UNIPHIER_SSCOQM_S_ALL (0x1 << 17) 42 #define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */ 43 #define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */ 44 #define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */ 45 #define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */ 46 #define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */ 47 #define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */ 48 #define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/ 49 #define UNIPHIER_SSCOPPQSEF_FE BIT(1) 50 #define UNIPHIER_SSCOPPQSEF_OE BIT(0) 51 #define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */ 52 #define UNIPHIER_SSCOLPQS_EF BIT(2) 53 #define UNIPHIER_SSCOLPQS_EST BIT(1) 54 #define UNIPHIER_SSCOLPQS_QST BIT(0) 55 56 /* Is the operation region specified by address range? */ 57 #define UNIPHIER_SSCOQM_S_IS_RANGE(op) \ 58 ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE) 59 60 /** 61 * struct uniphier_cache_data - UniPhier outer cache specific data 62 * 63 * @ctrl_base: virtual base address of control registers 64 * @rev_base: virtual base address of revision registers 65 * @op_base: virtual base address of operation registers 66 * @way_ctrl_base: virtual address of the way control registers for this 67 * SoC revision 68 * @way_mask: each bit specifies if the way is present 69 * @nsets: number of associativity sets 70 * @line_size: line size in bytes 71 * @range_op_max_size: max size that can be handled by a single range operation 72 * @list: list node to include this level in the whole cache hierarchy 73 */ 74 struct uniphier_cache_data { 75 void __iomem *ctrl_base; 76 void __iomem *rev_base; 77 void __iomem *op_base; 78 void __iomem *way_ctrl_base; 79 u32 way_mask; 80 u32 nsets; 81 u32 line_size; 82 u32 range_op_max_size; 83 struct list_head list; 84 }; 85 86 /* 87 * List of the whole outer cache hierarchy. This list is only modified during 88 * the early boot stage, so no mutex is taken for the access to the list. 89 */ 90 static LIST_HEAD(uniphier_cache_list); 91 92 /** 93 * __uniphier_cache_sync - perform a sync point for a particular cache level 94 * 95 * @data: cache controller specific data 96 */ 97 static void __uniphier_cache_sync(struct uniphier_cache_data *data) 98 { 99 /* This sequence need not be atomic. Do not disable IRQ. */ 100 writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC, 101 data->op_base + UNIPHIER_SSCOPE); 102 /* need a read back to confirm */ 103 readl_relaxed(data->op_base + UNIPHIER_SSCOPE); 104 } 105 106 /** 107 * __uniphier_cache_maint_common - run a queue operation for a particular level 108 * 109 * @data: cache controller specific data 110 * @start: start address of range operation (don't care for "all" operation) 111 * @size: data size of range operation (don't care for "all" operation) 112 * @operation: flags to specify the desired cache operation 113 */ 114 static void __uniphier_cache_maint_common(struct uniphier_cache_data *data, 115 unsigned long start, 116 unsigned long size, 117 u32 operation) 118 { 119 unsigned long flags; 120 121 /* 122 * No spin lock is necessary here because: 123 * 124 * [1] This outer cache controller is able to accept maintenance 125 * operations from multiple CPUs at a time in an SMP system; if a 126 * maintenance operation is under way and another operation is issued, 127 * the new one is stored in the queue. The controller performs one 128 * operation after another. If the queue is full, the status register, 129 * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has 130 * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have 131 * different instances for each CPU, i.e. each CPU can track the status 132 * of the maintenance operations triggered by itself. 133 * 134 * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ, 135 * SSCOQWN}, are shared between multiple CPUs, but the hardware still 136 * guarantees the registration sequence is atomic; the write access to 137 * them are arbitrated by the hardware. The first accessor to the 138 * register, UNIPHIER_SSCOQM, holds the access right and it is released 139 * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU 140 * is holding the access right, other CPUs fail to register operations. 141 * One CPU should not hold the access right for a long time, so local 142 * IRQs should be disabled while the following sequence. 143 */ 144 local_irq_save(flags); 145 146 /* clear the complete notification flag */ 147 writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS); 148 149 do { 150 /* set cache operation */ 151 writel_relaxed(UNIPHIER_SSCOQM_CE | operation, 152 data->op_base + UNIPHIER_SSCOQM); 153 154 /* set address range if needed */ 155 if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) { 156 writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD); 157 writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ); 158 } 159 } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) & 160 (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE))); 161 162 /* wait until the operation is completed */ 163 while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) != 164 UNIPHIER_SSCOLPQS_EF)) 165 cpu_relax(); 166 167 local_irq_restore(flags); 168 } 169 170 static void __uniphier_cache_maint_all(struct uniphier_cache_data *data, 171 u32 operation) 172 { 173 __uniphier_cache_maint_common(data, 0, 0, 174 UNIPHIER_SSCOQM_S_ALL | operation); 175 176 __uniphier_cache_sync(data); 177 } 178 179 static void __uniphier_cache_maint_range(struct uniphier_cache_data *data, 180 unsigned long start, unsigned long end, 181 u32 operation) 182 { 183 unsigned long size; 184 185 /* 186 * If the start address is not aligned, 187 * perform a cache operation for the first cache-line 188 */ 189 start = start & ~(data->line_size - 1); 190 191 size = end - start; 192 193 if (unlikely(size >= (unsigned long)(-data->line_size))) { 194 /* this means cache operation for all range */ 195 __uniphier_cache_maint_all(data, operation); 196 return; 197 } 198 199 /* 200 * If the end address is not aligned, 201 * perform a cache operation for the last cache-line 202 */ 203 size = ALIGN(size, data->line_size); 204 205 while (size) { 206 unsigned long chunk_size = min_t(unsigned long, size, 207 data->range_op_max_size); 208 209 __uniphier_cache_maint_common(data, start, chunk_size, 210 UNIPHIER_SSCOQM_S_RANGE | operation); 211 212 start += chunk_size; 213 size -= chunk_size; 214 } 215 216 __uniphier_cache_sync(data); 217 } 218 219 static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on) 220 { 221 u32 val = 0; 222 223 if (on) 224 val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON; 225 226 writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC); 227 } 228 229 static void __init __uniphier_cache_set_active_ways( 230 struct uniphier_cache_data *data) 231 { 232 unsigned int cpu; 233 234 for_each_possible_cpu(cpu) 235 writel_relaxed(data->way_mask, data->way_ctrl_base + 4 * cpu); 236 } 237 238 static void uniphier_cache_maint_range(unsigned long start, unsigned long end, 239 u32 operation) 240 { 241 struct uniphier_cache_data *data; 242 243 list_for_each_entry(data, &uniphier_cache_list, list) 244 __uniphier_cache_maint_range(data, start, end, operation); 245 } 246 247 static void uniphier_cache_maint_all(u32 operation) 248 { 249 struct uniphier_cache_data *data; 250 251 list_for_each_entry(data, &uniphier_cache_list, list) 252 __uniphier_cache_maint_all(data, operation); 253 } 254 255 static void uniphier_cache_inv_range(unsigned long start, unsigned long end) 256 { 257 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV); 258 } 259 260 static void uniphier_cache_clean_range(unsigned long start, unsigned long end) 261 { 262 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN); 263 } 264 265 static void uniphier_cache_flush_range(unsigned long start, unsigned long end) 266 { 267 uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH); 268 } 269 270 static void __init uniphier_cache_inv_all(void) 271 { 272 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV); 273 } 274 275 static void uniphier_cache_flush_all(void) 276 { 277 uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH); 278 } 279 280 static void uniphier_cache_disable(void) 281 { 282 struct uniphier_cache_data *data; 283 284 list_for_each_entry_reverse(data, &uniphier_cache_list, list) 285 __uniphier_cache_enable(data, false); 286 287 uniphier_cache_flush_all(); 288 } 289 290 static void __init uniphier_cache_enable(void) 291 { 292 struct uniphier_cache_data *data; 293 294 uniphier_cache_inv_all(); 295 296 list_for_each_entry(data, &uniphier_cache_list, list) { 297 __uniphier_cache_enable(data, true); 298 __uniphier_cache_set_active_ways(data); 299 } 300 } 301 302 static void uniphier_cache_sync(void) 303 { 304 struct uniphier_cache_data *data; 305 306 list_for_each_entry(data, &uniphier_cache_list, list) 307 __uniphier_cache_sync(data); 308 } 309 310 static const struct of_device_id uniphier_cache_match[] __initconst = { 311 { .compatible = "socionext,uniphier-system-cache" }, 312 { /* sentinel */ } 313 }; 314 315 static int __init __uniphier_cache_init(struct device_node *np, 316 unsigned int *cache_level) 317 { 318 struct uniphier_cache_data *data; 319 u32 level, cache_size; 320 struct device_node *next_np; 321 int ret = 0; 322 323 if (!of_match_node(uniphier_cache_match, np)) { 324 pr_err("L%d: not compatible with uniphier cache\n", 325 *cache_level); 326 return -EINVAL; 327 } 328 329 if (of_property_read_u32(np, "cache-level", &level)) { 330 pr_err("L%d: cache-level is not specified\n", *cache_level); 331 return -EINVAL; 332 } 333 334 if (level != *cache_level) { 335 pr_err("L%d: cache-level is unexpected value %d\n", 336 *cache_level, level); 337 return -EINVAL; 338 } 339 340 if (!of_property_read_bool(np, "cache-unified")) { 341 pr_err("L%d: cache-unified is not specified\n", *cache_level); 342 return -EINVAL; 343 } 344 345 data = kzalloc(sizeof(*data), GFP_KERNEL); 346 if (!data) 347 return -ENOMEM; 348 349 if (of_property_read_u32(np, "cache-line-size", &data->line_size) || 350 !is_power_of_2(data->line_size)) { 351 pr_err("L%d: cache-line-size is unspecified or invalid\n", 352 *cache_level); 353 ret = -EINVAL; 354 goto err; 355 } 356 357 if (of_property_read_u32(np, "cache-sets", &data->nsets) || 358 !is_power_of_2(data->nsets)) { 359 pr_err("L%d: cache-sets is unspecified or invalid\n", 360 *cache_level); 361 ret = -EINVAL; 362 goto err; 363 } 364 365 if (of_property_read_u32(np, "cache-size", &cache_size) || 366 cache_size == 0 || cache_size % (data->nsets * data->line_size)) { 367 pr_err("L%d: cache-size is unspecified or invalid\n", 368 *cache_level); 369 ret = -EINVAL; 370 goto err; 371 } 372 373 data->way_mask = GENMASK(cache_size / data->nsets / data->line_size - 1, 374 0); 375 376 data->ctrl_base = of_iomap(np, 0); 377 if (!data->ctrl_base) { 378 pr_err("L%d: failed to map control register\n", *cache_level); 379 ret = -ENOMEM; 380 goto err; 381 } 382 383 data->rev_base = of_iomap(np, 1); 384 if (!data->rev_base) { 385 pr_err("L%d: failed to map revision register\n", *cache_level); 386 ret = -ENOMEM; 387 goto err; 388 } 389 390 data->op_base = of_iomap(np, 2); 391 if (!data->op_base) { 392 pr_err("L%d: failed to map operation register\n", *cache_level); 393 ret = -ENOMEM; 394 goto err; 395 } 396 397 data->way_ctrl_base = data->ctrl_base + 0xc00; 398 399 if (*cache_level == 2) { 400 u32 revision = readl(data->rev_base + UNIPHIER_SSCID); 401 /* 402 * The size of range operation is limited to (1 << 22) or less 403 * for PH-sLD8 or older SoCs. 404 */ 405 if (revision <= 0x16) 406 data->range_op_max_size = (u32)1 << 22; 407 408 /* 409 * Unfortunatly, the offset address of active way control base 410 * varies from SoC to SoC. 411 */ 412 switch (revision) { 413 case 0x11: /* sLD3 */ 414 data->way_ctrl_base = data->ctrl_base + 0x870; 415 break; 416 case 0x12: /* LD4 */ 417 case 0x16: /* sld8 */ 418 data->way_ctrl_base = data->ctrl_base + 0x840; 419 break; 420 default: 421 break; 422 } 423 } 424 425 data->range_op_max_size -= data->line_size; 426 427 INIT_LIST_HEAD(&data->list); 428 list_add_tail(&data->list, &uniphier_cache_list); /* no mutex */ 429 430 /* 431 * OK, this level has been successfully initialized. Look for the next 432 * level cache. Do not roll back even if the initialization of the 433 * next level cache fails because we want to continue with available 434 * cache levels. 435 */ 436 next_np = of_find_next_cache_node(np); 437 if (next_np) { 438 (*cache_level)++; 439 ret = __uniphier_cache_init(next_np, cache_level); 440 } 441 of_node_put(next_np); 442 443 return ret; 444 err: 445 iounmap(data->op_base); 446 iounmap(data->rev_base); 447 iounmap(data->ctrl_base); 448 kfree(data); 449 450 return ret; 451 } 452 453 int __init uniphier_cache_init(void) 454 { 455 struct device_node *np = NULL; 456 unsigned int cache_level; 457 int ret = 0; 458 459 /* look for level 2 cache */ 460 while ((np = of_find_matching_node(np, uniphier_cache_match))) 461 if (!of_property_read_u32(np, "cache-level", &cache_level) && 462 cache_level == 2) 463 break; 464 465 if (!np) 466 return -ENODEV; 467 468 ret = __uniphier_cache_init(np, &cache_level); 469 of_node_put(np); 470 471 if (ret) { 472 /* 473 * Error out iif L2 initialization fails. Continue with any 474 * error on L3 or outer because they are optional. 475 */ 476 if (cache_level == 2) { 477 pr_err("failed to initialize L2 cache\n"); 478 return ret; 479 } 480 481 cache_level--; 482 ret = 0; 483 } 484 485 outer_cache.inv_range = uniphier_cache_inv_range; 486 outer_cache.clean_range = uniphier_cache_clean_range; 487 outer_cache.flush_range = uniphier_cache_flush_range; 488 outer_cache.flush_all = uniphier_cache_flush_all; 489 outer_cache.disable = uniphier_cache_disable; 490 outer_cache.sync = uniphier_cache_sync; 491 492 uniphier_cache_enable(); 493 494 pr_info("enabled outer cache (cache level: %d)\n", cache_level); 495 496 return ret; 497 } 498