1 /* 2 * CCI cache coherent interconnect driver 3 * 4 * Copyright (C) 2013 ARM Ltd. 5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any 12 * kind, whether express or implied; without even the implied warranty 13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 */ 16 17 #include <linux/arm-cci.h> 18 #include <linux/io.h> 19 #include <linux/module.h> 20 #include <linux/of_address.h> 21 #include <linux/of_platform.h> 22 #include <linux/platform_device.h> 23 #include <linux/slab.h> 24 25 #include <asm/cacheflush.h> 26 #include <asm/smp_plat.h> 27 28 static void __iomem *cci_ctrl_base __ro_after_init; 29 static unsigned long cci_ctrl_phys __ro_after_init; 30 31 #ifdef CONFIG_ARM_CCI400_PORT_CTRL 32 struct cci_nb_ports { 33 unsigned int nb_ace; 34 unsigned int nb_ace_lite; 35 }; 36 37 static const struct cci_nb_ports cci400_ports = { 38 .nb_ace = 2, 39 .nb_ace_lite = 3 40 }; 41 42 #define CCI400_PORTS_DATA (&cci400_ports) 43 #else 44 #define CCI400_PORTS_DATA (NULL) 45 #endif 46 47 static const struct of_device_id arm_cci_matches[] = { 48 #ifdef CONFIG_ARM_CCI400_COMMON 49 {.compatible = "arm,cci-400", .data = CCI400_PORTS_DATA }, 50 #endif 51 #ifdef CONFIG_ARM_CCI5xx_PMU 52 { .compatible = "arm,cci-500", }, 53 { .compatible = "arm,cci-550", }, 54 #endif 55 {}, 56 }; 57 58 static const struct of_dev_auxdata arm_cci_auxdata[] = { 59 OF_DEV_AUXDATA("arm,cci-400-pmu", 0, NULL, &cci_ctrl_base), 60 OF_DEV_AUXDATA("arm,cci-400-pmu,r0", 0, NULL, &cci_ctrl_base), 61 OF_DEV_AUXDATA("arm,cci-400-pmu,r1", 0, NULL, &cci_ctrl_base), 62 OF_DEV_AUXDATA("arm,cci-500-pmu,r0", 0, NULL, &cci_ctrl_base), 63 OF_DEV_AUXDATA("arm,cci-550-pmu,r0", 0, NULL, &cci_ctrl_base), 64 {} 65 }; 66 67 #define DRIVER_NAME "ARM-CCI" 68 69 static int cci_platform_probe(struct platform_device *pdev) 70 { 71 if (!cci_probed()) 72 return -ENODEV; 73 74 return of_platform_populate(pdev->dev.of_node, NULL, 75 arm_cci_auxdata, &pdev->dev); 76 } 77 78 static struct platform_driver cci_platform_driver = { 79 .driver = { 80 .name = DRIVER_NAME, 81 .of_match_table = arm_cci_matches, 82 }, 83 .probe = cci_platform_probe, 84 }; 85 86 static int __init cci_platform_init(void) 87 { 88 return platform_driver_register(&cci_platform_driver); 89 } 90 91 #ifdef CONFIG_ARM_CCI400_PORT_CTRL 92 93 #define CCI_PORT_CTRL 0x0 94 #define CCI_CTRL_STATUS 0xc 95 96 #define CCI_ENABLE_SNOOP_REQ 0x1 97 #define CCI_ENABLE_DVM_REQ 0x2 98 #define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ) 99 100 enum cci_ace_port_type { 101 ACE_INVALID_PORT = 0x0, 102 ACE_PORT, 103 ACE_LITE_PORT, 104 }; 105 106 struct cci_ace_port { 107 void __iomem *base; 108 unsigned long phys; 109 enum cci_ace_port_type type; 110 struct device_node *dn; 111 }; 112 113 static struct cci_ace_port *ports; 114 static unsigned int nb_cci_ports; 115 116 struct cpu_port { 117 u64 mpidr; 118 u32 port; 119 }; 120 121 /* 122 * Use the port MSB as valid flag, shift can be made dynamic 123 * by computing number of bits required for port indexes. 124 * Code disabling CCI cpu ports runs with D-cache invalidated 125 * and SCTLR bit clear so data accesses must be kept to a minimum 126 * to improve performance; for now shift is left static to 127 * avoid one more data access while disabling the CCI port. 128 */ 129 #define PORT_VALID_SHIFT 31 130 #define PORT_VALID (0x1 << PORT_VALID_SHIFT) 131 132 static inline void init_cpu_port(struct cpu_port *port, u32 index, u64 mpidr) 133 { 134 port->port = PORT_VALID | index; 135 port->mpidr = mpidr; 136 } 137 138 static inline bool cpu_port_is_valid(struct cpu_port *port) 139 { 140 return !!(port->port & PORT_VALID); 141 } 142 143 static inline bool cpu_port_match(struct cpu_port *port, u64 mpidr) 144 { 145 return port->mpidr == (mpidr & MPIDR_HWID_BITMASK); 146 } 147 148 static struct cpu_port cpu_port[NR_CPUS]; 149 150 /** 151 * __cci_ace_get_port - Function to retrieve the port index connected to 152 * a cpu or device. 153 * 154 * @dn: device node of the device to look-up 155 * @type: port type 156 * 157 * Return value: 158 * - CCI port index if success 159 * - -ENODEV if failure 160 */ 161 static int __cci_ace_get_port(struct device_node *dn, int type) 162 { 163 int i; 164 bool ace_match; 165 struct device_node *cci_portn; 166 167 cci_portn = of_parse_phandle(dn, "cci-control-port", 0); 168 for (i = 0; i < nb_cci_ports; i++) { 169 ace_match = ports[i].type == type; 170 if (ace_match && cci_portn == ports[i].dn) 171 return i; 172 } 173 return -ENODEV; 174 } 175 176 int cci_ace_get_port(struct device_node *dn) 177 { 178 return __cci_ace_get_port(dn, ACE_LITE_PORT); 179 } 180 EXPORT_SYMBOL_GPL(cci_ace_get_port); 181 182 static void cci_ace_init_ports(void) 183 { 184 int port, cpu; 185 struct device_node *cpun; 186 187 /* 188 * Port index look-up speeds up the function disabling ports by CPU, 189 * since the logical to port index mapping is done once and does 190 * not change after system boot. 191 * The stashed index array is initialized for all possible CPUs 192 * at probe time. 193 */ 194 for_each_possible_cpu(cpu) { 195 /* too early to use cpu->of_node */ 196 cpun = of_get_cpu_node(cpu, NULL); 197 198 if (WARN(!cpun, "Missing cpu device node\n")) 199 continue; 200 201 port = __cci_ace_get_port(cpun, ACE_PORT); 202 if (port < 0) 203 continue; 204 205 init_cpu_port(&cpu_port[cpu], port, cpu_logical_map(cpu)); 206 } 207 208 for_each_possible_cpu(cpu) { 209 WARN(!cpu_port_is_valid(&cpu_port[cpu]), 210 "CPU %u does not have an associated CCI port\n", 211 cpu); 212 } 213 } 214 /* 215 * Functions to enable/disable a CCI interconnect slave port 216 * 217 * They are called by low-level power management code to disable slave 218 * interfaces snoops and DVM broadcast. 219 * Since they may execute with cache data allocation disabled and 220 * after the caches have been cleaned and invalidated the functions provide 221 * no explicit locking since they may run with D-cache disabled, so normal 222 * cacheable kernel locks based on ldrex/strex may not work. 223 * Locking has to be provided by BSP implementations to ensure proper 224 * operations. 225 */ 226 227 /** 228 * cci_port_control() - function to control a CCI port 229 * 230 * @port: index of the port to setup 231 * @enable: if true enables the port, if false disables it 232 */ 233 static void notrace cci_port_control(unsigned int port, bool enable) 234 { 235 void __iomem *base = ports[port].base; 236 237 writel_relaxed(enable ? CCI_ENABLE_REQ : 0, base + CCI_PORT_CTRL); 238 /* 239 * This function is called from power down procedures 240 * and must not execute any instruction that might 241 * cause the processor to be put in a quiescent state 242 * (eg wfi). Hence, cpu_relax() can not be added to this 243 * read loop to optimize power, since it might hide possibly 244 * disruptive operations. 245 */ 246 while (readl_relaxed(cci_ctrl_base + CCI_CTRL_STATUS) & 0x1) 247 ; 248 } 249 250 /** 251 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU 252 * reference 253 * 254 * @mpidr: mpidr of the CPU whose CCI port should be disabled 255 * 256 * Disabling a CCI port for a CPU implies disabling the CCI port 257 * controlling that CPU cluster. Code disabling CPU CCI ports 258 * must make sure that the CPU running the code is the last active CPU 259 * in the cluster ie all other CPUs are quiescent in a low power state. 260 * 261 * Return: 262 * 0 on success 263 * -ENODEV on port look-up failure 264 */ 265 int notrace cci_disable_port_by_cpu(u64 mpidr) 266 { 267 int cpu; 268 bool is_valid; 269 for (cpu = 0; cpu < nr_cpu_ids; cpu++) { 270 is_valid = cpu_port_is_valid(&cpu_port[cpu]); 271 if (is_valid && cpu_port_match(&cpu_port[cpu], mpidr)) { 272 cci_port_control(cpu_port[cpu].port, false); 273 return 0; 274 } 275 } 276 return -ENODEV; 277 } 278 EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu); 279 280 /** 281 * cci_enable_port_for_self() - enable a CCI port for calling CPU 282 * 283 * Enabling a CCI port for the calling CPU implies enabling the CCI 284 * port controlling that CPU's cluster. Caller must make sure that the 285 * CPU running the code is the first active CPU in the cluster and all 286 * other CPUs are quiescent in a low power state or waiting for this CPU 287 * to complete the CCI initialization. 288 * 289 * Because this is called when the MMU is still off and with no stack, 290 * the code must be position independent and ideally rely on callee 291 * clobbered registers only. To achieve this we must code this function 292 * entirely in assembler. 293 * 294 * On success this returns with the proper CCI port enabled. In case of 295 * any failure this never returns as the inability to enable the CCI is 296 * fatal and there is no possible recovery at this stage. 297 */ 298 asmlinkage void __naked cci_enable_port_for_self(void) 299 { 300 asm volatile ("\n" 301 " .arch armv7-a\n" 302 " mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n" 303 " and r0, r0, #"__stringify(MPIDR_HWID_BITMASK)" \n" 304 " adr r1, 5f \n" 305 " ldr r2, [r1] \n" 306 " add r1, r1, r2 @ &cpu_port \n" 307 " add ip, r1, %[sizeof_cpu_port] \n" 308 309 /* Loop over the cpu_port array looking for a matching MPIDR */ 310 "1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n" 311 " cmp r2, r0 @ compare MPIDR \n" 312 " bne 2f \n" 313 314 /* Found a match, now test port validity */ 315 " ldr r3, [r1, %[offsetof_cpu_port_port]] \n" 316 " tst r3, #"__stringify(PORT_VALID)" \n" 317 " bne 3f \n" 318 319 /* no match, loop with the next cpu_port entry */ 320 "2: add r1, r1, %[sizeof_struct_cpu_port] \n" 321 " cmp r1, ip @ done? \n" 322 " blo 1b \n" 323 324 /* CCI port not found -- cheaply try to stall this CPU */ 325 "cci_port_not_found: \n" 326 " wfi \n" 327 " wfe \n" 328 " b cci_port_not_found \n" 329 330 /* Use matched port index to look up the corresponding ports entry */ 331 "3: bic r3, r3, #"__stringify(PORT_VALID)" \n" 332 " adr r0, 6f \n" 333 " ldmia r0, {r1, r2} \n" 334 " sub r1, r1, r0 @ virt - phys \n" 335 " ldr r0, [r0, r2] @ *(&ports) \n" 336 " mov r2, %[sizeof_struct_ace_port] \n" 337 " mla r0, r2, r3, r0 @ &ports[index] \n" 338 " sub r0, r0, r1 @ virt_to_phys() \n" 339 340 /* Enable the CCI port */ 341 " ldr r0, [r0, %[offsetof_port_phys]] \n" 342 " mov r3, %[cci_enable_req]\n" 343 " str r3, [r0, #"__stringify(CCI_PORT_CTRL)"] \n" 344 345 /* poll the status reg for completion */ 346 " adr r1, 7f \n" 347 " ldr r0, [r1] \n" 348 " ldr r0, [r0, r1] @ cci_ctrl_base \n" 349 "4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS)"] \n" 350 " tst r1, %[cci_control_status_bits] \n" 351 " bne 4b \n" 352 353 " mov r0, #0 \n" 354 " bx lr \n" 355 356 " .align 2 \n" 357 "5: .word cpu_port - . \n" 358 "6: .word . \n" 359 " .word ports - 6b \n" 360 "7: .word cci_ctrl_phys - . \n" 361 : : 362 [sizeof_cpu_port] "i" (sizeof(cpu_port)), 363 [cci_enable_req] "i" cpu_to_le32(CCI_ENABLE_REQ), 364 [cci_control_status_bits] "i" cpu_to_le32(1), 365 #ifndef __ARMEB__ 366 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)), 367 #else 368 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)+4), 369 #endif 370 [offsetof_cpu_port_port] "i" (offsetof(struct cpu_port, port)), 371 [sizeof_struct_cpu_port] "i" (sizeof(struct cpu_port)), 372 [sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)), 373 [offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) ); 374 375 unreachable(); 376 } 377 378 /** 379 * __cci_control_port_by_device() - function to control a CCI port by device 380 * reference 381 * 382 * @dn: device node pointer of the device whose CCI port should be 383 * controlled 384 * @enable: if true enables the port, if false disables it 385 * 386 * Return: 387 * 0 on success 388 * -ENODEV on port look-up failure 389 */ 390 int notrace __cci_control_port_by_device(struct device_node *dn, bool enable) 391 { 392 int port; 393 394 if (!dn) 395 return -ENODEV; 396 397 port = __cci_ace_get_port(dn, ACE_LITE_PORT); 398 if (WARN_ONCE(port < 0, "node %pOF ACE lite port look-up failure\n", 399 dn)) 400 return -ENODEV; 401 cci_port_control(port, enable); 402 return 0; 403 } 404 EXPORT_SYMBOL_GPL(__cci_control_port_by_device); 405 406 /** 407 * __cci_control_port_by_index() - function to control a CCI port by port index 408 * 409 * @port: port index previously retrieved with cci_ace_get_port() 410 * @enable: if true enables the port, if false disables it 411 * 412 * Return: 413 * 0 on success 414 * -ENODEV on port index out of range 415 * -EPERM if operation carried out on an ACE PORT 416 */ 417 int notrace __cci_control_port_by_index(u32 port, bool enable) 418 { 419 if (port >= nb_cci_ports || ports[port].type == ACE_INVALID_PORT) 420 return -ENODEV; 421 /* 422 * CCI control for ports connected to CPUS is extremely fragile 423 * and must be made to go through a specific and controlled 424 * interface (ie cci_disable_port_by_cpu(); control by general purpose 425 * indexing is therefore disabled for ACE ports. 426 */ 427 if (ports[port].type == ACE_PORT) 428 return -EPERM; 429 430 cci_port_control(port, enable); 431 return 0; 432 } 433 EXPORT_SYMBOL_GPL(__cci_control_port_by_index); 434 435 static const struct of_device_id arm_cci_ctrl_if_matches[] = { 436 {.compatible = "arm,cci-400-ctrl-if", }, 437 {}, 438 }; 439 440 static int cci_probe_ports(struct device_node *np) 441 { 442 struct cci_nb_ports const *cci_config; 443 int ret, i, nb_ace = 0, nb_ace_lite = 0; 444 struct device_node *cp; 445 struct resource res; 446 const char *match_str; 447 bool is_ace; 448 449 450 cci_config = of_match_node(arm_cci_matches, np)->data; 451 if (!cci_config) 452 return -ENODEV; 453 454 nb_cci_ports = cci_config->nb_ace + cci_config->nb_ace_lite; 455 456 ports = kcalloc(nb_cci_ports, sizeof(*ports), GFP_KERNEL); 457 if (!ports) 458 return -ENOMEM; 459 460 for_each_available_child_of_node(np, cp) { 461 if (!of_match_node(arm_cci_ctrl_if_matches, cp)) 462 continue; 463 464 i = nb_ace + nb_ace_lite; 465 466 if (i >= nb_cci_ports) 467 break; 468 469 if (of_property_read_string(cp, "interface-type", 470 &match_str)) { 471 WARN(1, "node %pOF missing interface-type property\n", 472 cp); 473 continue; 474 } 475 is_ace = strcmp(match_str, "ace") == 0; 476 if (!is_ace && strcmp(match_str, "ace-lite")) { 477 WARN(1, "node %pOF containing invalid interface-type property, skipping it\n", 478 cp); 479 continue; 480 } 481 482 ret = of_address_to_resource(cp, 0, &res); 483 if (!ret) { 484 ports[i].base = ioremap(res.start, resource_size(&res)); 485 ports[i].phys = res.start; 486 } 487 if (ret || !ports[i].base) { 488 WARN(1, "unable to ioremap CCI port %d\n", i); 489 continue; 490 } 491 492 if (is_ace) { 493 if (WARN_ON(nb_ace >= cci_config->nb_ace)) 494 continue; 495 ports[i].type = ACE_PORT; 496 ++nb_ace; 497 } else { 498 if (WARN_ON(nb_ace_lite >= cci_config->nb_ace_lite)) 499 continue; 500 ports[i].type = ACE_LITE_PORT; 501 ++nb_ace_lite; 502 } 503 ports[i].dn = cp; 504 } 505 506 /* 507 * If there is no CCI port that is under kernel control 508 * return early and report probe status. 509 */ 510 if (!nb_ace && !nb_ace_lite) 511 return -ENODEV; 512 513 /* initialize a stashed array of ACE ports to speed-up look-up */ 514 cci_ace_init_ports(); 515 516 /* 517 * Multi-cluster systems may need this data when non-coherent, during 518 * cluster power-up/power-down. Make sure it reaches main memory. 519 */ 520 sync_cache_w(&cci_ctrl_base); 521 sync_cache_w(&cci_ctrl_phys); 522 sync_cache_w(&ports); 523 sync_cache_w(&cpu_port); 524 __sync_cache_range_w(ports, sizeof(*ports) * nb_cci_ports); 525 pr_info("ARM CCI driver probed\n"); 526 527 return 0; 528 } 529 #else /* !CONFIG_ARM_CCI400_PORT_CTRL */ 530 static inline int cci_probe_ports(struct device_node *np) 531 { 532 return 0; 533 } 534 #endif /* CONFIG_ARM_CCI400_PORT_CTRL */ 535 536 static int cci_probe(void) 537 { 538 int ret; 539 struct device_node *np; 540 struct resource res; 541 542 np = of_find_matching_node(NULL, arm_cci_matches); 543 if (!of_device_is_available(np)) 544 return -ENODEV; 545 546 ret = of_address_to_resource(np, 0, &res); 547 if (!ret) { 548 cci_ctrl_base = ioremap(res.start, resource_size(&res)); 549 cci_ctrl_phys = res.start; 550 } 551 if (ret || !cci_ctrl_base) { 552 WARN(1, "unable to ioremap CCI ctrl\n"); 553 return -ENXIO; 554 } 555 556 return cci_probe_ports(np); 557 } 558 559 static int cci_init_status = -EAGAIN; 560 static DEFINE_MUTEX(cci_probing); 561 562 static int cci_init(void) 563 { 564 if (cci_init_status != -EAGAIN) 565 return cci_init_status; 566 567 mutex_lock(&cci_probing); 568 if (cci_init_status == -EAGAIN) 569 cci_init_status = cci_probe(); 570 mutex_unlock(&cci_probing); 571 return cci_init_status; 572 } 573 574 /* 575 * To sort out early init calls ordering a helper function is provided to 576 * check if the CCI driver has beed initialized. Function check if the driver 577 * has been initialized, if not it calls the init function that probes 578 * the driver and updates the return value. 579 */ 580 bool cci_probed(void) 581 { 582 return cci_init() == 0; 583 } 584 EXPORT_SYMBOL_GPL(cci_probed); 585 586 early_initcall(cci_init); 587 core_initcall(cci_platform_init); 588 MODULE_LICENSE("GPL"); 589 MODULE_DESCRIPTION("ARM CCI support"); 590