1 /* 2 * CCI cache coherent interconnect driver 3 * 4 * Copyright (C) 2013 ARM Ltd. 5 * Author: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed "as is" WITHOUT ANY WARRANTY of any 12 * kind, whether express or implied; without even the implied warranty 13 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 */ 16 17 #include <linux/arm-cci.h> 18 #include <linux/io.h> 19 #include <linux/interrupt.h> 20 #include <linux/module.h> 21 #include <linux/of_address.h> 22 #include <linux/of_irq.h> 23 #include <linux/of_platform.h> 24 #include <linux/perf_event.h> 25 #include <linux/platform_device.h> 26 #include <linux/slab.h> 27 #include <linux/spinlock.h> 28 29 #include <asm/cacheflush.h> 30 #include <asm/smp_plat.h> 31 32 static void __iomem *cci_ctrl_base; 33 static unsigned long cci_ctrl_phys; 34 35 #ifdef CONFIG_ARM_CCI400_PORT_CTRL 36 struct cci_nb_ports { 37 unsigned int nb_ace; 38 unsigned int nb_ace_lite; 39 }; 40 41 static const struct cci_nb_ports cci400_ports = { 42 .nb_ace = 2, 43 .nb_ace_lite = 3 44 }; 45 46 #define CCI400_PORTS_DATA (&cci400_ports) 47 #else 48 #define CCI400_PORTS_DATA (NULL) 49 #endif 50 51 static const struct of_device_id arm_cci_matches[] = { 52 #ifdef CONFIG_ARM_CCI400_COMMON 53 {.compatible = "arm,cci-400", .data = CCI400_PORTS_DATA }, 54 #endif 55 #ifdef CONFIG_ARM_CCI5xx_PMU 56 { .compatible = "arm,cci-500", }, 57 { .compatible = "arm,cci-550", }, 58 #endif 59 {}, 60 }; 61 62 #ifdef CONFIG_ARM_CCI_PMU 63 64 #define DRIVER_NAME "ARM-CCI" 65 #define DRIVER_NAME_PMU DRIVER_NAME " PMU" 66 67 #define CCI_PMCR 0x0100 68 #define CCI_PID2 0x0fe8 69 70 #define CCI_PMCR_CEN 0x00000001 71 #define CCI_PMCR_NCNT_MASK 0x0000f800 72 #define CCI_PMCR_NCNT_SHIFT 11 73 74 #define CCI_PID2_REV_MASK 0xf0 75 #define CCI_PID2_REV_SHIFT 4 76 77 #define CCI_PMU_EVT_SEL 0x000 78 #define CCI_PMU_CNTR 0x004 79 #define CCI_PMU_CNTR_CTRL 0x008 80 #define CCI_PMU_OVRFLW 0x00c 81 82 #define CCI_PMU_OVRFLW_FLAG 1 83 84 #define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size) 85 #define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model)) 86 #define CCI_PMU_CNTR_MASK ((1ULL << 32) -1) 87 #define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1) 88 89 #define CCI_PMU_MAX_HW_CNTRS(model) \ 90 ((model)->num_hw_cntrs + (model)->fixed_hw_cntrs) 91 92 /* Types of interfaces that can generate events */ 93 enum { 94 CCI_IF_SLAVE, 95 CCI_IF_MASTER, 96 #ifdef CONFIG_ARM_CCI5xx_PMU 97 CCI_IF_GLOBAL, 98 #endif 99 CCI_IF_MAX, 100 }; 101 102 struct event_range { 103 u32 min; 104 u32 max; 105 }; 106 107 struct cci_pmu_hw_events { 108 struct perf_event **events; 109 unsigned long *used_mask; 110 raw_spinlock_t pmu_lock; 111 }; 112 113 struct cci_pmu; 114 /* 115 * struct cci_pmu_model: 116 * @fixed_hw_cntrs - Number of fixed event counters 117 * @num_hw_cntrs - Maximum number of programmable event counters 118 * @cntr_size - Size of an event counter mapping 119 */ 120 struct cci_pmu_model { 121 char *name; 122 u32 fixed_hw_cntrs; 123 u32 num_hw_cntrs; 124 u32 cntr_size; 125 struct attribute **format_attrs; 126 struct attribute **event_attrs; 127 struct event_range event_ranges[CCI_IF_MAX]; 128 int (*validate_hw_event)(struct cci_pmu *, unsigned long); 129 int (*get_event_idx)(struct cci_pmu *, struct cci_pmu_hw_events *, unsigned long); 130 void (*write_counters)(struct cci_pmu *, unsigned long *); 131 }; 132 133 static struct cci_pmu_model cci_pmu_models[]; 134 135 struct cci_pmu { 136 void __iomem *base; 137 struct pmu pmu; 138 int nr_irqs; 139 int *irqs; 140 unsigned long active_irqs; 141 const struct cci_pmu_model *model; 142 struct cci_pmu_hw_events hw_events; 143 struct platform_device *plat_device; 144 int num_cntrs; 145 atomic_t active_events; 146 struct mutex reserve_mutex; 147 struct hlist_node node; 148 cpumask_t cpus; 149 }; 150 151 #define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu)) 152 153 enum cci_models { 154 #ifdef CONFIG_ARM_CCI400_PMU 155 CCI400_R0, 156 CCI400_R1, 157 #endif 158 #ifdef CONFIG_ARM_CCI5xx_PMU 159 CCI500_R0, 160 CCI550_R0, 161 #endif 162 CCI_MODEL_MAX 163 }; 164 165 static void pmu_write_counters(struct cci_pmu *cci_pmu, 166 unsigned long *mask); 167 static ssize_t cci_pmu_format_show(struct device *dev, 168 struct device_attribute *attr, char *buf); 169 static ssize_t cci_pmu_event_show(struct device *dev, 170 struct device_attribute *attr, char *buf); 171 172 #define CCI_EXT_ATTR_ENTRY(_name, _func, _config) \ 173 &((struct dev_ext_attribute[]) { \ 174 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \ 175 })[0].attr.attr 176 177 #define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \ 178 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_format_show, (char *)_config) 179 #define CCI_EVENT_EXT_ATTR_ENTRY(_name, _config) \ 180 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_event_show, (unsigned long)_config) 181 182 /* CCI400 PMU Specific definitions */ 183 184 #ifdef CONFIG_ARM_CCI400_PMU 185 186 /* Port ids */ 187 #define CCI400_PORT_S0 0 188 #define CCI400_PORT_S1 1 189 #define CCI400_PORT_S2 2 190 #define CCI400_PORT_S3 3 191 #define CCI400_PORT_S4 4 192 #define CCI400_PORT_M0 5 193 #define CCI400_PORT_M1 6 194 #define CCI400_PORT_M2 7 195 196 #define CCI400_R1_PX 5 197 198 /* 199 * Instead of an event id to monitor CCI cycles, a dedicated counter is 200 * provided. Use 0xff to represent CCI cycles and hope that no future revisions 201 * make use of this event in hardware. 202 */ 203 enum cci400_perf_events { 204 CCI400_PMU_CYCLES = 0xff 205 }; 206 207 #define CCI400_PMU_CYCLE_CNTR_IDX 0 208 #define CCI400_PMU_CNTR0_IDX 1 209 210 /* 211 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8 212 * ports and bits 4:0 are event codes. There are different event codes 213 * associated with each port type. 214 * 215 * Additionally, the range of events associated with the port types changed 216 * between Rev0 and Rev1. 217 * 218 * The constants below define the range of valid codes for each port type for 219 * the different revisions and are used to validate the event to be monitored. 220 */ 221 222 #define CCI400_PMU_EVENT_MASK 0xffUL 223 #define CCI400_PMU_EVENT_SOURCE_SHIFT 5 224 #define CCI400_PMU_EVENT_SOURCE_MASK 0x7 225 #define CCI400_PMU_EVENT_CODE_SHIFT 0 226 #define CCI400_PMU_EVENT_CODE_MASK 0x1f 227 #define CCI400_PMU_EVENT_SOURCE(event) \ 228 ((event >> CCI400_PMU_EVENT_SOURCE_SHIFT) & \ 229 CCI400_PMU_EVENT_SOURCE_MASK) 230 #define CCI400_PMU_EVENT_CODE(event) \ 231 ((event >> CCI400_PMU_EVENT_CODE_SHIFT) & CCI400_PMU_EVENT_CODE_MASK) 232 233 #define CCI400_R0_SLAVE_PORT_MIN_EV 0x00 234 #define CCI400_R0_SLAVE_PORT_MAX_EV 0x13 235 #define CCI400_R0_MASTER_PORT_MIN_EV 0x14 236 #define CCI400_R0_MASTER_PORT_MAX_EV 0x1a 237 238 #define CCI400_R1_SLAVE_PORT_MIN_EV 0x00 239 #define CCI400_R1_SLAVE_PORT_MAX_EV 0x14 240 #define CCI400_R1_MASTER_PORT_MIN_EV 0x00 241 #define CCI400_R1_MASTER_PORT_MAX_EV 0x11 242 243 #define CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(_name, _config) \ 244 CCI_EXT_ATTR_ENTRY(_name, cci400_pmu_cycle_event_show, \ 245 (unsigned long)_config) 246 247 static ssize_t cci400_pmu_cycle_event_show(struct device *dev, 248 struct device_attribute *attr, char *buf); 249 250 static struct attribute *cci400_pmu_format_attrs[] = { 251 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"), 252 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-7"), 253 NULL 254 }; 255 256 static struct attribute *cci400_r0_pmu_event_attrs[] = { 257 /* Slave events */ 258 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0), 259 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01), 260 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2), 261 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3), 262 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4), 263 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5), 264 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6), 265 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7), 266 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8), 267 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9), 268 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA), 269 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB), 270 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC), 271 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD), 272 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE), 273 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF), 274 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10), 275 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11), 276 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12), 277 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13), 278 /* Master events */ 279 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x14), 280 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_addr_hazard, 0x15), 281 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_id_hazard, 0x16), 282 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_tt_full, 0x17), 283 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x18), 284 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x19), 285 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_tt_full, 0x1A), 286 /* Special event for cycles counter */ 287 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff), 288 NULL 289 }; 290 291 static struct attribute *cci400_r1_pmu_event_attrs[] = { 292 /* Slave events */ 293 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0), 294 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01), 295 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2), 296 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3), 297 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4), 298 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5), 299 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6), 300 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7), 301 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8), 302 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9), 303 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA), 304 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB), 305 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC), 306 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD), 307 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE), 308 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF), 309 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10), 310 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11), 311 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12), 312 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13), 313 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_slave_id_hazard, 0x14), 314 /* Master events */ 315 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x0), 316 CCI_EVENT_EXT_ATTR_ENTRY(mi_stall_cycle_addr_hazard, 0x1), 317 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_master_id_hazard, 0x2), 318 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_hi_prio_rtq_full, 0x3), 319 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x4), 320 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x5), 321 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_wtq_full, 0x6), 322 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_low_prio_rtq_full, 0x7), 323 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_mid_prio_rtq_full, 0x8), 324 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn0, 0x9), 325 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn1, 0xA), 326 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn2, 0xB), 327 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn3, 0xC), 328 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn0, 0xD), 329 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn1, 0xE), 330 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn2, 0xF), 331 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn3, 0x10), 332 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_unique_or_line_unique_addr_hazard, 0x11), 333 /* Special event for cycles counter */ 334 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff), 335 NULL 336 }; 337 338 static ssize_t cci400_pmu_cycle_event_show(struct device *dev, 339 struct device_attribute *attr, char *buf) 340 { 341 struct dev_ext_attribute *eattr = container_of(attr, 342 struct dev_ext_attribute, attr); 343 return snprintf(buf, PAGE_SIZE, "config=0x%lx\n", (unsigned long)eattr->var); 344 } 345 346 static int cci400_get_event_idx(struct cci_pmu *cci_pmu, 347 struct cci_pmu_hw_events *hw, 348 unsigned long cci_event) 349 { 350 int idx; 351 352 /* cycles event idx is fixed */ 353 if (cci_event == CCI400_PMU_CYCLES) { 354 if (test_and_set_bit(CCI400_PMU_CYCLE_CNTR_IDX, hw->used_mask)) 355 return -EAGAIN; 356 357 return CCI400_PMU_CYCLE_CNTR_IDX; 358 } 359 360 for (idx = CCI400_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx) 361 if (!test_and_set_bit(idx, hw->used_mask)) 362 return idx; 363 364 /* No counters available */ 365 return -EAGAIN; 366 } 367 368 static int cci400_validate_hw_event(struct cci_pmu *cci_pmu, unsigned long hw_event) 369 { 370 u8 ev_source = CCI400_PMU_EVENT_SOURCE(hw_event); 371 u8 ev_code = CCI400_PMU_EVENT_CODE(hw_event); 372 int if_type; 373 374 if (hw_event & ~CCI400_PMU_EVENT_MASK) 375 return -ENOENT; 376 377 if (hw_event == CCI400_PMU_CYCLES) 378 return hw_event; 379 380 switch (ev_source) { 381 case CCI400_PORT_S0: 382 case CCI400_PORT_S1: 383 case CCI400_PORT_S2: 384 case CCI400_PORT_S3: 385 case CCI400_PORT_S4: 386 /* Slave Interface */ 387 if_type = CCI_IF_SLAVE; 388 break; 389 case CCI400_PORT_M0: 390 case CCI400_PORT_M1: 391 case CCI400_PORT_M2: 392 /* Master Interface */ 393 if_type = CCI_IF_MASTER; 394 break; 395 default: 396 return -ENOENT; 397 } 398 399 if (ev_code >= cci_pmu->model->event_ranges[if_type].min && 400 ev_code <= cci_pmu->model->event_ranges[if_type].max) 401 return hw_event; 402 403 return -ENOENT; 404 } 405 406 static int probe_cci400_revision(void) 407 { 408 int rev; 409 rev = readl_relaxed(cci_ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK; 410 rev >>= CCI_PID2_REV_SHIFT; 411 412 if (rev < CCI400_R1_PX) 413 return CCI400_R0; 414 else 415 return CCI400_R1; 416 } 417 418 static const struct cci_pmu_model *probe_cci_model(struct platform_device *pdev) 419 { 420 if (platform_has_secure_cci_access()) 421 return &cci_pmu_models[probe_cci400_revision()]; 422 return NULL; 423 } 424 #else /* !CONFIG_ARM_CCI400_PMU */ 425 static inline struct cci_pmu_model *probe_cci_model(struct platform_device *pdev) 426 { 427 return NULL; 428 } 429 #endif /* CONFIG_ARM_CCI400_PMU */ 430 431 #ifdef CONFIG_ARM_CCI5xx_PMU 432 433 /* 434 * CCI5xx PMU event id is an 9-bit value made of two parts. 435 * bits [8:5] - Source for the event 436 * bits [4:0] - Event code (specific to type of interface) 437 * 438 * 439 */ 440 441 /* Port ids */ 442 #define CCI5xx_PORT_S0 0x0 443 #define CCI5xx_PORT_S1 0x1 444 #define CCI5xx_PORT_S2 0x2 445 #define CCI5xx_PORT_S3 0x3 446 #define CCI5xx_PORT_S4 0x4 447 #define CCI5xx_PORT_S5 0x5 448 #define CCI5xx_PORT_S6 0x6 449 450 #define CCI5xx_PORT_M0 0x8 451 #define CCI5xx_PORT_M1 0x9 452 #define CCI5xx_PORT_M2 0xa 453 #define CCI5xx_PORT_M3 0xb 454 #define CCI5xx_PORT_M4 0xc 455 #define CCI5xx_PORT_M5 0xd 456 #define CCI5xx_PORT_M6 0xe 457 458 #define CCI5xx_PORT_GLOBAL 0xf 459 460 #define CCI5xx_PMU_EVENT_MASK 0x1ffUL 461 #define CCI5xx_PMU_EVENT_SOURCE_SHIFT 0x5 462 #define CCI5xx_PMU_EVENT_SOURCE_MASK 0xf 463 #define CCI5xx_PMU_EVENT_CODE_SHIFT 0x0 464 #define CCI5xx_PMU_EVENT_CODE_MASK 0x1f 465 466 #define CCI5xx_PMU_EVENT_SOURCE(event) \ 467 ((event >> CCI5xx_PMU_EVENT_SOURCE_SHIFT) & CCI5xx_PMU_EVENT_SOURCE_MASK) 468 #define CCI5xx_PMU_EVENT_CODE(event) \ 469 ((event >> CCI5xx_PMU_EVENT_CODE_SHIFT) & CCI5xx_PMU_EVENT_CODE_MASK) 470 471 #define CCI5xx_SLAVE_PORT_MIN_EV 0x00 472 #define CCI5xx_SLAVE_PORT_MAX_EV 0x1f 473 #define CCI5xx_MASTER_PORT_MIN_EV 0x00 474 #define CCI5xx_MASTER_PORT_MAX_EV 0x06 475 #define CCI5xx_GLOBAL_PORT_MIN_EV 0x00 476 #define CCI5xx_GLOBAL_PORT_MAX_EV 0x0f 477 478 479 #define CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(_name, _config) \ 480 CCI_EXT_ATTR_ENTRY(_name, cci5xx_pmu_global_event_show, \ 481 (unsigned long) _config) 482 483 static ssize_t cci5xx_pmu_global_event_show(struct device *dev, 484 struct device_attribute *attr, char *buf); 485 486 static struct attribute *cci5xx_pmu_format_attrs[] = { 487 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"), 488 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-8"), 489 NULL, 490 }; 491 492 static struct attribute *cci5xx_pmu_event_attrs[] = { 493 /* Slave events */ 494 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_arvalid, 0x0), 495 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_dev, 0x1), 496 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_nonshareable, 0x2), 497 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_non_alloc, 0x3), 498 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_alloc, 0x4), 499 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_invalidate, 0x5), 500 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maint, 0x6), 501 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7), 502 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rval, 0x8), 503 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rlast_snoop, 0x9), 504 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_awalid, 0xA), 505 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_dev, 0xB), 506 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_non_shareable, 0xC), 507 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wb, 0xD), 508 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wlu, 0xE), 509 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wunique, 0xF), 510 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_evict, 0x10), 511 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_wrevict, 0x11), 512 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_beat, 0x12), 513 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_acvalid, 0x13), 514 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_read, 0x14), 515 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_clean, 0x15), 516 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_data_transfer_low, 0x16), 517 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_arvalid, 0x17), 518 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall, 0x18), 519 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall, 0x19), 520 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_stall, 0x1A), 521 CCI_EVENT_EXT_ATTR_ENTRY(si_w_resp_stall, 0x1B), 522 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_stall, 0x1C), 523 CCI_EVENT_EXT_ATTR_ENTRY(si_s_data_stall, 0x1D), 524 CCI_EVENT_EXT_ATTR_ENTRY(si_rq_stall_ot_limit, 0x1E), 525 CCI_EVENT_EXT_ATTR_ENTRY(si_r_stall_arbit, 0x1F), 526 527 /* Master events */ 528 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_beat_any, 0x0), 529 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_beat_any, 0x1), 530 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall, 0x2), 531 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_stall, 0x3), 532 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall, 0x4), 533 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_stall, 0x5), 534 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_resp_stall, 0x6), 535 536 /* Global events */ 537 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_0_1, 0x0), 538 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_2_3, 0x1), 539 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_4_5, 0x2), 540 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_6_7, 0x3), 541 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_0_1, 0x4), 542 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_2_3, 0x5), 543 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_4_5, 0x6), 544 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_6_7, 0x7), 545 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_back_invalidation, 0x8), 546 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_alloc_busy, 0x9), 547 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_tt_full, 0xA), 548 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB), 549 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC), 550 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD), 551 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_stall_tt_full, 0xE), 552 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF), 553 NULL 554 }; 555 556 static ssize_t cci5xx_pmu_global_event_show(struct device *dev, 557 struct device_attribute *attr, char *buf) 558 { 559 struct dev_ext_attribute *eattr = container_of(attr, 560 struct dev_ext_attribute, attr); 561 /* Global events have single fixed source code */ 562 return snprintf(buf, PAGE_SIZE, "event=0x%lx,source=0x%x\n", 563 (unsigned long)eattr->var, CCI5xx_PORT_GLOBAL); 564 } 565 566 /* 567 * CCI500 provides 8 independent event counters that can count 568 * any of the events available. 569 * CCI500 PMU event source ids 570 * 0x0-0x6 - Slave interfaces 571 * 0x8-0xD - Master interfaces 572 * 0xf - Global Events 573 * 0x7,0xe - Reserved 574 */ 575 static int cci500_validate_hw_event(struct cci_pmu *cci_pmu, 576 unsigned long hw_event) 577 { 578 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event); 579 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event); 580 int if_type; 581 582 if (hw_event & ~CCI5xx_PMU_EVENT_MASK) 583 return -ENOENT; 584 585 switch (ev_source) { 586 case CCI5xx_PORT_S0: 587 case CCI5xx_PORT_S1: 588 case CCI5xx_PORT_S2: 589 case CCI5xx_PORT_S3: 590 case CCI5xx_PORT_S4: 591 case CCI5xx_PORT_S5: 592 case CCI5xx_PORT_S6: 593 if_type = CCI_IF_SLAVE; 594 break; 595 case CCI5xx_PORT_M0: 596 case CCI5xx_PORT_M1: 597 case CCI5xx_PORT_M2: 598 case CCI5xx_PORT_M3: 599 case CCI5xx_PORT_M4: 600 case CCI5xx_PORT_M5: 601 if_type = CCI_IF_MASTER; 602 break; 603 case CCI5xx_PORT_GLOBAL: 604 if_type = CCI_IF_GLOBAL; 605 break; 606 default: 607 return -ENOENT; 608 } 609 610 if (ev_code >= cci_pmu->model->event_ranges[if_type].min && 611 ev_code <= cci_pmu->model->event_ranges[if_type].max) 612 return hw_event; 613 614 return -ENOENT; 615 } 616 617 /* 618 * CCI550 provides 8 independent event counters that can count 619 * any of the events available. 620 * CCI550 PMU event source ids 621 * 0x0-0x6 - Slave interfaces 622 * 0x8-0xe - Master interfaces 623 * 0xf - Global Events 624 * 0x7 - Reserved 625 */ 626 static int cci550_validate_hw_event(struct cci_pmu *cci_pmu, 627 unsigned long hw_event) 628 { 629 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event); 630 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event); 631 int if_type; 632 633 if (hw_event & ~CCI5xx_PMU_EVENT_MASK) 634 return -ENOENT; 635 636 switch (ev_source) { 637 case CCI5xx_PORT_S0: 638 case CCI5xx_PORT_S1: 639 case CCI5xx_PORT_S2: 640 case CCI5xx_PORT_S3: 641 case CCI5xx_PORT_S4: 642 case CCI5xx_PORT_S5: 643 case CCI5xx_PORT_S6: 644 if_type = CCI_IF_SLAVE; 645 break; 646 case CCI5xx_PORT_M0: 647 case CCI5xx_PORT_M1: 648 case CCI5xx_PORT_M2: 649 case CCI5xx_PORT_M3: 650 case CCI5xx_PORT_M4: 651 case CCI5xx_PORT_M5: 652 case CCI5xx_PORT_M6: 653 if_type = CCI_IF_MASTER; 654 break; 655 case CCI5xx_PORT_GLOBAL: 656 if_type = CCI_IF_GLOBAL; 657 break; 658 default: 659 return -ENOENT; 660 } 661 662 if (ev_code >= cci_pmu->model->event_ranges[if_type].min && 663 ev_code <= cci_pmu->model->event_ranges[if_type].max) 664 return hw_event; 665 666 return -ENOENT; 667 } 668 669 #endif /* CONFIG_ARM_CCI5xx_PMU */ 670 671 /* 672 * Program the CCI PMU counters which have PERF_HES_ARCH set 673 * with the event period and mark them ready before we enable 674 * PMU. 675 */ 676 static void cci_pmu_sync_counters(struct cci_pmu *cci_pmu) 677 { 678 int i; 679 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events; 680 681 DECLARE_BITMAP(mask, cci_pmu->num_cntrs); 682 683 bitmap_zero(mask, cci_pmu->num_cntrs); 684 for_each_set_bit(i, cci_pmu->hw_events.used_mask, cci_pmu->num_cntrs) { 685 struct perf_event *event = cci_hw->events[i]; 686 687 if (WARN_ON(!event)) 688 continue; 689 690 /* Leave the events which are not counting */ 691 if (event->hw.state & PERF_HES_STOPPED) 692 continue; 693 if (event->hw.state & PERF_HES_ARCH) { 694 set_bit(i, mask); 695 event->hw.state &= ~PERF_HES_ARCH; 696 } 697 } 698 699 pmu_write_counters(cci_pmu, mask); 700 } 701 702 /* Should be called with cci_pmu->hw_events->pmu_lock held */ 703 static void __cci_pmu_enable_nosync(struct cci_pmu *cci_pmu) 704 { 705 u32 val; 706 707 /* Enable all the PMU counters. */ 708 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) | CCI_PMCR_CEN; 709 writel(val, cci_ctrl_base + CCI_PMCR); 710 } 711 712 /* Should be called with cci_pmu->hw_events->pmu_lock held */ 713 static void __cci_pmu_enable_sync(struct cci_pmu *cci_pmu) 714 { 715 cci_pmu_sync_counters(cci_pmu); 716 __cci_pmu_enable_nosync(cci_pmu); 717 } 718 719 /* Should be called with cci_pmu->hw_events->pmu_lock held */ 720 static void __cci_pmu_disable(void) 721 { 722 u32 val; 723 724 /* Disable all the PMU counters. */ 725 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN; 726 writel(val, cci_ctrl_base + CCI_PMCR); 727 } 728 729 static ssize_t cci_pmu_format_show(struct device *dev, 730 struct device_attribute *attr, char *buf) 731 { 732 struct dev_ext_attribute *eattr = container_of(attr, 733 struct dev_ext_attribute, attr); 734 return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var); 735 } 736 737 static ssize_t cci_pmu_event_show(struct device *dev, 738 struct device_attribute *attr, char *buf) 739 { 740 struct dev_ext_attribute *eattr = container_of(attr, 741 struct dev_ext_attribute, attr); 742 /* source parameter is mandatory for normal PMU events */ 743 return snprintf(buf, PAGE_SIZE, "source=?,event=0x%lx\n", 744 (unsigned long)eattr->var); 745 } 746 747 static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx) 748 { 749 return 0 <= idx && idx <= CCI_PMU_CNTR_LAST(cci_pmu); 750 } 751 752 static u32 pmu_read_register(struct cci_pmu *cci_pmu, int idx, unsigned int offset) 753 { 754 return readl_relaxed(cci_pmu->base + 755 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset); 756 } 757 758 static void pmu_write_register(struct cci_pmu *cci_pmu, u32 value, 759 int idx, unsigned int offset) 760 { 761 writel_relaxed(value, cci_pmu->base + 762 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset); 763 } 764 765 static void pmu_disable_counter(struct cci_pmu *cci_pmu, int idx) 766 { 767 pmu_write_register(cci_pmu, 0, idx, CCI_PMU_CNTR_CTRL); 768 } 769 770 static void pmu_enable_counter(struct cci_pmu *cci_pmu, int idx) 771 { 772 pmu_write_register(cci_pmu, 1, idx, CCI_PMU_CNTR_CTRL); 773 } 774 775 static bool __maybe_unused 776 pmu_counter_is_enabled(struct cci_pmu *cci_pmu, int idx) 777 { 778 return (pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR_CTRL) & 0x1) != 0; 779 } 780 781 static void pmu_set_event(struct cci_pmu *cci_pmu, int idx, unsigned long event) 782 { 783 pmu_write_register(cci_pmu, event, idx, CCI_PMU_EVT_SEL); 784 } 785 786 /* 787 * For all counters on the CCI-PMU, disable any 'enabled' counters, 788 * saving the changed counters in the mask, so that we can restore 789 * it later using pmu_restore_counters. The mask is private to the 790 * caller. We cannot rely on the used_mask maintained by the CCI_PMU 791 * as it only tells us if the counter is assigned to perf_event or not. 792 * The state of the perf_event cannot be locked by the PMU layer, hence 793 * we check the individual counter status (which can be locked by 794 * cci_pm->hw_events->pmu_lock). 795 * 796 * @mask should be initialised to empty by the caller. 797 */ 798 static void __maybe_unused 799 pmu_save_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 800 { 801 int i; 802 803 for (i = 0; i < cci_pmu->num_cntrs; i++) { 804 if (pmu_counter_is_enabled(cci_pmu, i)) { 805 set_bit(i, mask); 806 pmu_disable_counter(cci_pmu, i); 807 } 808 } 809 } 810 811 /* 812 * Restore the status of the counters. Reversal of the pmu_save_counters(). 813 * For each counter set in the mask, enable the counter back. 814 */ 815 static void __maybe_unused 816 pmu_restore_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 817 { 818 int i; 819 820 for_each_set_bit(i, mask, cci_pmu->num_cntrs) 821 pmu_enable_counter(cci_pmu, i); 822 } 823 824 /* 825 * Returns the number of programmable counters actually implemented 826 * by the cci 827 */ 828 static u32 pmu_get_max_counters(void) 829 { 830 return (readl_relaxed(cci_ctrl_base + CCI_PMCR) & 831 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT; 832 } 833 834 static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *event) 835 { 836 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 837 unsigned long cci_event = event->hw.config_base; 838 int idx; 839 840 if (cci_pmu->model->get_event_idx) 841 return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event); 842 843 /* Generic code to find an unused idx from the mask */ 844 for(idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) 845 if (!test_and_set_bit(idx, hw->used_mask)) 846 return idx; 847 848 /* No counters available */ 849 return -EAGAIN; 850 } 851 852 static int pmu_map_event(struct perf_event *event) 853 { 854 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 855 856 if (event->attr.type < PERF_TYPE_MAX || 857 !cci_pmu->model->validate_hw_event) 858 return -ENOENT; 859 860 return cci_pmu->model->validate_hw_event(cci_pmu, event->attr.config); 861 } 862 863 static int pmu_request_irq(struct cci_pmu *cci_pmu, irq_handler_t handler) 864 { 865 int i; 866 struct platform_device *pmu_device = cci_pmu->plat_device; 867 868 if (unlikely(!pmu_device)) 869 return -ENODEV; 870 871 if (cci_pmu->nr_irqs < 1) { 872 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n"); 873 return -ENODEV; 874 } 875 876 /* 877 * Register all available CCI PMU interrupts. In the interrupt handler 878 * we iterate over the counters checking for interrupt source (the 879 * overflowing counter) and clear it. 880 * 881 * This should allow handling of non-unique interrupt for the counters. 882 */ 883 for (i = 0; i < cci_pmu->nr_irqs; i++) { 884 int err = request_irq(cci_pmu->irqs[i], handler, IRQF_SHARED, 885 "arm-cci-pmu", cci_pmu); 886 if (err) { 887 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n", 888 cci_pmu->irqs[i]); 889 return err; 890 } 891 892 set_bit(i, &cci_pmu->active_irqs); 893 } 894 895 return 0; 896 } 897 898 static void pmu_free_irq(struct cci_pmu *cci_pmu) 899 { 900 int i; 901 902 for (i = 0; i < cci_pmu->nr_irqs; i++) { 903 if (!test_and_clear_bit(i, &cci_pmu->active_irqs)) 904 continue; 905 906 free_irq(cci_pmu->irqs[i], cci_pmu); 907 } 908 } 909 910 static u32 pmu_read_counter(struct perf_event *event) 911 { 912 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 913 struct hw_perf_event *hw_counter = &event->hw; 914 int idx = hw_counter->idx; 915 u32 value; 916 917 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 918 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 919 return 0; 920 } 921 value = pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR); 922 923 return value; 924 } 925 926 static void pmu_write_counter(struct cci_pmu *cci_pmu, u32 value, int idx) 927 { 928 pmu_write_register(cci_pmu, value, idx, CCI_PMU_CNTR); 929 } 930 931 static void __pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 932 { 933 int i; 934 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events; 935 936 for_each_set_bit(i, mask, cci_pmu->num_cntrs) { 937 struct perf_event *event = cci_hw->events[i]; 938 939 if (WARN_ON(!event)) 940 continue; 941 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i); 942 } 943 } 944 945 static void pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 946 { 947 if (cci_pmu->model->write_counters) 948 cci_pmu->model->write_counters(cci_pmu, mask); 949 else 950 __pmu_write_counters(cci_pmu, mask); 951 } 952 953 #ifdef CONFIG_ARM_CCI5xx_PMU 954 955 /* 956 * CCI-500/CCI-550 has advanced power saving policies, which could gate the 957 * clocks to the PMU counters, which makes the writes to them ineffective. 958 * The only way to write to those counters is when the global counters 959 * are enabled and the particular counter is enabled. 960 * 961 * So we do the following : 962 * 963 * 1) Disable all the PMU counters, saving their current state 964 * 2) Enable the global PMU profiling, now that all counters are 965 * disabled. 966 * 967 * For each counter to be programmed, repeat steps 3-7: 968 * 969 * 3) Write an invalid event code to the event control register for the 970 counter, so that the counters are not modified. 971 * 4) Enable the counter control for the counter. 972 * 5) Set the counter value 973 * 6) Disable the counter 974 * 7) Restore the event in the target counter 975 * 976 * 8) Disable the global PMU. 977 * 9) Restore the status of the rest of the counters. 978 * 979 * We choose an event which for CCI-5xx is guaranteed not to count. 980 * We use the highest possible event code (0x1f) for the master interface 0. 981 */ 982 #define CCI5xx_INVALID_EVENT ((CCI5xx_PORT_M0 << CCI5xx_PMU_EVENT_SOURCE_SHIFT) | \ 983 (CCI5xx_PMU_EVENT_CODE_MASK << CCI5xx_PMU_EVENT_CODE_SHIFT)) 984 static void cci5xx_pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask) 985 { 986 int i; 987 DECLARE_BITMAP(saved_mask, cci_pmu->num_cntrs); 988 989 bitmap_zero(saved_mask, cci_pmu->num_cntrs); 990 pmu_save_counters(cci_pmu, saved_mask); 991 992 /* 993 * Now that all the counters are disabled, we can safely turn the PMU on, 994 * without syncing the status of the counters 995 */ 996 __cci_pmu_enable_nosync(cci_pmu); 997 998 for_each_set_bit(i, mask, cci_pmu->num_cntrs) { 999 struct perf_event *event = cci_pmu->hw_events.events[i]; 1000 1001 if (WARN_ON(!event)) 1002 continue; 1003 1004 pmu_set_event(cci_pmu, i, CCI5xx_INVALID_EVENT); 1005 pmu_enable_counter(cci_pmu, i); 1006 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i); 1007 pmu_disable_counter(cci_pmu, i); 1008 pmu_set_event(cci_pmu, i, event->hw.config_base); 1009 } 1010 1011 __cci_pmu_disable(); 1012 1013 pmu_restore_counters(cci_pmu, saved_mask); 1014 } 1015 1016 #endif /* CONFIG_ARM_CCI5xx_PMU */ 1017 1018 static u64 pmu_event_update(struct perf_event *event) 1019 { 1020 struct hw_perf_event *hwc = &event->hw; 1021 u64 delta, prev_raw_count, new_raw_count; 1022 1023 do { 1024 prev_raw_count = local64_read(&hwc->prev_count); 1025 new_raw_count = pmu_read_counter(event); 1026 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 1027 new_raw_count) != prev_raw_count); 1028 1029 delta = (new_raw_count - prev_raw_count) & CCI_PMU_CNTR_MASK; 1030 1031 local64_add(delta, &event->count); 1032 1033 return new_raw_count; 1034 } 1035 1036 static void pmu_read(struct perf_event *event) 1037 { 1038 pmu_event_update(event); 1039 } 1040 1041 static void pmu_event_set_period(struct perf_event *event) 1042 { 1043 struct hw_perf_event *hwc = &event->hw; 1044 /* 1045 * The CCI PMU counters have a period of 2^32. To account for the 1046 * possiblity of extreme interrupt latency we program for a period of 1047 * half that. Hopefully we can handle the interrupt before another 2^31 1048 * events occur and the counter overtakes its previous value. 1049 */ 1050 u64 val = 1ULL << 31; 1051 local64_set(&hwc->prev_count, val); 1052 1053 /* 1054 * CCI PMU uses PERF_HES_ARCH to keep track of the counters, whose 1055 * values needs to be sync-ed with the s/w state before the PMU is 1056 * enabled. 1057 * Mark this counter for sync. 1058 */ 1059 hwc->state |= PERF_HES_ARCH; 1060 } 1061 1062 static irqreturn_t pmu_handle_irq(int irq_num, void *dev) 1063 { 1064 unsigned long flags; 1065 struct cci_pmu *cci_pmu = dev; 1066 struct cci_pmu_hw_events *events = &cci_pmu->hw_events; 1067 int idx, handled = IRQ_NONE; 1068 1069 raw_spin_lock_irqsave(&events->pmu_lock, flags); 1070 1071 /* Disable the PMU while we walk through the counters */ 1072 __cci_pmu_disable(); 1073 /* 1074 * Iterate over counters and update the corresponding perf events. 1075 * This should work regardless of whether we have per-counter overflow 1076 * interrupt or a combined overflow interrupt. 1077 */ 1078 for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) { 1079 struct perf_event *event = events->events[idx]; 1080 1081 if (!event) 1082 continue; 1083 1084 /* Did this counter overflow? */ 1085 if (!(pmu_read_register(cci_pmu, idx, CCI_PMU_OVRFLW) & 1086 CCI_PMU_OVRFLW_FLAG)) 1087 continue; 1088 1089 pmu_write_register(cci_pmu, CCI_PMU_OVRFLW_FLAG, idx, 1090 CCI_PMU_OVRFLW); 1091 1092 pmu_event_update(event); 1093 pmu_event_set_period(event); 1094 handled = IRQ_HANDLED; 1095 } 1096 1097 /* Enable the PMU and sync possibly overflowed counters */ 1098 __cci_pmu_enable_sync(cci_pmu); 1099 raw_spin_unlock_irqrestore(&events->pmu_lock, flags); 1100 1101 return IRQ_RETVAL(handled); 1102 } 1103 1104 static int cci_pmu_get_hw(struct cci_pmu *cci_pmu) 1105 { 1106 int ret = pmu_request_irq(cci_pmu, pmu_handle_irq); 1107 if (ret) { 1108 pmu_free_irq(cci_pmu); 1109 return ret; 1110 } 1111 return 0; 1112 } 1113 1114 static void cci_pmu_put_hw(struct cci_pmu *cci_pmu) 1115 { 1116 pmu_free_irq(cci_pmu); 1117 } 1118 1119 static void hw_perf_event_destroy(struct perf_event *event) 1120 { 1121 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1122 atomic_t *active_events = &cci_pmu->active_events; 1123 struct mutex *reserve_mutex = &cci_pmu->reserve_mutex; 1124 1125 if (atomic_dec_and_mutex_lock(active_events, reserve_mutex)) { 1126 cci_pmu_put_hw(cci_pmu); 1127 mutex_unlock(reserve_mutex); 1128 } 1129 } 1130 1131 static void cci_pmu_enable(struct pmu *pmu) 1132 { 1133 struct cci_pmu *cci_pmu = to_cci_pmu(pmu); 1134 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1135 int enabled = bitmap_weight(hw_events->used_mask, cci_pmu->num_cntrs); 1136 unsigned long flags; 1137 1138 if (!enabled) 1139 return; 1140 1141 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags); 1142 __cci_pmu_enable_sync(cci_pmu); 1143 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags); 1144 1145 } 1146 1147 static void cci_pmu_disable(struct pmu *pmu) 1148 { 1149 struct cci_pmu *cci_pmu = to_cci_pmu(pmu); 1150 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1151 unsigned long flags; 1152 1153 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags); 1154 __cci_pmu_disable(); 1155 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags); 1156 } 1157 1158 /* 1159 * Check if the idx represents a non-programmable counter. 1160 * All the fixed event counters are mapped before the programmable 1161 * counters. 1162 */ 1163 static bool pmu_fixed_hw_idx(struct cci_pmu *cci_pmu, int idx) 1164 { 1165 return (idx >= 0) && (idx < cci_pmu->model->fixed_hw_cntrs); 1166 } 1167 1168 static void cci_pmu_start(struct perf_event *event, int pmu_flags) 1169 { 1170 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1171 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1172 struct hw_perf_event *hwc = &event->hw; 1173 int idx = hwc->idx; 1174 unsigned long flags; 1175 1176 /* 1177 * To handle interrupt latency, we always reprogram the period 1178 * regardlesss of PERF_EF_RELOAD. 1179 */ 1180 if (pmu_flags & PERF_EF_RELOAD) 1181 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 1182 1183 hwc->state = 0; 1184 1185 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 1186 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 1187 return; 1188 } 1189 1190 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags); 1191 1192 /* Configure the counter unless you are counting a fixed event */ 1193 if (!pmu_fixed_hw_idx(cci_pmu, idx)) 1194 pmu_set_event(cci_pmu, idx, hwc->config_base); 1195 1196 pmu_event_set_period(event); 1197 pmu_enable_counter(cci_pmu, idx); 1198 1199 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags); 1200 } 1201 1202 static void cci_pmu_stop(struct perf_event *event, int pmu_flags) 1203 { 1204 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1205 struct hw_perf_event *hwc = &event->hw; 1206 int idx = hwc->idx; 1207 1208 if (hwc->state & PERF_HES_STOPPED) 1209 return; 1210 1211 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) { 1212 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx); 1213 return; 1214 } 1215 1216 /* 1217 * We always reprogram the counter, so ignore PERF_EF_UPDATE. See 1218 * cci_pmu_start() 1219 */ 1220 pmu_disable_counter(cci_pmu, idx); 1221 pmu_event_update(event); 1222 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 1223 } 1224 1225 static int cci_pmu_add(struct perf_event *event, int flags) 1226 { 1227 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1228 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1229 struct hw_perf_event *hwc = &event->hw; 1230 int idx; 1231 int err = 0; 1232 1233 perf_pmu_disable(event->pmu); 1234 1235 /* If we don't have a space for the counter then finish early. */ 1236 idx = pmu_get_event_idx(hw_events, event); 1237 if (idx < 0) { 1238 err = idx; 1239 goto out; 1240 } 1241 1242 event->hw.idx = idx; 1243 hw_events->events[idx] = event; 1244 1245 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 1246 if (flags & PERF_EF_START) 1247 cci_pmu_start(event, PERF_EF_RELOAD); 1248 1249 /* Propagate our changes to the userspace mapping. */ 1250 perf_event_update_userpage(event); 1251 1252 out: 1253 perf_pmu_enable(event->pmu); 1254 return err; 1255 } 1256 1257 static void cci_pmu_del(struct perf_event *event, int flags) 1258 { 1259 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1260 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events; 1261 struct hw_perf_event *hwc = &event->hw; 1262 int idx = hwc->idx; 1263 1264 cci_pmu_stop(event, PERF_EF_UPDATE); 1265 hw_events->events[idx] = NULL; 1266 clear_bit(idx, hw_events->used_mask); 1267 1268 perf_event_update_userpage(event); 1269 } 1270 1271 static int 1272 validate_event(struct pmu *cci_pmu, 1273 struct cci_pmu_hw_events *hw_events, 1274 struct perf_event *event) 1275 { 1276 if (is_software_event(event)) 1277 return 1; 1278 1279 /* 1280 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The 1281 * core perf code won't check that the pmu->ctx == leader->ctx 1282 * until after pmu->event_init(event). 1283 */ 1284 if (event->pmu != cci_pmu) 1285 return 0; 1286 1287 if (event->state < PERF_EVENT_STATE_OFF) 1288 return 1; 1289 1290 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec) 1291 return 1; 1292 1293 return pmu_get_event_idx(hw_events, event) >= 0; 1294 } 1295 1296 static int 1297 validate_group(struct perf_event *event) 1298 { 1299 struct perf_event *sibling, *leader = event->group_leader; 1300 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1301 unsigned long mask[BITS_TO_LONGS(cci_pmu->num_cntrs)]; 1302 struct cci_pmu_hw_events fake_pmu = { 1303 /* 1304 * Initialise the fake PMU. We only need to populate the 1305 * used_mask for the purposes of validation. 1306 */ 1307 .used_mask = mask, 1308 }; 1309 memset(mask, 0, BITS_TO_LONGS(cci_pmu->num_cntrs) * sizeof(unsigned long)); 1310 1311 if (!validate_event(event->pmu, &fake_pmu, leader)) 1312 return -EINVAL; 1313 1314 list_for_each_entry(sibling, &leader->sibling_list, group_entry) { 1315 if (!validate_event(event->pmu, &fake_pmu, sibling)) 1316 return -EINVAL; 1317 } 1318 1319 if (!validate_event(event->pmu, &fake_pmu, event)) 1320 return -EINVAL; 1321 1322 return 0; 1323 } 1324 1325 static int 1326 __hw_perf_event_init(struct perf_event *event) 1327 { 1328 struct hw_perf_event *hwc = &event->hw; 1329 int mapping; 1330 1331 mapping = pmu_map_event(event); 1332 1333 if (mapping < 0) { 1334 pr_debug("event %x:%llx not supported\n", event->attr.type, 1335 event->attr.config); 1336 return mapping; 1337 } 1338 1339 /* 1340 * We don't assign an index until we actually place the event onto 1341 * hardware. Use -1 to signify that we haven't decided where to put it 1342 * yet. 1343 */ 1344 hwc->idx = -1; 1345 hwc->config_base = 0; 1346 hwc->config = 0; 1347 hwc->event_base = 0; 1348 1349 /* 1350 * Store the event encoding into the config_base field. 1351 */ 1352 hwc->config_base |= (unsigned long)mapping; 1353 1354 /* 1355 * Limit the sample_period to half of the counter width. That way, the 1356 * new counter value is far less likely to overtake the previous one 1357 * unless you have some serious IRQ latency issues. 1358 */ 1359 hwc->sample_period = CCI_PMU_CNTR_MASK >> 1; 1360 hwc->last_period = hwc->sample_period; 1361 local64_set(&hwc->period_left, hwc->sample_period); 1362 1363 if (event->group_leader != event) { 1364 if (validate_group(event) != 0) 1365 return -EINVAL; 1366 } 1367 1368 return 0; 1369 } 1370 1371 static int cci_pmu_event_init(struct perf_event *event) 1372 { 1373 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu); 1374 atomic_t *active_events = &cci_pmu->active_events; 1375 int err = 0; 1376 int cpu; 1377 1378 if (event->attr.type != event->pmu->type) 1379 return -ENOENT; 1380 1381 /* Shared by all CPUs, no meaningful state to sample */ 1382 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 1383 return -EOPNOTSUPP; 1384 1385 /* We have no filtering of any kind */ 1386 if (event->attr.exclude_user || 1387 event->attr.exclude_kernel || 1388 event->attr.exclude_hv || 1389 event->attr.exclude_idle || 1390 event->attr.exclude_host || 1391 event->attr.exclude_guest) 1392 return -EINVAL; 1393 1394 /* 1395 * Following the example set by other "uncore" PMUs, we accept any CPU 1396 * and rewrite its affinity dynamically rather than having perf core 1397 * handle cpu == -1 and pid == -1 for this case. 1398 * 1399 * The perf core will pin online CPUs for the duration of this call and 1400 * the event being installed into its context, so the PMU's CPU can't 1401 * change under our feet. 1402 */ 1403 cpu = cpumask_first(&cci_pmu->cpus); 1404 if (event->cpu < 0 || cpu < 0) 1405 return -EINVAL; 1406 event->cpu = cpu; 1407 1408 event->destroy = hw_perf_event_destroy; 1409 if (!atomic_inc_not_zero(active_events)) { 1410 mutex_lock(&cci_pmu->reserve_mutex); 1411 if (atomic_read(active_events) == 0) 1412 err = cci_pmu_get_hw(cci_pmu); 1413 if (!err) 1414 atomic_inc(active_events); 1415 mutex_unlock(&cci_pmu->reserve_mutex); 1416 } 1417 if (err) 1418 return err; 1419 1420 err = __hw_perf_event_init(event); 1421 if (err) 1422 hw_perf_event_destroy(event); 1423 1424 return err; 1425 } 1426 1427 static ssize_t pmu_cpumask_attr_show(struct device *dev, 1428 struct device_attribute *attr, char *buf) 1429 { 1430 struct pmu *pmu = dev_get_drvdata(dev); 1431 struct cci_pmu *cci_pmu = to_cci_pmu(pmu); 1432 1433 int n = scnprintf(buf, PAGE_SIZE - 1, "%*pbl", 1434 cpumask_pr_args(&cci_pmu->cpus)); 1435 buf[n++] = '\n'; 1436 buf[n] = '\0'; 1437 return n; 1438 } 1439 1440 static struct device_attribute pmu_cpumask_attr = 1441 __ATTR(cpumask, S_IRUGO, pmu_cpumask_attr_show, NULL); 1442 1443 static struct attribute *pmu_attrs[] = { 1444 &pmu_cpumask_attr.attr, 1445 NULL, 1446 }; 1447 1448 static struct attribute_group pmu_attr_group = { 1449 .attrs = pmu_attrs, 1450 }; 1451 1452 static struct attribute_group pmu_format_attr_group = { 1453 .name = "format", 1454 .attrs = NULL, /* Filled in cci_pmu_init_attrs */ 1455 }; 1456 1457 static struct attribute_group pmu_event_attr_group = { 1458 .name = "events", 1459 .attrs = NULL, /* Filled in cci_pmu_init_attrs */ 1460 }; 1461 1462 static const struct attribute_group *pmu_attr_groups[] = { 1463 &pmu_attr_group, 1464 &pmu_format_attr_group, 1465 &pmu_event_attr_group, 1466 NULL 1467 }; 1468 1469 static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev) 1470 { 1471 const struct cci_pmu_model *model = cci_pmu->model; 1472 char *name = model->name; 1473 u32 num_cntrs; 1474 1475 pmu_event_attr_group.attrs = model->event_attrs; 1476 pmu_format_attr_group.attrs = model->format_attrs; 1477 1478 cci_pmu->pmu = (struct pmu) { 1479 .name = cci_pmu->model->name, 1480 .task_ctx_nr = perf_invalid_context, 1481 .pmu_enable = cci_pmu_enable, 1482 .pmu_disable = cci_pmu_disable, 1483 .event_init = cci_pmu_event_init, 1484 .add = cci_pmu_add, 1485 .del = cci_pmu_del, 1486 .start = cci_pmu_start, 1487 .stop = cci_pmu_stop, 1488 .read = pmu_read, 1489 .attr_groups = pmu_attr_groups, 1490 }; 1491 1492 cci_pmu->plat_device = pdev; 1493 num_cntrs = pmu_get_max_counters(); 1494 if (num_cntrs > cci_pmu->model->num_hw_cntrs) { 1495 dev_warn(&pdev->dev, 1496 "PMU implements more counters(%d) than supported by" 1497 " the model(%d), truncated.", 1498 num_cntrs, cci_pmu->model->num_hw_cntrs); 1499 num_cntrs = cci_pmu->model->num_hw_cntrs; 1500 } 1501 cci_pmu->num_cntrs = num_cntrs + cci_pmu->model->fixed_hw_cntrs; 1502 1503 return perf_pmu_register(&cci_pmu->pmu, name, -1); 1504 } 1505 1506 static int cci_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 1507 { 1508 struct cci_pmu *cci_pmu = hlist_entry_safe(node, struct cci_pmu, node); 1509 unsigned int target; 1510 1511 if (!cpumask_test_and_clear_cpu(cpu, &cci_pmu->cpus)) 1512 return 0; 1513 target = cpumask_any_but(cpu_online_mask, cpu); 1514 if (target >= nr_cpu_ids) 1515 return 0; 1516 /* 1517 * TODO: migrate context once core races on event->ctx have 1518 * been fixed. 1519 */ 1520 cpumask_set_cpu(target, &cci_pmu->cpus); 1521 return 0; 1522 } 1523 1524 static struct cci_pmu_model cci_pmu_models[] = { 1525 #ifdef CONFIG_ARM_CCI400_PMU 1526 [CCI400_R0] = { 1527 .name = "CCI_400", 1528 .fixed_hw_cntrs = 1, /* Cycle counter */ 1529 .num_hw_cntrs = 4, 1530 .cntr_size = SZ_4K, 1531 .format_attrs = cci400_pmu_format_attrs, 1532 .event_attrs = cci400_r0_pmu_event_attrs, 1533 .event_ranges = { 1534 [CCI_IF_SLAVE] = { 1535 CCI400_R0_SLAVE_PORT_MIN_EV, 1536 CCI400_R0_SLAVE_PORT_MAX_EV, 1537 }, 1538 [CCI_IF_MASTER] = { 1539 CCI400_R0_MASTER_PORT_MIN_EV, 1540 CCI400_R0_MASTER_PORT_MAX_EV, 1541 }, 1542 }, 1543 .validate_hw_event = cci400_validate_hw_event, 1544 .get_event_idx = cci400_get_event_idx, 1545 }, 1546 [CCI400_R1] = { 1547 .name = "CCI_400_r1", 1548 .fixed_hw_cntrs = 1, /* Cycle counter */ 1549 .num_hw_cntrs = 4, 1550 .cntr_size = SZ_4K, 1551 .format_attrs = cci400_pmu_format_attrs, 1552 .event_attrs = cci400_r1_pmu_event_attrs, 1553 .event_ranges = { 1554 [CCI_IF_SLAVE] = { 1555 CCI400_R1_SLAVE_PORT_MIN_EV, 1556 CCI400_R1_SLAVE_PORT_MAX_EV, 1557 }, 1558 [CCI_IF_MASTER] = { 1559 CCI400_R1_MASTER_PORT_MIN_EV, 1560 CCI400_R1_MASTER_PORT_MAX_EV, 1561 }, 1562 }, 1563 .validate_hw_event = cci400_validate_hw_event, 1564 .get_event_idx = cci400_get_event_idx, 1565 }, 1566 #endif 1567 #ifdef CONFIG_ARM_CCI5xx_PMU 1568 [CCI500_R0] = { 1569 .name = "CCI_500", 1570 .fixed_hw_cntrs = 0, 1571 .num_hw_cntrs = 8, 1572 .cntr_size = SZ_64K, 1573 .format_attrs = cci5xx_pmu_format_attrs, 1574 .event_attrs = cci5xx_pmu_event_attrs, 1575 .event_ranges = { 1576 [CCI_IF_SLAVE] = { 1577 CCI5xx_SLAVE_PORT_MIN_EV, 1578 CCI5xx_SLAVE_PORT_MAX_EV, 1579 }, 1580 [CCI_IF_MASTER] = { 1581 CCI5xx_MASTER_PORT_MIN_EV, 1582 CCI5xx_MASTER_PORT_MAX_EV, 1583 }, 1584 [CCI_IF_GLOBAL] = { 1585 CCI5xx_GLOBAL_PORT_MIN_EV, 1586 CCI5xx_GLOBAL_PORT_MAX_EV, 1587 }, 1588 }, 1589 .validate_hw_event = cci500_validate_hw_event, 1590 .write_counters = cci5xx_pmu_write_counters, 1591 }, 1592 [CCI550_R0] = { 1593 .name = "CCI_550", 1594 .fixed_hw_cntrs = 0, 1595 .num_hw_cntrs = 8, 1596 .cntr_size = SZ_64K, 1597 .format_attrs = cci5xx_pmu_format_attrs, 1598 .event_attrs = cci5xx_pmu_event_attrs, 1599 .event_ranges = { 1600 [CCI_IF_SLAVE] = { 1601 CCI5xx_SLAVE_PORT_MIN_EV, 1602 CCI5xx_SLAVE_PORT_MAX_EV, 1603 }, 1604 [CCI_IF_MASTER] = { 1605 CCI5xx_MASTER_PORT_MIN_EV, 1606 CCI5xx_MASTER_PORT_MAX_EV, 1607 }, 1608 [CCI_IF_GLOBAL] = { 1609 CCI5xx_GLOBAL_PORT_MIN_EV, 1610 CCI5xx_GLOBAL_PORT_MAX_EV, 1611 }, 1612 }, 1613 .validate_hw_event = cci550_validate_hw_event, 1614 .write_counters = cci5xx_pmu_write_counters, 1615 }, 1616 #endif 1617 }; 1618 1619 static const struct of_device_id arm_cci_pmu_matches[] = { 1620 #ifdef CONFIG_ARM_CCI400_PMU 1621 { 1622 .compatible = "arm,cci-400-pmu", 1623 .data = NULL, 1624 }, 1625 { 1626 .compatible = "arm,cci-400-pmu,r0", 1627 .data = &cci_pmu_models[CCI400_R0], 1628 }, 1629 { 1630 .compatible = "arm,cci-400-pmu,r1", 1631 .data = &cci_pmu_models[CCI400_R1], 1632 }, 1633 #endif 1634 #ifdef CONFIG_ARM_CCI5xx_PMU 1635 { 1636 .compatible = "arm,cci-500-pmu,r0", 1637 .data = &cci_pmu_models[CCI500_R0], 1638 }, 1639 { 1640 .compatible = "arm,cci-550-pmu,r0", 1641 .data = &cci_pmu_models[CCI550_R0], 1642 }, 1643 #endif 1644 {}, 1645 }; 1646 1647 static inline const struct cci_pmu_model *get_cci_model(struct platform_device *pdev) 1648 { 1649 const struct of_device_id *match = of_match_node(arm_cci_pmu_matches, 1650 pdev->dev.of_node); 1651 if (!match) 1652 return NULL; 1653 if (match->data) 1654 return match->data; 1655 1656 dev_warn(&pdev->dev, "DEPRECATED compatible property," 1657 "requires secure access to CCI registers"); 1658 return probe_cci_model(pdev); 1659 } 1660 1661 static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs) 1662 { 1663 int i; 1664 1665 for (i = 0; i < nr_irqs; i++) 1666 if (irq == irqs[i]) 1667 return true; 1668 1669 return false; 1670 } 1671 1672 static struct cci_pmu *cci_pmu_alloc(struct platform_device *pdev) 1673 { 1674 struct cci_pmu *cci_pmu; 1675 const struct cci_pmu_model *model; 1676 1677 /* 1678 * All allocations are devm_* hence we don't have to free 1679 * them explicitly on an error, as it would end up in driver 1680 * detach. 1681 */ 1682 model = get_cci_model(pdev); 1683 if (!model) { 1684 dev_warn(&pdev->dev, "CCI PMU version not supported\n"); 1685 return ERR_PTR(-ENODEV); 1686 } 1687 1688 cci_pmu = devm_kzalloc(&pdev->dev, sizeof(*cci_pmu), GFP_KERNEL); 1689 if (!cci_pmu) 1690 return ERR_PTR(-ENOMEM); 1691 1692 cci_pmu->model = model; 1693 cci_pmu->irqs = devm_kcalloc(&pdev->dev, CCI_PMU_MAX_HW_CNTRS(model), 1694 sizeof(*cci_pmu->irqs), GFP_KERNEL); 1695 if (!cci_pmu->irqs) 1696 return ERR_PTR(-ENOMEM); 1697 cci_pmu->hw_events.events = devm_kcalloc(&pdev->dev, 1698 CCI_PMU_MAX_HW_CNTRS(model), 1699 sizeof(*cci_pmu->hw_events.events), 1700 GFP_KERNEL); 1701 if (!cci_pmu->hw_events.events) 1702 return ERR_PTR(-ENOMEM); 1703 cci_pmu->hw_events.used_mask = devm_kcalloc(&pdev->dev, 1704 BITS_TO_LONGS(CCI_PMU_MAX_HW_CNTRS(model)), 1705 sizeof(*cci_pmu->hw_events.used_mask), 1706 GFP_KERNEL); 1707 if (!cci_pmu->hw_events.used_mask) 1708 return ERR_PTR(-ENOMEM); 1709 1710 return cci_pmu; 1711 } 1712 1713 1714 static int cci_pmu_probe(struct platform_device *pdev) 1715 { 1716 struct resource *res; 1717 struct cci_pmu *cci_pmu; 1718 int i, ret, irq; 1719 1720 cci_pmu = cci_pmu_alloc(pdev); 1721 if (IS_ERR(cci_pmu)) 1722 return PTR_ERR(cci_pmu); 1723 1724 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1725 cci_pmu->base = devm_ioremap_resource(&pdev->dev, res); 1726 if (IS_ERR(cci_pmu->base)) 1727 return -ENOMEM; 1728 1729 /* 1730 * CCI PMU has one overflow interrupt per counter; but some may be tied 1731 * together to a common interrupt. 1732 */ 1733 cci_pmu->nr_irqs = 0; 1734 for (i = 0; i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model); i++) { 1735 irq = platform_get_irq(pdev, i); 1736 if (irq < 0) 1737 break; 1738 1739 if (is_duplicate_irq(irq, cci_pmu->irqs, cci_pmu->nr_irqs)) 1740 continue; 1741 1742 cci_pmu->irqs[cci_pmu->nr_irqs++] = irq; 1743 } 1744 1745 /* 1746 * Ensure that the device tree has as many interrupts as the number 1747 * of counters. 1748 */ 1749 if (i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)) { 1750 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n", 1751 i, CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)); 1752 return -EINVAL; 1753 } 1754 1755 raw_spin_lock_init(&cci_pmu->hw_events.pmu_lock); 1756 mutex_init(&cci_pmu->reserve_mutex); 1757 atomic_set(&cci_pmu->active_events, 0); 1758 cpumask_set_cpu(smp_processor_id(), &cci_pmu->cpus); 1759 1760 ret = cci_pmu_init(cci_pmu, pdev); 1761 if (ret) 1762 return ret; 1763 1764 cpuhp_state_add_instance_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE, 1765 &cci_pmu->node); 1766 pr_info("ARM %s PMU driver probed", cci_pmu->model->name); 1767 return 0; 1768 } 1769 1770 static int cci_platform_probe(struct platform_device *pdev) 1771 { 1772 if (!cci_probed()) 1773 return -ENODEV; 1774 1775 return of_platform_populate(pdev->dev.of_node, NULL, NULL, &pdev->dev); 1776 } 1777 1778 static struct platform_driver cci_pmu_driver = { 1779 .driver = { 1780 .name = DRIVER_NAME_PMU, 1781 .of_match_table = arm_cci_pmu_matches, 1782 }, 1783 .probe = cci_pmu_probe, 1784 }; 1785 1786 static struct platform_driver cci_platform_driver = { 1787 .driver = { 1788 .name = DRIVER_NAME, 1789 .of_match_table = arm_cci_matches, 1790 }, 1791 .probe = cci_platform_probe, 1792 }; 1793 1794 static int __init cci_platform_init(void) 1795 { 1796 int ret; 1797 1798 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_CCI_ONLINE, 1799 "perf/arm/cci:online", NULL, 1800 cci_pmu_offline_cpu); 1801 if (ret) 1802 return ret; 1803 1804 ret = platform_driver_register(&cci_pmu_driver); 1805 if (ret) 1806 return ret; 1807 1808 return platform_driver_register(&cci_platform_driver); 1809 } 1810 1811 #else /* !CONFIG_ARM_CCI_PMU */ 1812 1813 static int __init cci_platform_init(void) 1814 { 1815 return 0; 1816 } 1817 1818 #endif /* CONFIG_ARM_CCI_PMU */ 1819 1820 #ifdef CONFIG_ARM_CCI400_PORT_CTRL 1821 1822 #define CCI_PORT_CTRL 0x0 1823 #define CCI_CTRL_STATUS 0xc 1824 1825 #define CCI_ENABLE_SNOOP_REQ 0x1 1826 #define CCI_ENABLE_DVM_REQ 0x2 1827 #define CCI_ENABLE_REQ (CCI_ENABLE_SNOOP_REQ | CCI_ENABLE_DVM_REQ) 1828 1829 enum cci_ace_port_type { 1830 ACE_INVALID_PORT = 0x0, 1831 ACE_PORT, 1832 ACE_LITE_PORT, 1833 }; 1834 1835 struct cci_ace_port { 1836 void __iomem *base; 1837 unsigned long phys; 1838 enum cci_ace_port_type type; 1839 struct device_node *dn; 1840 }; 1841 1842 static struct cci_ace_port *ports; 1843 static unsigned int nb_cci_ports; 1844 1845 struct cpu_port { 1846 u64 mpidr; 1847 u32 port; 1848 }; 1849 1850 /* 1851 * Use the port MSB as valid flag, shift can be made dynamic 1852 * by computing number of bits required for port indexes. 1853 * Code disabling CCI cpu ports runs with D-cache invalidated 1854 * and SCTLR bit clear so data accesses must be kept to a minimum 1855 * to improve performance; for now shift is left static to 1856 * avoid one more data access while disabling the CCI port. 1857 */ 1858 #define PORT_VALID_SHIFT 31 1859 #define PORT_VALID (0x1 << PORT_VALID_SHIFT) 1860 1861 static inline void init_cpu_port(struct cpu_port *port, u32 index, u64 mpidr) 1862 { 1863 port->port = PORT_VALID | index; 1864 port->mpidr = mpidr; 1865 } 1866 1867 static inline bool cpu_port_is_valid(struct cpu_port *port) 1868 { 1869 return !!(port->port & PORT_VALID); 1870 } 1871 1872 static inline bool cpu_port_match(struct cpu_port *port, u64 mpidr) 1873 { 1874 return port->mpidr == (mpidr & MPIDR_HWID_BITMASK); 1875 } 1876 1877 static struct cpu_port cpu_port[NR_CPUS]; 1878 1879 /** 1880 * __cci_ace_get_port - Function to retrieve the port index connected to 1881 * a cpu or device. 1882 * 1883 * @dn: device node of the device to look-up 1884 * @type: port type 1885 * 1886 * Return value: 1887 * - CCI port index if success 1888 * - -ENODEV if failure 1889 */ 1890 static int __cci_ace_get_port(struct device_node *dn, int type) 1891 { 1892 int i; 1893 bool ace_match; 1894 struct device_node *cci_portn; 1895 1896 cci_portn = of_parse_phandle(dn, "cci-control-port", 0); 1897 for (i = 0; i < nb_cci_ports; i++) { 1898 ace_match = ports[i].type == type; 1899 if (ace_match && cci_portn == ports[i].dn) 1900 return i; 1901 } 1902 return -ENODEV; 1903 } 1904 1905 int cci_ace_get_port(struct device_node *dn) 1906 { 1907 return __cci_ace_get_port(dn, ACE_LITE_PORT); 1908 } 1909 EXPORT_SYMBOL_GPL(cci_ace_get_port); 1910 1911 static void cci_ace_init_ports(void) 1912 { 1913 int port, cpu; 1914 struct device_node *cpun; 1915 1916 /* 1917 * Port index look-up speeds up the function disabling ports by CPU, 1918 * since the logical to port index mapping is done once and does 1919 * not change after system boot. 1920 * The stashed index array is initialized for all possible CPUs 1921 * at probe time. 1922 */ 1923 for_each_possible_cpu(cpu) { 1924 /* too early to use cpu->of_node */ 1925 cpun = of_get_cpu_node(cpu, NULL); 1926 1927 if (WARN(!cpun, "Missing cpu device node\n")) 1928 continue; 1929 1930 port = __cci_ace_get_port(cpun, ACE_PORT); 1931 if (port < 0) 1932 continue; 1933 1934 init_cpu_port(&cpu_port[cpu], port, cpu_logical_map(cpu)); 1935 } 1936 1937 for_each_possible_cpu(cpu) { 1938 WARN(!cpu_port_is_valid(&cpu_port[cpu]), 1939 "CPU %u does not have an associated CCI port\n", 1940 cpu); 1941 } 1942 } 1943 /* 1944 * Functions to enable/disable a CCI interconnect slave port 1945 * 1946 * They are called by low-level power management code to disable slave 1947 * interfaces snoops and DVM broadcast. 1948 * Since they may execute with cache data allocation disabled and 1949 * after the caches have been cleaned and invalidated the functions provide 1950 * no explicit locking since they may run with D-cache disabled, so normal 1951 * cacheable kernel locks based on ldrex/strex may not work. 1952 * Locking has to be provided by BSP implementations to ensure proper 1953 * operations. 1954 */ 1955 1956 /** 1957 * cci_port_control() - function to control a CCI port 1958 * 1959 * @port: index of the port to setup 1960 * @enable: if true enables the port, if false disables it 1961 */ 1962 static void notrace cci_port_control(unsigned int port, bool enable) 1963 { 1964 void __iomem *base = ports[port].base; 1965 1966 writel_relaxed(enable ? CCI_ENABLE_REQ : 0, base + CCI_PORT_CTRL); 1967 /* 1968 * This function is called from power down procedures 1969 * and must not execute any instruction that might 1970 * cause the processor to be put in a quiescent state 1971 * (eg wfi). Hence, cpu_relax() can not be added to this 1972 * read loop to optimize power, since it might hide possibly 1973 * disruptive operations. 1974 */ 1975 while (readl_relaxed(cci_ctrl_base + CCI_CTRL_STATUS) & 0x1) 1976 ; 1977 } 1978 1979 /** 1980 * cci_disable_port_by_cpu() - function to disable a CCI port by CPU 1981 * reference 1982 * 1983 * @mpidr: mpidr of the CPU whose CCI port should be disabled 1984 * 1985 * Disabling a CCI port for a CPU implies disabling the CCI port 1986 * controlling that CPU cluster. Code disabling CPU CCI ports 1987 * must make sure that the CPU running the code is the last active CPU 1988 * in the cluster ie all other CPUs are quiescent in a low power state. 1989 * 1990 * Return: 1991 * 0 on success 1992 * -ENODEV on port look-up failure 1993 */ 1994 int notrace cci_disable_port_by_cpu(u64 mpidr) 1995 { 1996 int cpu; 1997 bool is_valid; 1998 for (cpu = 0; cpu < nr_cpu_ids; cpu++) { 1999 is_valid = cpu_port_is_valid(&cpu_port[cpu]); 2000 if (is_valid && cpu_port_match(&cpu_port[cpu], mpidr)) { 2001 cci_port_control(cpu_port[cpu].port, false); 2002 return 0; 2003 } 2004 } 2005 return -ENODEV; 2006 } 2007 EXPORT_SYMBOL_GPL(cci_disable_port_by_cpu); 2008 2009 /** 2010 * cci_enable_port_for_self() - enable a CCI port for calling CPU 2011 * 2012 * Enabling a CCI port for the calling CPU implies enabling the CCI 2013 * port controlling that CPU's cluster. Caller must make sure that the 2014 * CPU running the code is the first active CPU in the cluster and all 2015 * other CPUs are quiescent in a low power state or waiting for this CPU 2016 * to complete the CCI initialization. 2017 * 2018 * Because this is called when the MMU is still off and with no stack, 2019 * the code must be position independent and ideally rely on callee 2020 * clobbered registers only. To achieve this we must code this function 2021 * entirely in assembler. 2022 * 2023 * On success this returns with the proper CCI port enabled. In case of 2024 * any failure this never returns as the inability to enable the CCI is 2025 * fatal and there is no possible recovery at this stage. 2026 */ 2027 asmlinkage void __naked cci_enable_port_for_self(void) 2028 { 2029 asm volatile ("\n" 2030 " .arch armv7-a\n" 2031 " mrc p15, 0, r0, c0, c0, 5 @ get MPIDR value \n" 2032 " and r0, r0, #"__stringify(MPIDR_HWID_BITMASK)" \n" 2033 " adr r1, 5f \n" 2034 " ldr r2, [r1] \n" 2035 " add r1, r1, r2 @ &cpu_port \n" 2036 " add ip, r1, %[sizeof_cpu_port] \n" 2037 2038 /* Loop over the cpu_port array looking for a matching MPIDR */ 2039 "1: ldr r2, [r1, %[offsetof_cpu_port_mpidr_lsb]] \n" 2040 " cmp r2, r0 @ compare MPIDR \n" 2041 " bne 2f \n" 2042 2043 /* Found a match, now test port validity */ 2044 " ldr r3, [r1, %[offsetof_cpu_port_port]] \n" 2045 " tst r3, #"__stringify(PORT_VALID)" \n" 2046 " bne 3f \n" 2047 2048 /* no match, loop with the next cpu_port entry */ 2049 "2: add r1, r1, %[sizeof_struct_cpu_port] \n" 2050 " cmp r1, ip @ done? \n" 2051 " blo 1b \n" 2052 2053 /* CCI port not found -- cheaply try to stall this CPU */ 2054 "cci_port_not_found: \n" 2055 " wfi \n" 2056 " wfe \n" 2057 " b cci_port_not_found \n" 2058 2059 /* Use matched port index to look up the corresponding ports entry */ 2060 "3: bic r3, r3, #"__stringify(PORT_VALID)" \n" 2061 " adr r0, 6f \n" 2062 " ldmia r0, {r1, r2} \n" 2063 " sub r1, r1, r0 @ virt - phys \n" 2064 " ldr r0, [r0, r2] @ *(&ports) \n" 2065 " mov r2, %[sizeof_struct_ace_port] \n" 2066 " mla r0, r2, r3, r0 @ &ports[index] \n" 2067 " sub r0, r0, r1 @ virt_to_phys() \n" 2068 2069 /* Enable the CCI port */ 2070 " ldr r0, [r0, %[offsetof_port_phys]] \n" 2071 " mov r3, %[cci_enable_req]\n" 2072 " str r3, [r0, #"__stringify(CCI_PORT_CTRL)"] \n" 2073 2074 /* poll the status reg for completion */ 2075 " adr r1, 7f \n" 2076 " ldr r0, [r1] \n" 2077 " ldr r0, [r0, r1] @ cci_ctrl_base \n" 2078 "4: ldr r1, [r0, #"__stringify(CCI_CTRL_STATUS)"] \n" 2079 " tst r1, %[cci_control_status_bits] \n" 2080 " bne 4b \n" 2081 2082 " mov r0, #0 \n" 2083 " bx lr \n" 2084 2085 " .align 2 \n" 2086 "5: .word cpu_port - . \n" 2087 "6: .word . \n" 2088 " .word ports - 6b \n" 2089 "7: .word cci_ctrl_phys - . \n" 2090 : : 2091 [sizeof_cpu_port] "i" (sizeof(cpu_port)), 2092 [cci_enable_req] "i" cpu_to_le32(CCI_ENABLE_REQ), 2093 [cci_control_status_bits] "i" cpu_to_le32(1), 2094 #ifndef __ARMEB__ 2095 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)), 2096 #else 2097 [offsetof_cpu_port_mpidr_lsb] "i" (offsetof(struct cpu_port, mpidr)+4), 2098 #endif 2099 [offsetof_cpu_port_port] "i" (offsetof(struct cpu_port, port)), 2100 [sizeof_struct_cpu_port] "i" (sizeof(struct cpu_port)), 2101 [sizeof_struct_ace_port] "i" (sizeof(struct cci_ace_port)), 2102 [offsetof_port_phys] "i" (offsetof(struct cci_ace_port, phys)) ); 2103 2104 unreachable(); 2105 } 2106 2107 /** 2108 * __cci_control_port_by_device() - function to control a CCI port by device 2109 * reference 2110 * 2111 * @dn: device node pointer of the device whose CCI port should be 2112 * controlled 2113 * @enable: if true enables the port, if false disables it 2114 * 2115 * Return: 2116 * 0 on success 2117 * -ENODEV on port look-up failure 2118 */ 2119 int notrace __cci_control_port_by_device(struct device_node *dn, bool enable) 2120 { 2121 int port; 2122 2123 if (!dn) 2124 return -ENODEV; 2125 2126 port = __cci_ace_get_port(dn, ACE_LITE_PORT); 2127 if (WARN_ONCE(port < 0, "node %s ACE lite port look-up failure\n", 2128 dn->full_name)) 2129 return -ENODEV; 2130 cci_port_control(port, enable); 2131 return 0; 2132 } 2133 EXPORT_SYMBOL_GPL(__cci_control_port_by_device); 2134 2135 /** 2136 * __cci_control_port_by_index() - function to control a CCI port by port index 2137 * 2138 * @port: port index previously retrieved with cci_ace_get_port() 2139 * @enable: if true enables the port, if false disables it 2140 * 2141 * Return: 2142 * 0 on success 2143 * -ENODEV on port index out of range 2144 * -EPERM if operation carried out on an ACE PORT 2145 */ 2146 int notrace __cci_control_port_by_index(u32 port, bool enable) 2147 { 2148 if (port >= nb_cci_ports || ports[port].type == ACE_INVALID_PORT) 2149 return -ENODEV; 2150 /* 2151 * CCI control for ports connected to CPUS is extremely fragile 2152 * and must be made to go through a specific and controlled 2153 * interface (ie cci_disable_port_by_cpu(); control by general purpose 2154 * indexing is therefore disabled for ACE ports. 2155 */ 2156 if (ports[port].type == ACE_PORT) 2157 return -EPERM; 2158 2159 cci_port_control(port, enable); 2160 return 0; 2161 } 2162 EXPORT_SYMBOL_GPL(__cci_control_port_by_index); 2163 2164 static const struct of_device_id arm_cci_ctrl_if_matches[] = { 2165 {.compatible = "arm,cci-400-ctrl-if", }, 2166 {}, 2167 }; 2168 2169 static int cci_probe_ports(struct device_node *np) 2170 { 2171 struct cci_nb_ports const *cci_config; 2172 int ret, i, nb_ace = 0, nb_ace_lite = 0; 2173 struct device_node *cp; 2174 struct resource res; 2175 const char *match_str; 2176 bool is_ace; 2177 2178 2179 cci_config = of_match_node(arm_cci_matches, np)->data; 2180 if (!cci_config) 2181 return -ENODEV; 2182 2183 nb_cci_ports = cci_config->nb_ace + cci_config->nb_ace_lite; 2184 2185 ports = kcalloc(nb_cci_ports, sizeof(*ports), GFP_KERNEL); 2186 if (!ports) 2187 return -ENOMEM; 2188 2189 for_each_child_of_node(np, cp) { 2190 if (!of_match_node(arm_cci_ctrl_if_matches, cp)) 2191 continue; 2192 2193 if (!of_device_is_available(cp)) 2194 continue; 2195 2196 i = nb_ace + nb_ace_lite; 2197 2198 if (i >= nb_cci_ports) 2199 break; 2200 2201 if (of_property_read_string(cp, "interface-type", 2202 &match_str)) { 2203 WARN(1, "node %s missing interface-type property\n", 2204 cp->full_name); 2205 continue; 2206 } 2207 is_ace = strcmp(match_str, "ace") == 0; 2208 if (!is_ace && strcmp(match_str, "ace-lite")) { 2209 WARN(1, "node %s containing invalid interface-type property, skipping it\n", 2210 cp->full_name); 2211 continue; 2212 } 2213 2214 ret = of_address_to_resource(cp, 0, &res); 2215 if (!ret) { 2216 ports[i].base = ioremap(res.start, resource_size(&res)); 2217 ports[i].phys = res.start; 2218 } 2219 if (ret || !ports[i].base) { 2220 WARN(1, "unable to ioremap CCI port %d\n", i); 2221 continue; 2222 } 2223 2224 if (is_ace) { 2225 if (WARN_ON(nb_ace >= cci_config->nb_ace)) 2226 continue; 2227 ports[i].type = ACE_PORT; 2228 ++nb_ace; 2229 } else { 2230 if (WARN_ON(nb_ace_lite >= cci_config->nb_ace_lite)) 2231 continue; 2232 ports[i].type = ACE_LITE_PORT; 2233 ++nb_ace_lite; 2234 } 2235 ports[i].dn = cp; 2236 } 2237 2238 /* 2239 * If there is no CCI port that is under kernel control 2240 * return early and report probe status. 2241 */ 2242 if (!nb_ace && !nb_ace_lite) 2243 return -ENODEV; 2244 2245 /* initialize a stashed array of ACE ports to speed-up look-up */ 2246 cci_ace_init_ports(); 2247 2248 /* 2249 * Multi-cluster systems may need this data when non-coherent, during 2250 * cluster power-up/power-down. Make sure it reaches main memory. 2251 */ 2252 sync_cache_w(&cci_ctrl_base); 2253 sync_cache_w(&cci_ctrl_phys); 2254 sync_cache_w(&ports); 2255 sync_cache_w(&cpu_port); 2256 __sync_cache_range_w(ports, sizeof(*ports) * nb_cci_ports); 2257 pr_info("ARM CCI driver probed\n"); 2258 2259 return 0; 2260 } 2261 #else /* !CONFIG_ARM_CCI400_PORT_CTRL */ 2262 static inline int cci_probe_ports(struct device_node *np) 2263 { 2264 return 0; 2265 } 2266 #endif /* CONFIG_ARM_CCI400_PORT_CTRL */ 2267 2268 static int cci_probe(void) 2269 { 2270 int ret; 2271 struct device_node *np; 2272 struct resource res; 2273 2274 np = of_find_matching_node(NULL, arm_cci_matches); 2275 if(!np || !of_device_is_available(np)) 2276 return -ENODEV; 2277 2278 ret = of_address_to_resource(np, 0, &res); 2279 if (!ret) { 2280 cci_ctrl_base = ioremap(res.start, resource_size(&res)); 2281 cci_ctrl_phys = res.start; 2282 } 2283 if (ret || !cci_ctrl_base) { 2284 WARN(1, "unable to ioremap CCI ctrl\n"); 2285 return -ENXIO; 2286 } 2287 2288 return cci_probe_ports(np); 2289 } 2290 2291 static int cci_init_status = -EAGAIN; 2292 static DEFINE_MUTEX(cci_probing); 2293 2294 static int cci_init(void) 2295 { 2296 if (cci_init_status != -EAGAIN) 2297 return cci_init_status; 2298 2299 mutex_lock(&cci_probing); 2300 if (cci_init_status == -EAGAIN) 2301 cci_init_status = cci_probe(); 2302 mutex_unlock(&cci_probing); 2303 return cci_init_status; 2304 } 2305 2306 /* 2307 * To sort out early init calls ordering a helper function is provided to 2308 * check if the CCI driver has beed initialized. Function check if the driver 2309 * has been initialized, if not it calls the init function that probes 2310 * the driver and updates the return value. 2311 */ 2312 bool cci_probed(void) 2313 { 2314 return cci_init() == 0; 2315 } 2316 EXPORT_SYMBOL_GPL(cci_probed); 2317 2318 early_initcall(cci_init); 2319 core_initcall(cci_platform_init); 2320 MODULE_LICENSE("GPL"); 2321 MODULE_DESCRIPTION("ARM CCI support"); 2322