1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * APM X-Gene SoC PMU (Performance Monitor Unit) 4 * 5 * Copyright (c) 2016, Applied Micro Circuits Corporation 6 * Author: Hoan Tran <hotran@apm.com> 7 * Tai Nguyen <ttnguyen@apm.com> 8 */ 9 10 #include <linux/acpi.h> 11 #include <linux/clk.h> 12 #include <linux/cpuhotplug.h> 13 #include <linux/cpumask.h> 14 #include <linux/interrupt.h> 15 #include <linux/io.h> 16 #include <linux/mfd/syscon.h> 17 #include <linux/module.h> 18 #include <linux/of_address.h> 19 #include <linux/perf_event.h> 20 #include <linux/platform_device.h> 21 #include <linux/property.h> 22 #include <linux/regmap.h> 23 #include <linux/slab.h> 24 25 #define CSW_CSWCR 0x0000 26 #define CSW_CSWCR_DUALMCB_MASK BIT(0) 27 #define CSW_CSWCR_MCB0_ROUTING(x) (((x) & 0x0C) >> 2) 28 #define CSW_CSWCR_MCB1_ROUTING(x) (((x) & 0x30) >> 4) 29 #define MCBADDRMR 0x0000 30 #define MCBADDRMR_DUALMCU_MODE_MASK BIT(2) 31 32 #define PCPPMU_INTSTATUS_REG 0x000 33 #define PCPPMU_INTMASK_REG 0x004 34 #define PCPPMU_INTMASK 0x0000000F 35 #define PCPPMU_INTENMASK 0xFFFFFFFF 36 #define PCPPMU_INTCLRMASK 0xFFFFFFF0 37 #define PCPPMU_INT_MCU BIT(0) 38 #define PCPPMU_INT_MCB BIT(1) 39 #define PCPPMU_INT_L3C BIT(2) 40 #define PCPPMU_INT_IOB BIT(3) 41 42 #define PCPPMU_V3_INTMASK 0x00FF33FF 43 #define PCPPMU_V3_INTENMASK 0xFFFFFFFF 44 #define PCPPMU_V3_INTCLRMASK 0xFF00CC00 45 #define PCPPMU_V3_INT_MCU 0x000000FF 46 #define PCPPMU_V3_INT_MCB 0x00000300 47 #define PCPPMU_V3_INT_L3C 0x00FF0000 48 #define PCPPMU_V3_INT_IOB 0x00003000 49 50 #define PMU_MAX_COUNTERS 4 51 #define PMU_CNT_MAX_PERIOD 0xFFFFFFFFULL 52 #define PMU_V3_CNT_MAX_PERIOD 0xFFFFFFFFFFFFFFFFULL 53 #define PMU_OVERFLOW_MASK 0xF 54 #define PMU_PMCR_E BIT(0) 55 #define PMU_PMCR_P BIT(1) 56 57 #define PMU_PMEVCNTR0 0x000 58 #define PMU_PMEVCNTR1 0x004 59 #define PMU_PMEVCNTR2 0x008 60 #define PMU_PMEVCNTR3 0x00C 61 #define PMU_PMEVTYPER0 0x400 62 #define PMU_PMEVTYPER1 0x404 63 #define PMU_PMEVTYPER2 0x408 64 #define PMU_PMEVTYPER3 0x40C 65 #define PMU_PMAMR0 0xA00 66 #define PMU_PMAMR1 0xA04 67 #define PMU_PMCNTENSET 0xC00 68 #define PMU_PMCNTENCLR 0xC20 69 #define PMU_PMINTENSET 0xC40 70 #define PMU_PMINTENCLR 0xC60 71 #define PMU_PMOVSR 0xC80 72 #define PMU_PMCR 0xE04 73 74 /* PMU registers for V3 */ 75 #define PMU_PMOVSCLR 0xC80 76 #define PMU_PMOVSSET 0xCC0 77 78 #define to_pmu_dev(p) container_of(p, struct xgene_pmu_dev, pmu) 79 #define GET_CNTR(ev) (ev->hw.idx) 80 #define GET_EVENTID(ev) (ev->hw.config & 0xFFULL) 81 #define GET_AGENTID(ev) (ev->hw.config_base & 0xFFFFFFFFUL) 82 #define GET_AGENT1ID(ev) ((ev->hw.config_base >> 32) & 0xFFFFFFFFUL) 83 84 struct hw_pmu_info { 85 u32 type; 86 u32 enable_mask; 87 void __iomem *csr; 88 }; 89 90 struct xgene_pmu_dev { 91 struct hw_pmu_info *inf; 92 struct xgene_pmu *parent; 93 struct pmu pmu; 94 u8 max_counters; 95 DECLARE_BITMAP(cntr_assign_mask, PMU_MAX_COUNTERS); 96 u64 max_period; 97 const struct attribute_group **attr_groups; 98 struct perf_event *pmu_counter_event[PMU_MAX_COUNTERS]; 99 }; 100 101 struct xgene_pmu_ops { 102 void (*mask_int)(struct xgene_pmu *pmu); 103 void (*unmask_int)(struct xgene_pmu *pmu); 104 u64 (*read_counter)(struct xgene_pmu_dev *pmu, int idx); 105 void (*write_counter)(struct xgene_pmu_dev *pmu, int idx, u64 val); 106 void (*write_evttype)(struct xgene_pmu_dev *pmu_dev, int idx, u32 val); 107 void (*write_agentmsk)(struct xgene_pmu_dev *pmu_dev, u32 val); 108 void (*write_agent1msk)(struct xgene_pmu_dev *pmu_dev, u32 val); 109 void (*enable_counter)(struct xgene_pmu_dev *pmu_dev, int idx); 110 void (*disable_counter)(struct xgene_pmu_dev *pmu_dev, int idx); 111 void (*enable_counter_int)(struct xgene_pmu_dev *pmu_dev, int idx); 112 void (*disable_counter_int)(struct xgene_pmu_dev *pmu_dev, int idx); 113 void (*reset_counters)(struct xgene_pmu_dev *pmu_dev); 114 void (*start_counters)(struct xgene_pmu_dev *pmu_dev); 115 void (*stop_counters)(struct xgene_pmu_dev *pmu_dev); 116 }; 117 118 struct xgene_pmu { 119 struct device *dev; 120 struct hlist_node node; 121 int version; 122 void __iomem *pcppmu_csr; 123 u32 mcb_active_mask; 124 u32 mc_active_mask; 125 u32 l3c_active_mask; 126 cpumask_t cpu; 127 int irq; 128 raw_spinlock_t lock; 129 const struct xgene_pmu_ops *ops; 130 struct list_head l3cpmus; 131 struct list_head iobpmus; 132 struct list_head mcbpmus; 133 struct list_head mcpmus; 134 }; 135 136 struct xgene_pmu_dev_ctx { 137 char *name; 138 struct list_head next; 139 struct xgene_pmu_dev *pmu_dev; 140 struct hw_pmu_info inf; 141 }; 142 143 struct xgene_pmu_data { 144 int id; 145 u32 data; 146 }; 147 148 enum xgene_pmu_version { 149 PCP_PMU_V1 = 1, 150 PCP_PMU_V2, 151 PCP_PMU_V3, 152 }; 153 154 enum xgene_pmu_dev_type { 155 PMU_TYPE_L3C = 0, 156 PMU_TYPE_IOB, 157 PMU_TYPE_IOB_SLOW, 158 PMU_TYPE_MCB, 159 PMU_TYPE_MC, 160 }; 161 162 /* 163 * sysfs format attributes 164 */ 165 static ssize_t xgene_pmu_format_show(struct device *dev, 166 struct device_attribute *attr, char *buf) 167 { 168 struct dev_ext_attribute *eattr; 169 170 eattr = container_of(attr, struct dev_ext_attribute, attr); 171 return sysfs_emit(buf, "%s\n", (char *) eattr->var); 172 } 173 174 #define XGENE_PMU_FORMAT_ATTR(_name, _config) \ 175 (&((struct dev_ext_attribute[]) { \ 176 { .attr = __ATTR(_name, S_IRUGO, xgene_pmu_format_show, NULL), \ 177 .var = (void *) _config, } \ 178 })[0].attr.attr) 179 180 static struct attribute *l3c_pmu_format_attrs[] = { 181 XGENE_PMU_FORMAT_ATTR(l3c_eventid, "config:0-7"), 182 XGENE_PMU_FORMAT_ATTR(l3c_agentid, "config1:0-9"), 183 NULL, 184 }; 185 186 static struct attribute *iob_pmu_format_attrs[] = { 187 XGENE_PMU_FORMAT_ATTR(iob_eventid, "config:0-7"), 188 XGENE_PMU_FORMAT_ATTR(iob_agentid, "config1:0-63"), 189 NULL, 190 }; 191 192 static struct attribute *mcb_pmu_format_attrs[] = { 193 XGENE_PMU_FORMAT_ATTR(mcb_eventid, "config:0-5"), 194 XGENE_PMU_FORMAT_ATTR(mcb_agentid, "config1:0-9"), 195 NULL, 196 }; 197 198 static struct attribute *mc_pmu_format_attrs[] = { 199 XGENE_PMU_FORMAT_ATTR(mc_eventid, "config:0-28"), 200 NULL, 201 }; 202 203 static const struct attribute_group l3c_pmu_format_attr_group = { 204 .name = "format", 205 .attrs = l3c_pmu_format_attrs, 206 }; 207 208 static const struct attribute_group iob_pmu_format_attr_group = { 209 .name = "format", 210 .attrs = iob_pmu_format_attrs, 211 }; 212 213 static const struct attribute_group mcb_pmu_format_attr_group = { 214 .name = "format", 215 .attrs = mcb_pmu_format_attrs, 216 }; 217 218 static const struct attribute_group mc_pmu_format_attr_group = { 219 .name = "format", 220 .attrs = mc_pmu_format_attrs, 221 }; 222 223 static struct attribute *l3c_pmu_v3_format_attrs[] = { 224 XGENE_PMU_FORMAT_ATTR(l3c_eventid, "config:0-39"), 225 NULL, 226 }; 227 228 static struct attribute *iob_pmu_v3_format_attrs[] = { 229 XGENE_PMU_FORMAT_ATTR(iob_eventid, "config:0-47"), 230 NULL, 231 }; 232 233 static struct attribute *iob_slow_pmu_v3_format_attrs[] = { 234 XGENE_PMU_FORMAT_ATTR(iob_slow_eventid, "config:0-16"), 235 NULL, 236 }; 237 238 static struct attribute *mcb_pmu_v3_format_attrs[] = { 239 XGENE_PMU_FORMAT_ATTR(mcb_eventid, "config:0-35"), 240 NULL, 241 }; 242 243 static struct attribute *mc_pmu_v3_format_attrs[] = { 244 XGENE_PMU_FORMAT_ATTR(mc_eventid, "config:0-44"), 245 NULL, 246 }; 247 248 static const struct attribute_group l3c_pmu_v3_format_attr_group = { 249 .name = "format", 250 .attrs = l3c_pmu_v3_format_attrs, 251 }; 252 253 static const struct attribute_group iob_pmu_v3_format_attr_group = { 254 .name = "format", 255 .attrs = iob_pmu_v3_format_attrs, 256 }; 257 258 static const struct attribute_group iob_slow_pmu_v3_format_attr_group = { 259 .name = "format", 260 .attrs = iob_slow_pmu_v3_format_attrs, 261 }; 262 263 static const struct attribute_group mcb_pmu_v3_format_attr_group = { 264 .name = "format", 265 .attrs = mcb_pmu_v3_format_attrs, 266 }; 267 268 static const struct attribute_group mc_pmu_v3_format_attr_group = { 269 .name = "format", 270 .attrs = mc_pmu_v3_format_attrs, 271 }; 272 273 /* 274 * sysfs event attributes 275 */ 276 static ssize_t xgene_pmu_event_show(struct device *dev, 277 struct device_attribute *attr, char *buf) 278 { 279 struct perf_pmu_events_attr *pmu_attr = 280 container_of(attr, struct perf_pmu_events_attr, attr); 281 282 return sysfs_emit(buf, "config=0x%llx\n", pmu_attr->id); 283 } 284 285 #define XGENE_PMU_EVENT_ATTR(_name, _config) \ 286 PMU_EVENT_ATTR_ID(_name, xgene_pmu_event_show, _config) 287 288 static struct attribute *l3c_pmu_events_attrs[] = { 289 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), 290 XGENE_PMU_EVENT_ATTR(cycle-count-div-64, 0x01), 291 XGENE_PMU_EVENT_ATTR(read-hit, 0x02), 292 XGENE_PMU_EVENT_ATTR(read-miss, 0x03), 293 XGENE_PMU_EVENT_ATTR(write-need-replacement, 0x06), 294 XGENE_PMU_EVENT_ATTR(write-not-need-replacement, 0x07), 295 XGENE_PMU_EVENT_ATTR(tq-full, 0x08), 296 XGENE_PMU_EVENT_ATTR(ackq-full, 0x09), 297 XGENE_PMU_EVENT_ATTR(wdb-full, 0x0a), 298 XGENE_PMU_EVENT_ATTR(bank-fifo-full, 0x0b), 299 XGENE_PMU_EVENT_ATTR(odb-full, 0x0c), 300 XGENE_PMU_EVENT_ATTR(wbq-full, 0x0d), 301 XGENE_PMU_EVENT_ATTR(bank-conflict-fifo-issue, 0x0e), 302 XGENE_PMU_EVENT_ATTR(bank-fifo-issue, 0x0f), 303 NULL, 304 }; 305 306 static struct attribute *iob_pmu_events_attrs[] = { 307 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), 308 XGENE_PMU_EVENT_ATTR(cycle-count-div-64, 0x01), 309 XGENE_PMU_EVENT_ATTR(axi0-read, 0x02), 310 XGENE_PMU_EVENT_ATTR(axi0-read-partial, 0x03), 311 XGENE_PMU_EVENT_ATTR(axi1-read, 0x04), 312 XGENE_PMU_EVENT_ATTR(axi1-read-partial, 0x05), 313 XGENE_PMU_EVENT_ATTR(csw-read-block, 0x06), 314 XGENE_PMU_EVENT_ATTR(csw-read-partial, 0x07), 315 XGENE_PMU_EVENT_ATTR(axi0-write, 0x10), 316 XGENE_PMU_EVENT_ATTR(axi0-write-partial, 0x11), 317 XGENE_PMU_EVENT_ATTR(axi1-write, 0x13), 318 XGENE_PMU_EVENT_ATTR(axi1-write-partial, 0x14), 319 XGENE_PMU_EVENT_ATTR(csw-inbound-dirty, 0x16), 320 NULL, 321 }; 322 323 static struct attribute *mcb_pmu_events_attrs[] = { 324 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), 325 XGENE_PMU_EVENT_ATTR(cycle-count-div-64, 0x01), 326 XGENE_PMU_EVENT_ATTR(csw-read, 0x02), 327 XGENE_PMU_EVENT_ATTR(csw-write-request, 0x03), 328 XGENE_PMU_EVENT_ATTR(mcb-csw-stall, 0x04), 329 XGENE_PMU_EVENT_ATTR(cancel-read-gack, 0x05), 330 NULL, 331 }; 332 333 static struct attribute *mc_pmu_events_attrs[] = { 334 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), 335 XGENE_PMU_EVENT_ATTR(cycle-count-div-64, 0x01), 336 XGENE_PMU_EVENT_ATTR(act-cmd-sent, 0x02), 337 XGENE_PMU_EVENT_ATTR(pre-cmd-sent, 0x03), 338 XGENE_PMU_EVENT_ATTR(rd-cmd-sent, 0x04), 339 XGENE_PMU_EVENT_ATTR(rda-cmd-sent, 0x05), 340 XGENE_PMU_EVENT_ATTR(wr-cmd-sent, 0x06), 341 XGENE_PMU_EVENT_ATTR(wra-cmd-sent, 0x07), 342 XGENE_PMU_EVENT_ATTR(pde-cmd-sent, 0x08), 343 XGENE_PMU_EVENT_ATTR(sre-cmd-sent, 0x09), 344 XGENE_PMU_EVENT_ATTR(prea-cmd-sent, 0x0a), 345 XGENE_PMU_EVENT_ATTR(ref-cmd-sent, 0x0b), 346 XGENE_PMU_EVENT_ATTR(rd-rda-cmd-sent, 0x0c), 347 XGENE_PMU_EVENT_ATTR(wr-wra-cmd-sent, 0x0d), 348 XGENE_PMU_EVENT_ATTR(in-rd-collision, 0x0e), 349 XGENE_PMU_EVENT_ATTR(in-wr-collision, 0x0f), 350 XGENE_PMU_EVENT_ATTR(collision-queue-not-empty, 0x10), 351 XGENE_PMU_EVENT_ATTR(collision-queue-full, 0x11), 352 XGENE_PMU_EVENT_ATTR(mcu-request, 0x12), 353 XGENE_PMU_EVENT_ATTR(mcu-rd-request, 0x13), 354 XGENE_PMU_EVENT_ATTR(mcu-hp-rd-request, 0x14), 355 XGENE_PMU_EVENT_ATTR(mcu-wr-request, 0x15), 356 XGENE_PMU_EVENT_ATTR(mcu-rd-proceed-all, 0x16), 357 XGENE_PMU_EVENT_ATTR(mcu-rd-proceed-cancel, 0x17), 358 XGENE_PMU_EVENT_ATTR(mcu-rd-response, 0x18), 359 XGENE_PMU_EVENT_ATTR(mcu-rd-proceed-speculative-all, 0x19), 360 XGENE_PMU_EVENT_ATTR(mcu-rd-proceed-speculative-cancel, 0x1a), 361 XGENE_PMU_EVENT_ATTR(mcu-wr-proceed-all, 0x1b), 362 XGENE_PMU_EVENT_ATTR(mcu-wr-proceed-cancel, 0x1c), 363 NULL, 364 }; 365 366 static const struct attribute_group l3c_pmu_events_attr_group = { 367 .name = "events", 368 .attrs = l3c_pmu_events_attrs, 369 }; 370 371 static const struct attribute_group iob_pmu_events_attr_group = { 372 .name = "events", 373 .attrs = iob_pmu_events_attrs, 374 }; 375 376 static const struct attribute_group mcb_pmu_events_attr_group = { 377 .name = "events", 378 .attrs = mcb_pmu_events_attrs, 379 }; 380 381 static const struct attribute_group mc_pmu_events_attr_group = { 382 .name = "events", 383 .attrs = mc_pmu_events_attrs, 384 }; 385 386 static struct attribute *l3c_pmu_v3_events_attrs[] = { 387 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), 388 XGENE_PMU_EVENT_ATTR(read-hit, 0x01), 389 XGENE_PMU_EVENT_ATTR(read-miss, 0x02), 390 XGENE_PMU_EVENT_ATTR(index-flush-eviction, 0x03), 391 XGENE_PMU_EVENT_ATTR(write-caused-replacement, 0x04), 392 XGENE_PMU_EVENT_ATTR(write-not-caused-replacement, 0x05), 393 XGENE_PMU_EVENT_ATTR(clean-eviction, 0x06), 394 XGENE_PMU_EVENT_ATTR(dirty-eviction, 0x07), 395 XGENE_PMU_EVENT_ATTR(read, 0x08), 396 XGENE_PMU_EVENT_ATTR(write, 0x09), 397 XGENE_PMU_EVENT_ATTR(request, 0x0a), 398 XGENE_PMU_EVENT_ATTR(tq-bank-conflict-issue-stall, 0x0b), 399 XGENE_PMU_EVENT_ATTR(tq-full, 0x0c), 400 XGENE_PMU_EVENT_ATTR(ackq-full, 0x0d), 401 XGENE_PMU_EVENT_ATTR(wdb-full, 0x0e), 402 XGENE_PMU_EVENT_ATTR(odb-full, 0x10), 403 XGENE_PMU_EVENT_ATTR(wbq-full, 0x11), 404 XGENE_PMU_EVENT_ATTR(input-req-async-fifo-stall, 0x12), 405 XGENE_PMU_EVENT_ATTR(output-req-async-fifo-stall, 0x13), 406 XGENE_PMU_EVENT_ATTR(output-data-async-fifo-stall, 0x14), 407 XGENE_PMU_EVENT_ATTR(total-insertion, 0x15), 408 XGENE_PMU_EVENT_ATTR(sip-insertions-r-set, 0x16), 409 XGENE_PMU_EVENT_ATTR(sip-insertions-r-clear, 0x17), 410 XGENE_PMU_EVENT_ATTR(dip-insertions-r-set, 0x18), 411 XGENE_PMU_EVENT_ATTR(dip-insertions-r-clear, 0x19), 412 XGENE_PMU_EVENT_ATTR(dip-insertions-force-r-set, 0x1a), 413 XGENE_PMU_EVENT_ATTR(egression, 0x1b), 414 XGENE_PMU_EVENT_ATTR(replacement, 0x1c), 415 XGENE_PMU_EVENT_ATTR(old-replacement, 0x1d), 416 XGENE_PMU_EVENT_ATTR(young-replacement, 0x1e), 417 XGENE_PMU_EVENT_ATTR(r-set-replacement, 0x1f), 418 XGENE_PMU_EVENT_ATTR(r-clear-replacement, 0x20), 419 XGENE_PMU_EVENT_ATTR(old-r-replacement, 0x21), 420 XGENE_PMU_EVENT_ATTR(old-nr-replacement, 0x22), 421 XGENE_PMU_EVENT_ATTR(young-r-replacement, 0x23), 422 XGENE_PMU_EVENT_ATTR(young-nr-replacement, 0x24), 423 XGENE_PMU_EVENT_ATTR(bloomfilter-clearing, 0x25), 424 XGENE_PMU_EVENT_ATTR(generation-flip, 0x26), 425 XGENE_PMU_EVENT_ATTR(vcc-droop-detected, 0x27), 426 NULL, 427 }; 428 429 static struct attribute *iob_fast_pmu_v3_events_attrs[] = { 430 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), 431 XGENE_PMU_EVENT_ATTR(pa-req-buf-alloc-all, 0x01), 432 XGENE_PMU_EVENT_ATTR(pa-req-buf-alloc-rd, 0x02), 433 XGENE_PMU_EVENT_ATTR(pa-req-buf-alloc-wr, 0x03), 434 XGENE_PMU_EVENT_ATTR(pa-all-cp-req, 0x04), 435 XGENE_PMU_EVENT_ATTR(pa-cp-blk-req, 0x05), 436 XGENE_PMU_EVENT_ATTR(pa-cp-ptl-req, 0x06), 437 XGENE_PMU_EVENT_ATTR(pa-cp-rd-req, 0x07), 438 XGENE_PMU_EVENT_ATTR(pa-cp-wr-req, 0x08), 439 XGENE_PMU_EVENT_ATTR(ba-all-req, 0x09), 440 XGENE_PMU_EVENT_ATTR(ba-rd-req, 0x0a), 441 XGENE_PMU_EVENT_ATTR(ba-wr-req, 0x0b), 442 XGENE_PMU_EVENT_ATTR(pa-rd-shared-req-issued, 0x10), 443 XGENE_PMU_EVENT_ATTR(pa-rd-exclusive-req-issued, 0x11), 444 XGENE_PMU_EVENT_ATTR(pa-wr-invalidate-req-issued-stashable, 0x12), 445 XGENE_PMU_EVENT_ATTR(pa-wr-invalidate-req-issued-nonstashable, 0x13), 446 XGENE_PMU_EVENT_ATTR(pa-wr-back-req-issued-stashable, 0x14), 447 XGENE_PMU_EVENT_ATTR(pa-wr-back-req-issued-nonstashable, 0x15), 448 XGENE_PMU_EVENT_ATTR(pa-ptl-wr-req, 0x16), 449 XGENE_PMU_EVENT_ATTR(pa-ptl-rd-req, 0x17), 450 XGENE_PMU_EVENT_ATTR(pa-wr-back-clean-data, 0x18), 451 XGENE_PMU_EVENT_ATTR(pa-wr-back-cancelled-on-SS, 0x1b), 452 XGENE_PMU_EVENT_ATTR(pa-barrier-occurrence, 0x1c), 453 XGENE_PMU_EVENT_ATTR(pa-barrier-cycles, 0x1d), 454 XGENE_PMU_EVENT_ATTR(pa-total-cp-snoops, 0x20), 455 XGENE_PMU_EVENT_ATTR(pa-rd-shared-snoop, 0x21), 456 XGENE_PMU_EVENT_ATTR(pa-rd-shared-snoop-hit, 0x22), 457 XGENE_PMU_EVENT_ATTR(pa-rd-exclusive-snoop, 0x23), 458 XGENE_PMU_EVENT_ATTR(pa-rd-exclusive-snoop-hit, 0x24), 459 XGENE_PMU_EVENT_ATTR(pa-rd-wr-invalid-snoop, 0x25), 460 XGENE_PMU_EVENT_ATTR(pa-rd-wr-invalid-snoop-hit, 0x26), 461 XGENE_PMU_EVENT_ATTR(pa-req-buffer-full, 0x28), 462 XGENE_PMU_EVENT_ATTR(cswlf-outbound-req-fifo-full, 0x29), 463 XGENE_PMU_EVENT_ATTR(cswlf-inbound-snoop-fifo-backpressure, 0x2a), 464 XGENE_PMU_EVENT_ATTR(cswlf-outbound-lack-fifo-full, 0x2b), 465 XGENE_PMU_EVENT_ATTR(cswlf-inbound-gack-fifo-backpressure, 0x2c), 466 XGENE_PMU_EVENT_ATTR(cswlf-outbound-data-fifo-full, 0x2d), 467 XGENE_PMU_EVENT_ATTR(cswlf-inbound-data-fifo-backpressure, 0x2e), 468 XGENE_PMU_EVENT_ATTR(cswlf-inbound-req-backpressure, 0x2f), 469 NULL, 470 }; 471 472 static struct attribute *iob_slow_pmu_v3_events_attrs[] = { 473 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), 474 XGENE_PMU_EVENT_ATTR(pa-axi0-rd-req, 0x01), 475 XGENE_PMU_EVENT_ATTR(pa-axi0-wr-req, 0x02), 476 XGENE_PMU_EVENT_ATTR(pa-axi1-rd-req, 0x03), 477 XGENE_PMU_EVENT_ATTR(pa-axi1-wr-req, 0x04), 478 XGENE_PMU_EVENT_ATTR(ba-all-axi-req, 0x07), 479 XGENE_PMU_EVENT_ATTR(ba-axi-rd-req, 0x08), 480 XGENE_PMU_EVENT_ATTR(ba-axi-wr-req, 0x09), 481 XGENE_PMU_EVENT_ATTR(ba-free-list-empty, 0x10), 482 NULL, 483 }; 484 485 static struct attribute *mcb_pmu_v3_events_attrs[] = { 486 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), 487 XGENE_PMU_EVENT_ATTR(req-receive, 0x01), 488 XGENE_PMU_EVENT_ATTR(rd-req-recv, 0x02), 489 XGENE_PMU_EVENT_ATTR(rd-req-recv-2, 0x03), 490 XGENE_PMU_EVENT_ATTR(wr-req-recv, 0x04), 491 XGENE_PMU_EVENT_ATTR(wr-req-recv-2, 0x05), 492 XGENE_PMU_EVENT_ATTR(rd-req-sent-to-mcu, 0x06), 493 XGENE_PMU_EVENT_ATTR(rd-req-sent-to-mcu-2, 0x07), 494 XGENE_PMU_EVENT_ATTR(rd-req-sent-to-spec-mcu, 0x08), 495 XGENE_PMU_EVENT_ATTR(rd-req-sent-to-spec-mcu-2, 0x09), 496 XGENE_PMU_EVENT_ATTR(glbl-ack-recv-for-rd-sent-to-spec-mcu, 0x0a), 497 XGENE_PMU_EVENT_ATTR(glbl-ack-go-recv-for-rd-sent-to-spec-mcu, 0x0b), 498 XGENE_PMU_EVENT_ATTR(glbl-ack-nogo-recv-for-rd-sent-to-spec-mcu, 0x0c), 499 XGENE_PMU_EVENT_ATTR(glbl-ack-go-recv-any-rd-req, 0x0d), 500 XGENE_PMU_EVENT_ATTR(glbl-ack-go-recv-any-rd-req-2, 0x0e), 501 XGENE_PMU_EVENT_ATTR(wr-req-sent-to-mcu, 0x0f), 502 XGENE_PMU_EVENT_ATTR(gack-recv, 0x10), 503 XGENE_PMU_EVENT_ATTR(rd-gack-recv, 0x11), 504 XGENE_PMU_EVENT_ATTR(wr-gack-recv, 0x12), 505 XGENE_PMU_EVENT_ATTR(cancel-rd-gack, 0x13), 506 XGENE_PMU_EVENT_ATTR(cancel-wr-gack, 0x14), 507 XGENE_PMU_EVENT_ATTR(mcb-csw-req-stall, 0x15), 508 XGENE_PMU_EVENT_ATTR(mcu-req-intf-blocked, 0x16), 509 XGENE_PMU_EVENT_ATTR(mcb-mcu-rd-intf-stall, 0x17), 510 XGENE_PMU_EVENT_ATTR(csw-rd-intf-blocked, 0x18), 511 XGENE_PMU_EVENT_ATTR(csw-local-ack-intf-blocked, 0x19), 512 XGENE_PMU_EVENT_ATTR(mcu-req-table-full, 0x1a), 513 XGENE_PMU_EVENT_ATTR(mcu-stat-table-full, 0x1b), 514 XGENE_PMU_EVENT_ATTR(mcu-wr-table-full, 0x1c), 515 XGENE_PMU_EVENT_ATTR(mcu-rdreceipt-resp, 0x1d), 516 XGENE_PMU_EVENT_ATTR(mcu-wrcomplete-resp, 0x1e), 517 XGENE_PMU_EVENT_ATTR(mcu-retryack-resp, 0x1f), 518 XGENE_PMU_EVENT_ATTR(mcu-pcrdgrant-resp, 0x20), 519 XGENE_PMU_EVENT_ATTR(mcu-req-from-lastload, 0x21), 520 XGENE_PMU_EVENT_ATTR(mcu-req-from-bypass, 0x22), 521 XGENE_PMU_EVENT_ATTR(volt-droop-detect, 0x23), 522 NULL, 523 }; 524 525 static struct attribute *mc_pmu_v3_events_attrs[] = { 526 XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), 527 XGENE_PMU_EVENT_ATTR(act-sent, 0x01), 528 XGENE_PMU_EVENT_ATTR(pre-sent, 0x02), 529 XGENE_PMU_EVENT_ATTR(rd-sent, 0x03), 530 XGENE_PMU_EVENT_ATTR(rda-sent, 0x04), 531 XGENE_PMU_EVENT_ATTR(wr-sent, 0x05), 532 XGENE_PMU_EVENT_ATTR(wra-sent, 0x06), 533 XGENE_PMU_EVENT_ATTR(pd-entry-vld, 0x07), 534 XGENE_PMU_EVENT_ATTR(sref-entry-vld, 0x08), 535 XGENE_PMU_EVENT_ATTR(prea-sent, 0x09), 536 XGENE_PMU_EVENT_ATTR(ref-sent, 0x0a), 537 XGENE_PMU_EVENT_ATTR(rd-rda-sent, 0x0b), 538 XGENE_PMU_EVENT_ATTR(wr-wra-sent, 0x0c), 539 XGENE_PMU_EVENT_ATTR(raw-hazard, 0x0d), 540 XGENE_PMU_EVENT_ATTR(war-hazard, 0x0e), 541 XGENE_PMU_EVENT_ATTR(waw-hazard, 0x0f), 542 XGENE_PMU_EVENT_ATTR(rar-hazard, 0x10), 543 XGENE_PMU_EVENT_ATTR(raw-war-waw-hazard, 0x11), 544 XGENE_PMU_EVENT_ATTR(hprd-lprd-wr-req-vld, 0x12), 545 XGENE_PMU_EVENT_ATTR(lprd-req-vld, 0x13), 546 XGENE_PMU_EVENT_ATTR(hprd-req-vld, 0x14), 547 XGENE_PMU_EVENT_ATTR(hprd-lprd-req-vld, 0x15), 548 XGENE_PMU_EVENT_ATTR(wr-req-vld, 0x16), 549 XGENE_PMU_EVENT_ATTR(partial-wr-req-vld, 0x17), 550 XGENE_PMU_EVENT_ATTR(rd-retry, 0x18), 551 XGENE_PMU_EVENT_ATTR(wr-retry, 0x19), 552 XGENE_PMU_EVENT_ATTR(retry-gnt, 0x1a), 553 XGENE_PMU_EVENT_ATTR(rank-change, 0x1b), 554 XGENE_PMU_EVENT_ATTR(dir-change, 0x1c), 555 XGENE_PMU_EVENT_ATTR(rank-dir-change, 0x1d), 556 XGENE_PMU_EVENT_ATTR(rank-active, 0x1e), 557 XGENE_PMU_EVENT_ATTR(rank-idle, 0x1f), 558 XGENE_PMU_EVENT_ATTR(rank-pd, 0x20), 559 XGENE_PMU_EVENT_ATTR(rank-sref, 0x21), 560 XGENE_PMU_EVENT_ATTR(queue-fill-gt-thresh, 0x22), 561 XGENE_PMU_EVENT_ATTR(queue-rds-gt-thresh, 0x23), 562 XGENE_PMU_EVENT_ATTR(queue-wrs-gt-thresh, 0x24), 563 XGENE_PMU_EVENT_ATTR(phy-updt-complt, 0x25), 564 XGENE_PMU_EVENT_ATTR(tz-fail, 0x26), 565 XGENE_PMU_EVENT_ATTR(dram-errc, 0x27), 566 XGENE_PMU_EVENT_ATTR(dram-errd, 0x28), 567 XGENE_PMU_EVENT_ATTR(rd-enq, 0x29), 568 XGENE_PMU_EVENT_ATTR(wr-enq, 0x2a), 569 XGENE_PMU_EVENT_ATTR(tmac-limit-reached, 0x2b), 570 XGENE_PMU_EVENT_ATTR(tmaw-tracker-full, 0x2c), 571 NULL, 572 }; 573 574 static const struct attribute_group l3c_pmu_v3_events_attr_group = { 575 .name = "events", 576 .attrs = l3c_pmu_v3_events_attrs, 577 }; 578 579 static const struct attribute_group iob_fast_pmu_v3_events_attr_group = { 580 .name = "events", 581 .attrs = iob_fast_pmu_v3_events_attrs, 582 }; 583 584 static const struct attribute_group iob_slow_pmu_v3_events_attr_group = { 585 .name = "events", 586 .attrs = iob_slow_pmu_v3_events_attrs, 587 }; 588 589 static const struct attribute_group mcb_pmu_v3_events_attr_group = { 590 .name = "events", 591 .attrs = mcb_pmu_v3_events_attrs, 592 }; 593 594 static const struct attribute_group mc_pmu_v3_events_attr_group = { 595 .name = "events", 596 .attrs = mc_pmu_v3_events_attrs, 597 }; 598 599 /* 600 * sysfs cpumask attributes 601 */ 602 static ssize_t cpumask_show(struct device *dev, 603 struct device_attribute *attr, char *buf) 604 { 605 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(dev_get_drvdata(dev)); 606 607 return cpumap_print_to_pagebuf(true, buf, &pmu_dev->parent->cpu); 608 } 609 610 static DEVICE_ATTR_RO(cpumask); 611 612 static struct attribute *xgene_pmu_cpumask_attrs[] = { 613 &dev_attr_cpumask.attr, 614 NULL, 615 }; 616 617 static const struct attribute_group pmu_cpumask_attr_group = { 618 .attrs = xgene_pmu_cpumask_attrs, 619 }; 620 621 /* 622 * Per PMU device attribute groups of PMU v1 and v2 623 */ 624 static const struct attribute_group *l3c_pmu_attr_groups[] = { 625 &l3c_pmu_format_attr_group, 626 &pmu_cpumask_attr_group, 627 &l3c_pmu_events_attr_group, 628 NULL 629 }; 630 631 static const struct attribute_group *iob_pmu_attr_groups[] = { 632 &iob_pmu_format_attr_group, 633 &pmu_cpumask_attr_group, 634 &iob_pmu_events_attr_group, 635 NULL 636 }; 637 638 static const struct attribute_group *mcb_pmu_attr_groups[] = { 639 &mcb_pmu_format_attr_group, 640 &pmu_cpumask_attr_group, 641 &mcb_pmu_events_attr_group, 642 NULL 643 }; 644 645 static const struct attribute_group *mc_pmu_attr_groups[] = { 646 &mc_pmu_format_attr_group, 647 &pmu_cpumask_attr_group, 648 &mc_pmu_events_attr_group, 649 NULL 650 }; 651 652 /* 653 * Per PMU device attribute groups of PMU v3 654 */ 655 static const struct attribute_group *l3c_pmu_v3_attr_groups[] = { 656 &l3c_pmu_v3_format_attr_group, 657 &pmu_cpumask_attr_group, 658 &l3c_pmu_v3_events_attr_group, 659 NULL 660 }; 661 662 static const struct attribute_group *iob_fast_pmu_v3_attr_groups[] = { 663 &iob_pmu_v3_format_attr_group, 664 &pmu_cpumask_attr_group, 665 &iob_fast_pmu_v3_events_attr_group, 666 NULL 667 }; 668 669 static const struct attribute_group *iob_slow_pmu_v3_attr_groups[] = { 670 &iob_slow_pmu_v3_format_attr_group, 671 &pmu_cpumask_attr_group, 672 &iob_slow_pmu_v3_events_attr_group, 673 NULL 674 }; 675 676 static const struct attribute_group *mcb_pmu_v3_attr_groups[] = { 677 &mcb_pmu_v3_format_attr_group, 678 &pmu_cpumask_attr_group, 679 &mcb_pmu_v3_events_attr_group, 680 NULL 681 }; 682 683 static const struct attribute_group *mc_pmu_v3_attr_groups[] = { 684 &mc_pmu_v3_format_attr_group, 685 &pmu_cpumask_attr_group, 686 &mc_pmu_v3_events_attr_group, 687 NULL 688 }; 689 690 static int get_next_avail_cntr(struct xgene_pmu_dev *pmu_dev) 691 { 692 int cntr; 693 694 cntr = find_first_zero_bit(pmu_dev->cntr_assign_mask, 695 pmu_dev->max_counters); 696 if (cntr == pmu_dev->max_counters) 697 return -ENOSPC; 698 set_bit(cntr, pmu_dev->cntr_assign_mask); 699 700 return cntr; 701 } 702 703 static void clear_avail_cntr(struct xgene_pmu_dev *pmu_dev, int cntr) 704 { 705 clear_bit(cntr, pmu_dev->cntr_assign_mask); 706 } 707 708 static inline void xgene_pmu_mask_int(struct xgene_pmu *xgene_pmu) 709 { 710 writel(PCPPMU_INTENMASK, xgene_pmu->pcppmu_csr + PCPPMU_INTMASK_REG); 711 } 712 713 static inline void xgene_pmu_v3_mask_int(struct xgene_pmu *xgene_pmu) 714 { 715 writel(PCPPMU_V3_INTENMASK, xgene_pmu->pcppmu_csr + PCPPMU_INTMASK_REG); 716 } 717 718 static inline void xgene_pmu_unmask_int(struct xgene_pmu *xgene_pmu) 719 { 720 writel(PCPPMU_INTCLRMASK, xgene_pmu->pcppmu_csr + PCPPMU_INTMASK_REG); 721 } 722 723 static inline void xgene_pmu_v3_unmask_int(struct xgene_pmu *xgene_pmu) 724 { 725 writel(PCPPMU_V3_INTCLRMASK, 726 xgene_pmu->pcppmu_csr + PCPPMU_INTMASK_REG); 727 } 728 729 static inline u64 xgene_pmu_read_counter32(struct xgene_pmu_dev *pmu_dev, 730 int idx) 731 { 732 return readl(pmu_dev->inf->csr + PMU_PMEVCNTR0 + (4 * idx)); 733 } 734 735 static inline u64 xgene_pmu_read_counter64(struct xgene_pmu_dev *pmu_dev, 736 int idx) 737 { 738 u32 lo, hi; 739 740 /* 741 * v3 has 64-bit counter registers composed by 2 32-bit registers 742 * This can be a problem if the counter increases and carries 743 * out of bit [31] between 2 reads. The extra reads would help 744 * to prevent this issue. 745 */ 746 do { 747 hi = xgene_pmu_read_counter32(pmu_dev, 2 * idx + 1); 748 lo = xgene_pmu_read_counter32(pmu_dev, 2 * idx); 749 } while (hi != xgene_pmu_read_counter32(pmu_dev, 2 * idx + 1)); 750 751 return (((u64)hi << 32) | lo); 752 } 753 754 static inline void 755 xgene_pmu_write_counter32(struct xgene_pmu_dev *pmu_dev, int idx, u64 val) 756 { 757 writel(val, pmu_dev->inf->csr + PMU_PMEVCNTR0 + (4 * idx)); 758 } 759 760 static inline void 761 xgene_pmu_write_counter64(struct xgene_pmu_dev *pmu_dev, int idx, u64 val) 762 { 763 u32 cnt_lo, cnt_hi; 764 765 cnt_hi = upper_32_bits(val); 766 cnt_lo = lower_32_bits(val); 767 768 /* v3 has 64-bit counter registers composed by 2 32-bit registers */ 769 xgene_pmu_write_counter32(pmu_dev, 2 * idx, cnt_lo); 770 xgene_pmu_write_counter32(pmu_dev, 2 * idx + 1, cnt_hi); 771 } 772 773 static inline void 774 xgene_pmu_write_evttype(struct xgene_pmu_dev *pmu_dev, int idx, u32 val) 775 { 776 writel(val, pmu_dev->inf->csr + PMU_PMEVTYPER0 + (4 * idx)); 777 } 778 779 static inline void 780 xgene_pmu_write_agentmsk(struct xgene_pmu_dev *pmu_dev, u32 val) 781 { 782 writel(val, pmu_dev->inf->csr + PMU_PMAMR0); 783 } 784 785 static inline void 786 xgene_pmu_v3_write_agentmsk(struct xgene_pmu_dev *pmu_dev, u32 val) { } 787 788 static inline void 789 xgene_pmu_write_agent1msk(struct xgene_pmu_dev *pmu_dev, u32 val) 790 { 791 writel(val, pmu_dev->inf->csr + PMU_PMAMR1); 792 } 793 794 static inline void 795 xgene_pmu_v3_write_agent1msk(struct xgene_pmu_dev *pmu_dev, u32 val) { } 796 797 static inline void 798 xgene_pmu_enable_counter(struct xgene_pmu_dev *pmu_dev, int idx) 799 { 800 u32 val; 801 802 val = readl(pmu_dev->inf->csr + PMU_PMCNTENSET); 803 val |= 1 << idx; 804 writel(val, pmu_dev->inf->csr + PMU_PMCNTENSET); 805 } 806 807 static inline void 808 xgene_pmu_disable_counter(struct xgene_pmu_dev *pmu_dev, int idx) 809 { 810 u32 val; 811 812 val = readl(pmu_dev->inf->csr + PMU_PMCNTENCLR); 813 val |= 1 << idx; 814 writel(val, pmu_dev->inf->csr + PMU_PMCNTENCLR); 815 } 816 817 static inline void 818 xgene_pmu_enable_counter_int(struct xgene_pmu_dev *pmu_dev, int idx) 819 { 820 u32 val; 821 822 val = readl(pmu_dev->inf->csr + PMU_PMINTENSET); 823 val |= 1 << idx; 824 writel(val, pmu_dev->inf->csr + PMU_PMINTENSET); 825 } 826 827 static inline void 828 xgene_pmu_disable_counter_int(struct xgene_pmu_dev *pmu_dev, int idx) 829 { 830 u32 val; 831 832 val = readl(pmu_dev->inf->csr + PMU_PMINTENCLR); 833 val |= 1 << idx; 834 writel(val, pmu_dev->inf->csr + PMU_PMINTENCLR); 835 } 836 837 static inline void xgene_pmu_reset_counters(struct xgene_pmu_dev *pmu_dev) 838 { 839 u32 val; 840 841 val = readl(pmu_dev->inf->csr + PMU_PMCR); 842 val |= PMU_PMCR_P; 843 writel(val, pmu_dev->inf->csr + PMU_PMCR); 844 } 845 846 static inline void xgene_pmu_start_counters(struct xgene_pmu_dev *pmu_dev) 847 { 848 u32 val; 849 850 val = readl(pmu_dev->inf->csr + PMU_PMCR); 851 val |= PMU_PMCR_E; 852 writel(val, pmu_dev->inf->csr + PMU_PMCR); 853 } 854 855 static inline void xgene_pmu_stop_counters(struct xgene_pmu_dev *pmu_dev) 856 { 857 u32 val; 858 859 val = readl(pmu_dev->inf->csr + PMU_PMCR); 860 val &= ~PMU_PMCR_E; 861 writel(val, pmu_dev->inf->csr + PMU_PMCR); 862 } 863 864 static void xgene_perf_pmu_enable(struct pmu *pmu) 865 { 866 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(pmu); 867 struct xgene_pmu *xgene_pmu = pmu_dev->parent; 868 bool enabled = !bitmap_empty(pmu_dev->cntr_assign_mask, 869 pmu_dev->max_counters); 870 871 if (!enabled) 872 return; 873 874 xgene_pmu->ops->start_counters(pmu_dev); 875 } 876 877 static void xgene_perf_pmu_disable(struct pmu *pmu) 878 { 879 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(pmu); 880 struct xgene_pmu *xgene_pmu = pmu_dev->parent; 881 882 xgene_pmu->ops->stop_counters(pmu_dev); 883 } 884 885 static int xgene_perf_event_init(struct perf_event *event) 886 { 887 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu); 888 struct hw_perf_event *hw = &event->hw; 889 struct perf_event *sibling; 890 891 /* Test the event attr type check for PMU enumeration */ 892 if (event->attr.type != event->pmu->type) 893 return -ENOENT; 894 895 /* 896 * SOC PMU counters are shared across all cores. 897 * Therefore, it does not support per-process mode. 898 * Also, it does not support event sampling mode. 899 */ 900 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 901 return -EINVAL; 902 903 if (event->cpu < 0) 904 return -EINVAL; 905 /* 906 * Many perf core operations (eg. events rotation) operate on a 907 * single CPU context. This is obvious for CPU PMUs, where one 908 * expects the same sets of events being observed on all CPUs, 909 * but can lead to issues for off-core PMUs, where each 910 * event could be theoretically assigned to a different CPU. To 911 * mitigate this, we enforce CPU assignment to one, selected 912 * processor (the one described in the "cpumask" attribute). 913 */ 914 event->cpu = cpumask_first(&pmu_dev->parent->cpu); 915 916 hw->config = event->attr.config; 917 /* 918 * Each bit of the config1 field represents an agent from which the 919 * request of the event come. The event is counted only if it's caused 920 * by a request of an agent has the bit cleared. 921 * By default, the event is counted for all agents. 922 */ 923 hw->config_base = event->attr.config1; 924 925 /* 926 * We must NOT create groups containing mixed PMUs, although software 927 * events are acceptable 928 */ 929 if (event->group_leader->pmu != event->pmu && 930 !is_software_event(event->group_leader)) 931 return -EINVAL; 932 933 for_each_sibling_event(sibling, event->group_leader) { 934 if (sibling->pmu != event->pmu && 935 !is_software_event(sibling)) 936 return -EINVAL; 937 } 938 939 return 0; 940 } 941 942 static void xgene_perf_enable_event(struct perf_event *event) 943 { 944 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu); 945 struct xgene_pmu *xgene_pmu = pmu_dev->parent; 946 947 xgene_pmu->ops->write_evttype(pmu_dev, GET_CNTR(event), 948 GET_EVENTID(event)); 949 xgene_pmu->ops->write_agentmsk(pmu_dev, ~((u32)GET_AGENTID(event))); 950 if (pmu_dev->inf->type == PMU_TYPE_IOB) 951 xgene_pmu->ops->write_agent1msk(pmu_dev, 952 ~((u32)GET_AGENT1ID(event))); 953 954 xgene_pmu->ops->enable_counter(pmu_dev, GET_CNTR(event)); 955 xgene_pmu->ops->enable_counter_int(pmu_dev, GET_CNTR(event)); 956 } 957 958 static void xgene_perf_disable_event(struct perf_event *event) 959 { 960 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu); 961 struct xgene_pmu *xgene_pmu = pmu_dev->parent; 962 963 xgene_pmu->ops->disable_counter(pmu_dev, GET_CNTR(event)); 964 xgene_pmu->ops->disable_counter_int(pmu_dev, GET_CNTR(event)); 965 } 966 967 static void xgene_perf_event_set_period(struct perf_event *event) 968 { 969 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu); 970 struct xgene_pmu *xgene_pmu = pmu_dev->parent; 971 struct hw_perf_event *hw = &event->hw; 972 /* 973 * For 32 bit counter, it has a period of 2^32. To account for the 974 * possibility of extreme interrupt latency we program for a period of 975 * half that. Hopefully, we can handle the interrupt before another 2^31 976 * events occur and the counter overtakes its previous value. 977 * For 64 bit counter, we don't expect it overflow. 978 */ 979 u64 val = 1ULL << 31; 980 981 local64_set(&hw->prev_count, val); 982 xgene_pmu->ops->write_counter(pmu_dev, hw->idx, val); 983 } 984 985 static void xgene_perf_event_update(struct perf_event *event) 986 { 987 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu); 988 struct xgene_pmu *xgene_pmu = pmu_dev->parent; 989 struct hw_perf_event *hw = &event->hw; 990 u64 delta, prev_raw_count, new_raw_count; 991 992 again: 993 prev_raw_count = local64_read(&hw->prev_count); 994 new_raw_count = xgene_pmu->ops->read_counter(pmu_dev, GET_CNTR(event)); 995 996 if (local64_cmpxchg(&hw->prev_count, prev_raw_count, 997 new_raw_count) != prev_raw_count) 998 goto again; 999 1000 delta = (new_raw_count - prev_raw_count) & pmu_dev->max_period; 1001 1002 local64_add(delta, &event->count); 1003 } 1004 1005 static void xgene_perf_read(struct perf_event *event) 1006 { 1007 xgene_perf_event_update(event); 1008 } 1009 1010 static void xgene_perf_start(struct perf_event *event, int flags) 1011 { 1012 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu); 1013 struct xgene_pmu *xgene_pmu = pmu_dev->parent; 1014 struct hw_perf_event *hw = &event->hw; 1015 1016 if (WARN_ON_ONCE(!(hw->state & PERF_HES_STOPPED))) 1017 return; 1018 1019 WARN_ON_ONCE(!(hw->state & PERF_HES_UPTODATE)); 1020 hw->state = 0; 1021 1022 xgene_perf_event_set_period(event); 1023 1024 if (flags & PERF_EF_RELOAD) { 1025 u64 prev_raw_count = local64_read(&hw->prev_count); 1026 1027 xgene_pmu->ops->write_counter(pmu_dev, GET_CNTR(event), 1028 prev_raw_count); 1029 } 1030 1031 xgene_perf_enable_event(event); 1032 perf_event_update_userpage(event); 1033 } 1034 1035 static void xgene_perf_stop(struct perf_event *event, int flags) 1036 { 1037 struct hw_perf_event *hw = &event->hw; 1038 1039 if (hw->state & PERF_HES_UPTODATE) 1040 return; 1041 1042 xgene_perf_disable_event(event); 1043 WARN_ON_ONCE(hw->state & PERF_HES_STOPPED); 1044 hw->state |= PERF_HES_STOPPED; 1045 1046 if (hw->state & PERF_HES_UPTODATE) 1047 return; 1048 1049 xgene_perf_read(event); 1050 hw->state |= PERF_HES_UPTODATE; 1051 } 1052 1053 static int xgene_perf_add(struct perf_event *event, int flags) 1054 { 1055 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu); 1056 struct hw_perf_event *hw = &event->hw; 1057 1058 hw->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1059 1060 /* Allocate an event counter */ 1061 hw->idx = get_next_avail_cntr(pmu_dev); 1062 if (hw->idx < 0) 1063 return -EAGAIN; 1064 1065 /* Update counter event pointer for Interrupt handler */ 1066 pmu_dev->pmu_counter_event[hw->idx] = event; 1067 1068 if (flags & PERF_EF_START) 1069 xgene_perf_start(event, PERF_EF_RELOAD); 1070 1071 return 0; 1072 } 1073 1074 static void xgene_perf_del(struct perf_event *event, int flags) 1075 { 1076 struct xgene_pmu_dev *pmu_dev = to_pmu_dev(event->pmu); 1077 struct hw_perf_event *hw = &event->hw; 1078 1079 xgene_perf_stop(event, PERF_EF_UPDATE); 1080 1081 /* clear the assigned counter */ 1082 clear_avail_cntr(pmu_dev, GET_CNTR(event)); 1083 1084 perf_event_update_userpage(event); 1085 pmu_dev->pmu_counter_event[hw->idx] = NULL; 1086 } 1087 1088 static int xgene_init_perf(struct xgene_pmu_dev *pmu_dev, char *name) 1089 { 1090 struct xgene_pmu *xgene_pmu; 1091 1092 if (pmu_dev->parent->version == PCP_PMU_V3) 1093 pmu_dev->max_period = PMU_V3_CNT_MAX_PERIOD; 1094 else 1095 pmu_dev->max_period = PMU_CNT_MAX_PERIOD; 1096 /* First version PMU supports only single event counter */ 1097 xgene_pmu = pmu_dev->parent; 1098 if (xgene_pmu->version == PCP_PMU_V1) 1099 pmu_dev->max_counters = 1; 1100 else 1101 pmu_dev->max_counters = PMU_MAX_COUNTERS; 1102 1103 /* Perf driver registration */ 1104 pmu_dev->pmu = (struct pmu) { 1105 .attr_groups = pmu_dev->attr_groups, 1106 .task_ctx_nr = perf_invalid_context, 1107 .pmu_enable = xgene_perf_pmu_enable, 1108 .pmu_disable = xgene_perf_pmu_disable, 1109 .event_init = xgene_perf_event_init, 1110 .add = xgene_perf_add, 1111 .del = xgene_perf_del, 1112 .start = xgene_perf_start, 1113 .stop = xgene_perf_stop, 1114 .read = xgene_perf_read, 1115 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 1116 }; 1117 1118 /* Hardware counter init */ 1119 xgene_pmu->ops->stop_counters(pmu_dev); 1120 xgene_pmu->ops->reset_counters(pmu_dev); 1121 1122 return perf_pmu_register(&pmu_dev->pmu, name, -1); 1123 } 1124 1125 static int 1126 xgene_pmu_dev_add(struct xgene_pmu *xgene_pmu, struct xgene_pmu_dev_ctx *ctx) 1127 { 1128 struct device *dev = xgene_pmu->dev; 1129 struct xgene_pmu_dev *pmu; 1130 1131 pmu = devm_kzalloc(dev, sizeof(*pmu), GFP_KERNEL); 1132 if (!pmu) 1133 return -ENOMEM; 1134 pmu->parent = xgene_pmu; 1135 pmu->inf = &ctx->inf; 1136 ctx->pmu_dev = pmu; 1137 1138 switch (pmu->inf->type) { 1139 case PMU_TYPE_L3C: 1140 if (!(xgene_pmu->l3c_active_mask & pmu->inf->enable_mask)) 1141 return -ENODEV; 1142 if (xgene_pmu->version == PCP_PMU_V3) 1143 pmu->attr_groups = l3c_pmu_v3_attr_groups; 1144 else 1145 pmu->attr_groups = l3c_pmu_attr_groups; 1146 break; 1147 case PMU_TYPE_IOB: 1148 if (xgene_pmu->version == PCP_PMU_V3) 1149 pmu->attr_groups = iob_fast_pmu_v3_attr_groups; 1150 else 1151 pmu->attr_groups = iob_pmu_attr_groups; 1152 break; 1153 case PMU_TYPE_IOB_SLOW: 1154 if (xgene_pmu->version == PCP_PMU_V3) 1155 pmu->attr_groups = iob_slow_pmu_v3_attr_groups; 1156 break; 1157 case PMU_TYPE_MCB: 1158 if (!(xgene_pmu->mcb_active_mask & pmu->inf->enable_mask)) 1159 return -ENODEV; 1160 if (xgene_pmu->version == PCP_PMU_V3) 1161 pmu->attr_groups = mcb_pmu_v3_attr_groups; 1162 else 1163 pmu->attr_groups = mcb_pmu_attr_groups; 1164 break; 1165 case PMU_TYPE_MC: 1166 if (!(xgene_pmu->mc_active_mask & pmu->inf->enable_mask)) 1167 return -ENODEV; 1168 if (xgene_pmu->version == PCP_PMU_V3) 1169 pmu->attr_groups = mc_pmu_v3_attr_groups; 1170 else 1171 pmu->attr_groups = mc_pmu_attr_groups; 1172 break; 1173 default: 1174 return -EINVAL; 1175 } 1176 1177 if (xgene_init_perf(pmu, ctx->name)) { 1178 dev_err(dev, "%s PMU: Failed to init perf driver\n", ctx->name); 1179 return -ENODEV; 1180 } 1181 1182 dev_info(dev, "%s PMU registered\n", ctx->name); 1183 1184 return 0; 1185 } 1186 1187 static void _xgene_pmu_isr(int irq, struct xgene_pmu_dev *pmu_dev) 1188 { 1189 struct xgene_pmu *xgene_pmu = pmu_dev->parent; 1190 void __iomem *csr = pmu_dev->inf->csr; 1191 u32 pmovsr; 1192 int idx; 1193 1194 xgene_pmu->ops->stop_counters(pmu_dev); 1195 1196 if (xgene_pmu->version == PCP_PMU_V3) 1197 pmovsr = readl(csr + PMU_PMOVSSET) & PMU_OVERFLOW_MASK; 1198 else 1199 pmovsr = readl(csr + PMU_PMOVSR) & PMU_OVERFLOW_MASK; 1200 1201 if (!pmovsr) 1202 goto out; 1203 1204 /* Clear interrupt flag */ 1205 if (xgene_pmu->version == PCP_PMU_V1) 1206 writel(0x0, csr + PMU_PMOVSR); 1207 else if (xgene_pmu->version == PCP_PMU_V2) 1208 writel(pmovsr, csr + PMU_PMOVSR); 1209 else 1210 writel(pmovsr, csr + PMU_PMOVSCLR); 1211 1212 for (idx = 0; idx < PMU_MAX_COUNTERS; idx++) { 1213 struct perf_event *event = pmu_dev->pmu_counter_event[idx]; 1214 int overflowed = pmovsr & BIT(idx); 1215 1216 /* Ignore if we don't have an event. */ 1217 if (!event || !overflowed) 1218 continue; 1219 xgene_perf_event_update(event); 1220 xgene_perf_event_set_period(event); 1221 } 1222 1223 out: 1224 xgene_pmu->ops->start_counters(pmu_dev); 1225 } 1226 1227 static irqreturn_t xgene_pmu_isr(int irq, void *dev_id) 1228 { 1229 u32 intr_mcu, intr_mcb, intr_l3c, intr_iob; 1230 struct xgene_pmu_dev_ctx *ctx; 1231 struct xgene_pmu *xgene_pmu = dev_id; 1232 u32 val; 1233 1234 raw_spin_lock(&xgene_pmu->lock); 1235 1236 /* Get Interrupt PMU source */ 1237 val = readl(xgene_pmu->pcppmu_csr + PCPPMU_INTSTATUS_REG); 1238 if (xgene_pmu->version == PCP_PMU_V3) { 1239 intr_mcu = PCPPMU_V3_INT_MCU; 1240 intr_mcb = PCPPMU_V3_INT_MCB; 1241 intr_l3c = PCPPMU_V3_INT_L3C; 1242 intr_iob = PCPPMU_V3_INT_IOB; 1243 } else { 1244 intr_mcu = PCPPMU_INT_MCU; 1245 intr_mcb = PCPPMU_INT_MCB; 1246 intr_l3c = PCPPMU_INT_L3C; 1247 intr_iob = PCPPMU_INT_IOB; 1248 } 1249 if (val & intr_mcu) { 1250 list_for_each_entry(ctx, &xgene_pmu->mcpmus, next) { 1251 _xgene_pmu_isr(irq, ctx->pmu_dev); 1252 } 1253 } 1254 if (val & intr_mcb) { 1255 list_for_each_entry(ctx, &xgene_pmu->mcbpmus, next) { 1256 _xgene_pmu_isr(irq, ctx->pmu_dev); 1257 } 1258 } 1259 if (val & intr_l3c) { 1260 list_for_each_entry(ctx, &xgene_pmu->l3cpmus, next) { 1261 _xgene_pmu_isr(irq, ctx->pmu_dev); 1262 } 1263 } 1264 if (val & intr_iob) { 1265 list_for_each_entry(ctx, &xgene_pmu->iobpmus, next) { 1266 _xgene_pmu_isr(irq, ctx->pmu_dev); 1267 } 1268 } 1269 1270 raw_spin_unlock(&xgene_pmu->lock); 1271 1272 return IRQ_HANDLED; 1273 } 1274 1275 static int acpi_pmu_probe_active_mcb_mcu_l3c(struct xgene_pmu *xgene_pmu, 1276 struct platform_device *pdev) 1277 { 1278 void __iomem *csw_csr, *mcba_csr, *mcbb_csr; 1279 unsigned int reg; 1280 1281 csw_csr = devm_platform_ioremap_resource(pdev, 1); 1282 if (IS_ERR(csw_csr)) { 1283 dev_err(&pdev->dev, "ioremap failed for CSW CSR resource\n"); 1284 return PTR_ERR(csw_csr); 1285 } 1286 1287 mcba_csr = devm_platform_ioremap_resource(pdev, 2); 1288 if (IS_ERR(mcba_csr)) { 1289 dev_err(&pdev->dev, "ioremap failed for MCBA CSR resource\n"); 1290 return PTR_ERR(mcba_csr); 1291 } 1292 1293 mcbb_csr = devm_platform_ioremap_resource(pdev, 3); 1294 if (IS_ERR(mcbb_csr)) { 1295 dev_err(&pdev->dev, "ioremap failed for MCBB CSR resource\n"); 1296 return PTR_ERR(mcbb_csr); 1297 } 1298 1299 xgene_pmu->l3c_active_mask = 0x1; 1300 1301 reg = readl(csw_csr + CSW_CSWCR); 1302 if (reg & CSW_CSWCR_DUALMCB_MASK) { 1303 /* Dual MCB active */ 1304 xgene_pmu->mcb_active_mask = 0x3; 1305 /* Probe all active MC(s) */ 1306 reg = readl(mcbb_csr + CSW_CSWCR); 1307 xgene_pmu->mc_active_mask = 1308 (reg & MCBADDRMR_DUALMCU_MODE_MASK) ? 0xF : 0x5; 1309 } else { 1310 /* Single MCB active */ 1311 xgene_pmu->mcb_active_mask = 0x1; 1312 /* Probe all active MC(s) */ 1313 reg = readl(mcba_csr + CSW_CSWCR); 1314 xgene_pmu->mc_active_mask = 1315 (reg & MCBADDRMR_DUALMCU_MODE_MASK) ? 0x3 : 0x1; 1316 } 1317 1318 return 0; 1319 } 1320 1321 static int acpi_pmu_v3_probe_active_mcb_mcu_l3c(struct xgene_pmu *xgene_pmu, 1322 struct platform_device *pdev) 1323 { 1324 void __iomem *csw_csr; 1325 unsigned int reg; 1326 u32 mcb0routing; 1327 u32 mcb1routing; 1328 1329 csw_csr = devm_platform_ioremap_resource(pdev, 1); 1330 if (IS_ERR(csw_csr)) { 1331 dev_err(&pdev->dev, "ioremap failed for CSW CSR resource\n"); 1332 return PTR_ERR(csw_csr); 1333 } 1334 1335 reg = readl(csw_csr + CSW_CSWCR); 1336 mcb0routing = CSW_CSWCR_MCB0_ROUTING(reg); 1337 mcb1routing = CSW_CSWCR_MCB1_ROUTING(reg); 1338 if (reg & CSW_CSWCR_DUALMCB_MASK) { 1339 /* Dual MCB active */ 1340 xgene_pmu->mcb_active_mask = 0x3; 1341 /* Probe all active L3C(s), maximum is 8 */ 1342 xgene_pmu->l3c_active_mask = 0xFF; 1343 /* Probe all active MC(s), maximum is 8 */ 1344 if ((mcb0routing == 0x2) && (mcb1routing == 0x2)) 1345 xgene_pmu->mc_active_mask = 0xFF; 1346 else if ((mcb0routing == 0x1) && (mcb1routing == 0x1)) 1347 xgene_pmu->mc_active_mask = 0x33; 1348 else 1349 xgene_pmu->mc_active_mask = 0x11; 1350 } else { 1351 /* Single MCB active */ 1352 xgene_pmu->mcb_active_mask = 0x1; 1353 /* Probe all active L3C(s), maximum is 4 */ 1354 xgene_pmu->l3c_active_mask = 0x0F; 1355 /* Probe all active MC(s), maximum is 4 */ 1356 if (mcb0routing == 0x2) 1357 xgene_pmu->mc_active_mask = 0x0F; 1358 else if (mcb0routing == 0x1) 1359 xgene_pmu->mc_active_mask = 0x03; 1360 else 1361 xgene_pmu->mc_active_mask = 0x01; 1362 } 1363 1364 return 0; 1365 } 1366 1367 static int fdt_pmu_probe_active_mcb_mcu_l3c(struct xgene_pmu *xgene_pmu, 1368 struct platform_device *pdev) 1369 { 1370 struct regmap *csw_map, *mcba_map, *mcbb_map; 1371 struct device_node *np = pdev->dev.of_node; 1372 unsigned int reg; 1373 1374 csw_map = syscon_regmap_lookup_by_phandle(np, "regmap-csw"); 1375 if (IS_ERR(csw_map)) { 1376 dev_err(&pdev->dev, "unable to get syscon regmap csw\n"); 1377 return PTR_ERR(csw_map); 1378 } 1379 1380 mcba_map = syscon_regmap_lookup_by_phandle(np, "regmap-mcba"); 1381 if (IS_ERR(mcba_map)) { 1382 dev_err(&pdev->dev, "unable to get syscon regmap mcba\n"); 1383 return PTR_ERR(mcba_map); 1384 } 1385 1386 mcbb_map = syscon_regmap_lookup_by_phandle(np, "regmap-mcbb"); 1387 if (IS_ERR(mcbb_map)) { 1388 dev_err(&pdev->dev, "unable to get syscon regmap mcbb\n"); 1389 return PTR_ERR(mcbb_map); 1390 } 1391 1392 xgene_pmu->l3c_active_mask = 0x1; 1393 if (regmap_read(csw_map, CSW_CSWCR, ®)) 1394 return -EINVAL; 1395 1396 if (reg & CSW_CSWCR_DUALMCB_MASK) { 1397 /* Dual MCB active */ 1398 xgene_pmu->mcb_active_mask = 0x3; 1399 /* Probe all active MC(s) */ 1400 if (regmap_read(mcbb_map, MCBADDRMR, ®)) 1401 return 0; 1402 xgene_pmu->mc_active_mask = 1403 (reg & MCBADDRMR_DUALMCU_MODE_MASK) ? 0xF : 0x5; 1404 } else { 1405 /* Single MCB active */ 1406 xgene_pmu->mcb_active_mask = 0x1; 1407 /* Probe all active MC(s) */ 1408 if (regmap_read(mcba_map, MCBADDRMR, ®)) 1409 return 0; 1410 xgene_pmu->mc_active_mask = 1411 (reg & MCBADDRMR_DUALMCU_MODE_MASK) ? 0x3 : 0x1; 1412 } 1413 1414 return 0; 1415 } 1416 1417 static int xgene_pmu_probe_active_mcb_mcu_l3c(struct xgene_pmu *xgene_pmu, 1418 struct platform_device *pdev) 1419 { 1420 if (has_acpi_companion(&pdev->dev)) { 1421 if (xgene_pmu->version == PCP_PMU_V3) 1422 return acpi_pmu_v3_probe_active_mcb_mcu_l3c(xgene_pmu, 1423 pdev); 1424 else 1425 return acpi_pmu_probe_active_mcb_mcu_l3c(xgene_pmu, 1426 pdev); 1427 } 1428 return fdt_pmu_probe_active_mcb_mcu_l3c(xgene_pmu, pdev); 1429 } 1430 1431 static char *xgene_pmu_dev_name(struct device *dev, u32 type, int id) 1432 { 1433 switch (type) { 1434 case PMU_TYPE_L3C: 1435 return devm_kasprintf(dev, GFP_KERNEL, "l3c%d", id); 1436 case PMU_TYPE_IOB: 1437 return devm_kasprintf(dev, GFP_KERNEL, "iob%d", id); 1438 case PMU_TYPE_IOB_SLOW: 1439 return devm_kasprintf(dev, GFP_KERNEL, "iob_slow%d", id); 1440 case PMU_TYPE_MCB: 1441 return devm_kasprintf(dev, GFP_KERNEL, "mcb%d", id); 1442 case PMU_TYPE_MC: 1443 return devm_kasprintf(dev, GFP_KERNEL, "mc%d", id); 1444 default: 1445 return devm_kasprintf(dev, GFP_KERNEL, "unknown"); 1446 } 1447 } 1448 1449 #if defined(CONFIG_ACPI) 1450 static struct 1451 xgene_pmu_dev_ctx *acpi_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, 1452 struct acpi_device *adev, u32 type) 1453 { 1454 struct device *dev = xgene_pmu->dev; 1455 struct list_head resource_list; 1456 struct xgene_pmu_dev_ctx *ctx; 1457 const union acpi_object *obj; 1458 struct hw_pmu_info *inf; 1459 void __iomem *dev_csr; 1460 struct resource res; 1461 struct resource_entry *rentry; 1462 int enable_bit; 1463 int rc; 1464 1465 ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); 1466 if (!ctx) 1467 return NULL; 1468 1469 INIT_LIST_HEAD(&resource_list); 1470 rc = acpi_dev_get_resources(adev, &resource_list, NULL, NULL); 1471 if (rc <= 0) { 1472 dev_err(dev, "PMU type %d: No resources found\n", type); 1473 return NULL; 1474 } 1475 1476 list_for_each_entry(rentry, &resource_list, node) { 1477 if (resource_type(rentry->res) == IORESOURCE_MEM) { 1478 res = *rentry->res; 1479 rentry = NULL; 1480 break; 1481 } 1482 } 1483 acpi_dev_free_resource_list(&resource_list); 1484 1485 if (rentry) { 1486 dev_err(dev, "PMU type %d: No memory resource found\n", type); 1487 return NULL; 1488 } 1489 1490 dev_csr = devm_ioremap_resource(dev, &res); 1491 if (IS_ERR(dev_csr)) { 1492 dev_err(dev, "PMU type %d: Fail to map resource\n", type); 1493 return NULL; 1494 } 1495 1496 /* A PMU device node without enable-bit-index is always enabled */ 1497 rc = acpi_dev_get_property(adev, "enable-bit-index", 1498 ACPI_TYPE_INTEGER, &obj); 1499 if (rc < 0) 1500 enable_bit = 0; 1501 else 1502 enable_bit = (int) obj->integer.value; 1503 1504 ctx->name = xgene_pmu_dev_name(dev, type, enable_bit); 1505 if (!ctx->name) { 1506 dev_err(dev, "PMU type %d: Fail to get device name\n", type); 1507 return NULL; 1508 } 1509 inf = &ctx->inf; 1510 inf->type = type; 1511 inf->csr = dev_csr; 1512 inf->enable_mask = 1 << enable_bit; 1513 1514 return ctx; 1515 } 1516 1517 static const struct acpi_device_id xgene_pmu_acpi_type_match[] = { 1518 {"APMC0D5D", PMU_TYPE_L3C}, 1519 {"APMC0D5E", PMU_TYPE_IOB}, 1520 {"APMC0D5F", PMU_TYPE_MCB}, 1521 {"APMC0D60", PMU_TYPE_MC}, 1522 {"APMC0D84", PMU_TYPE_L3C}, 1523 {"APMC0D85", PMU_TYPE_IOB}, 1524 {"APMC0D86", PMU_TYPE_IOB_SLOW}, 1525 {"APMC0D87", PMU_TYPE_MCB}, 1526 {"APMC0D88", PMU_TYPE_MC}, 1527 {}, 1528 }; 1529 1530 static const struct acpi_device_id *xgene_pmu_acpi_match_type( 1531 const struct acpi_device_id *ids, 1532 struct acpi_device *adev) 1533 { 1534 const struct acpi_device_id *match_id = NULL; 1535 const struct acpi_device_id *id; 1536 1537 for (id = ids; id->id[0] || id->cls; id++) { 1538 if (!acpi_match_device_ids(adev, id)) 1539 match_id = id; 1540 else if (match_id) 1541 break; 1542 } 1543 1544 return match_id; 1545 } 1546 1547 static acpi_status acpi_pmu_dev_add(acpi_handle handle, u32 level, 1548 void *data, void **return_value) 1549 { 1550 struct acpi_device *adev = acpi_fetch_acpi_dev(handle); 1551 const struct acpi_device_id *acpi_id; 1552 struct xgene_pmu *xgene_pmu = data; 1553 struct xgene_pmu_dev_ctx *ctx; 1554 1555 if (!adev || acpi_bus_get_status(adev) || !adev->status.present) 1556 return AE_OK; 1557 1558 acpi_id = xgene_pmu_acpi_match_type(xgene_pmu_acpi_type_match, adev); 1559 if (!acpi_id) 1560 return AE_OK; 1561 1562 ctx = acpi_get_pmu_hw_inf(xgene_pmu, adev, (u32)acpi_id->driver_data); 1563 if (!ctx) 1564 return AE_OK; 1565 1566 if (xgene_pmu_dev_add(xgene_pmu, ctx)) { 1567 /* Can't add the PMU device, skip it */ 1568 devm_kfree(xgene_pmu->dev, ctx); 1569 return AE_OK; 1570 } 1571 1572 switch (ctx->inf.type) { 1573 case PMU_TYPE_L3C: 1574 list_add(&ctx->next, &xgene_pmu->l3cpmus); 1575 break; 1576 case PMU_TYPE_IOB: 1577 list_add(&ctx->next, &xgene_pmu->iobpmus); 1578 break; 1579 case PMU_TYPE_IOB_SLOW: 1580 list_add(&ctx->next, &xgene_pmu->iobpmus); 1581 break; 1582 case PMU_TYPE_MCB: 1583 list_add(&ctx->next, &xgene_pmu->mcbpmus); 1584 break; 1585 case PMU_TYPE_MC: 1586 list_add(&ctx->next, &xgene_pmu->mcpmus); 1587 break; 1588 } 1589 return AE_OK; 1590 } 1591 1592 static int acpi_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu, 1593 struct platform_device *pdev) 1594 { 1595 struct device *dev = xgene_pmu->dev; 1596 acpi_handle handle; 1597 acpi_status status; 1598 1599 handle = ACPI_HANDLE(dev); 1600 if (!handle) 1601 return -EINVAL; 1602 1603 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle, 1, 1604 acpi_pmu_dev_add, NULL, xgene_pmu, NULL); 1605 if (ACPI_FAILURE(status)) { 1606 dev_err(dev, "failed to probe PMU devices\n"); 1607 return -ENODEV; 1608 } 1609 1610 return 0; 1611 } 1612 #else 1613 static int acpi_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu, 1614 struct platform_device *pdev) 1615 { 1616 return 0; 1617 } 1618 #endif 1619 1620 static struct 1621 xgene_pmu_dev_ctx *fdt_get_pmu_hw_inf(struct xgene_pmu *xgene_pmu, 1622 struct device_node *np, u32 type) 1623 { 1624 struct device *dev = xgene_pmu->dev; 1625 struct xgene_pmu_dev_ctx *ctx; 1626 struct hw_pmu_info *inf; 1627 void __iomem *dev_csr; 1628 struct resource res; 1629 int enable_bit; 1630 1631 ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL); 1632 if (!ctx) 1633 return NULL; 1634 1635 if (of_address_to_resource(np, 0, &res) < 0) { 1636 dev_err(dev, "PMU type %d: No resource address found\n", type); 1637 return NULL; 1638 } 1639 1640 dev_csr = devm_ioremap_resource(dev, &res); 1641 if (IS_ERR(dev_csr)) { 1642 dev_err(dev, "PMU type %d: Fail to map resource\n", type); 1643 return NULL; 1644 } 1645 1646 /* A PMU device node without enable-bit-index is always enabled */ 1647 if (of_property_read_u32(np, "enable-bit-index", &enable_bit)) 1648 enable_bit = 0; 1649 1650 ctx->name = xgene_pmu_dev_name(dev, type, enable_bit); 1651 if (!ctx->name) { 1652 dev_err(dev, "PMU type %d: Fail to get device name\n", type); 1653 return NULL; 1654 } 1655 1656 inf = &ctx->inf; 1657 inf->type = type; 1658 inf->csr = dev_csr; 1659 inf->enable_mask = 1 << enable_bit; 1660 1661 return ctx; 1662 } 1663 1664 static int fdt_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu, 1665 struct platform_device *pdev) 1666 { 1667 struct xgene_pmu_dev_ctx *ctx; 1668 struct device_node *np; 1669 1670 for_each_child_of_node(pdev->dev.of_node, np) { 1671 if (!of_device_is_available(np)) 1672 continue; 1673 1674 if (of_device_is_compatible(np, "apm,xgene-pmu-l3c")) 1675 ctx = fdt_get_pmu_hw_inf(xgene_pmu, np, PMU_TYPE_L3C); 1676 else if (of_device_is_compatible(np, "apm,xgene-pmu-iob")) 1677 ctx = fdt_get_pmu_hw_inf(xgene_pmu, np, PMU_TYPE_IOB); 1678 else if (of_device_is_compatible(np, "apm,xgene-pmu-mcb")) 1679 ctx = fdt_get_pmu_hw_inf(xgene_pmu, np, PMU_TYPE_MCB); 1680 else if (of_device_is_compatible(np, "apm,xgene-pmu-mc")) 1681 ctx = fdt_get_pmu_hw_inf(xgene_pmu, np, PMU_TYPE_MC); 1682 else 1683 ctx = NULL; 1684 1685 if (!ctx) 1686 continue; 1687 1688 if (xgene_pmu_dev_add(xgene_pmu, ctx)) { 1689 /* Can't add the PMU device, skip it */ 1690 devm_kfree(xgene_pmu->dev, ctx); 1691 continue; 1692 } 1693 1694 switch (ctx->inf.type) { 1695 case PMU_TYPE_L3C: 1696 list_add(&ctx->next, &xgene_pmu->l3cpmus); 1697 break; 1698 case PMU_TYPE_IOB: 1699 list_add(&ctx->next, &xgene_pmu->iobpmus); 1700 break; 1701 case PMU_TYPE_IOB_SLOW: 1702 list_add(&ctx->next, &xgene_pmu->iobpmus); 1703 break; 1704 case PMU_TYPE_MCB: 1705 list_add(&ctx->next, &xgene_pmu->mcbpmus); 1706 break; 1707 case PMU_TYPE_MC: 1708 list_add(&ctx->next, &xgene_pmu->mcpmus); 1709 break; 1710 } 1711 } 1712 1713 return 0; 1714 } 1715 1716 static int xgene_pmu_probe_pmu_dev(struct xgene_pmu *xgene_pmu, 1717 struct platform_device *pdev) 1718 { 1719 if (has_acpi_companion(&pdev->dev)) 1720 return acpi_pmu_probe_pmu_dev(xgene_pmu, pdev); 1721 return fdt_pmu_probe_pmu_dev(xgene_pmu, pdev); 1722 } 1723 1724 static const struct xgene_pmu_data xgene_pmu_data = { 1725 .id = PCP_PMU_V1, 1726 }; 1727 1728 static const struct xgene_pmu_data xgene_pmu_v2_data = { 1729 .id = PCP_PMU_V2, 1730 }; 1731 1732 #ifdef CONFIG_ACPI 1733 static const struct xgene_pmu_data xgene_pmu_v3_data = { 1734 .id = PCP_PMU_V3, 1735 }; 1736 #endif 1737 1738 static const struct xgene_pmu_ops xgene_pmu_ops = { 1739 .mask_int = xgene_pmu_mask_int, 1740 .unmask_int = xgene_pmu_unmask_int, 1741 .read_counter = xgene_pmu_read_counter32, 1742 .write_counter = xgene_pmu_write_counter32, 1743 .write_evttype = xgene_pmu_write_evttype, 1744 .write_agentmsk = xgene_pmu_write_agentmsk, 1745 .write_agent1msk = xgene_pmu_write_agent1msk, 1746 .enable_counter = xgene_pmu_enable_counter, 1747 .disable_counter = xgene_pmu_disable_counter, 1748 .enable_counter_int = xgene_pmu_enable_counter_int, 1749 .disable_counter_int = xgene_pmu_disable_counter_int, 1750 .reset_counters = xgene_pmu_reset_counters, 1751 .start_counters = xgene_pmu_start_counters, 1752 .stop_counters = xgene_pmu_stop_counters, 1753 }; 1754 1755 static const struct xgene_pmu_ops xgene_pmu_v3_ops = { 1756 .mask_int = xgene_pmu_v3_mask_int, 1757 .unmask_int = xgene_pmu_v3_unmask_int, 1758 .read_counter = xgene_pmu_read_counter64, 1759 .write_counter = xgene_pmu_write_counter64, 1760 .write_evttype = xgene_pmu_write_evttype, 1761 .write_agentmsk = xgene_pmu_v3_write_agentmsk, 1762 .write_agent1msk = xgene_pmu_v3_write_agent1msk, 1763 .enable_counter = xgene_pmu_enable_counter, 1764 .disable_counter = xgene_pmu_disable_counter, 1765 .enable_counter_int = xgene_pmu_enable_counter_int, 1766 .disable_counter_int = xgene_pmu_disable_counter_int, 1767 .reset_counters = xgene_pmu_reset_counters, 1768 .start_counters = xgene_pmu_start_counters, 1769 .stop_counters = xgene_pmu_stop_counters, 1770 }; 1771 1772 static const struct of_device_id xgene_pmu_of_match[] = { 1773 { .compatible = "apm,xgene-pmu", .data = &xgene_pmu_data }, 1774 { .compatible = "apm,xgene-pmu-v2", .data = &xgene_pmu_v2_data }, 1775 {}, 1776 }; 1777 MODULE_DEVICE_TABLE(of, xgene_pmu_of_match); 1778 #ifdef CONFIG_ACPI 1779 static const struct acpi_device_id xgene_pmu_acpi_match[] = { 1780 {"APMC0D5B", (kernel_ulong_t)&xgene_pmu_data}, 1781 {"APMC0D5C", (kernel_ulong_t)&xgene_pmu_v2_data}, 1782 {"APMC0D83", (kernel_ulong_t)&xgene_pmu_v3_data}, 1783 {}, 1784 }; 1785 MODULE_DEVICE_TABLE(acpi, xgene_pmu_acpi_match); 1786 #endif 1787 1788 static int xgene_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) 1789 { 1790 struct xgene_pmu *xgene_pmu = hlist_entry_safe(node, struct xgene_pmu, 1791 node); 1792 1793 if (cpumask_empty(&xgene_pmu->cpu)) 1794 cpumask_set_cpu(cpu, &xgene_pmu->cpu); 1795 1796 /* Overflow interrupt also should use the same CPU */ 1797 WARN_ON(irq_set_affinity(xgene_pmu->irq, &xgene_pmu->cpu)); 1798 1799 return 0; 1800 } 1801 1802 static int xgene_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 1803 { 1804 struct xgene_pmu *xgene_pmu = hlist_entry_safe(node, struct xgene_pmu, 1805 node); 1806 struct xgene_pmu_dev_ctx *ctx; 1807 unsigned int target; 1808 1809 if (!cpumask_test_and_clear_cpu(cpu, &xgene_pmu->cpu)) 1810 return 0; 1811 target = cpumask_any_but(cpu_online_mask, cpu); 1812 if (target >= nr_cpu_ids) 1813 return 0; 1814 1815 list_for_each_entry(ctx, &xgene_pmu->mcpmus, next) { 1816 perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target); 1817 } 1818 list_for_each_entry(ctx, &xgene_pmu->mcbpmus, next) { 1819 perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target); 1820 } 1821 list_for_each_entry(ctx, &xgene_pmu->l3cpmus, next) { 1822 perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target); 1823 } 1824 list_for_each_entry(ctx, &xgene_pmu->iobpmus, next) { 1825 perf_pmu_migrate_context(&ctx->pmu_dev->pmu, cpu, target); 1826 } 1827 1828 cpumask_set_cpu(target, &xgene_pmu->cpu); 1829 /* Overflow interrupt also should use the same CPU */ 1830 WARN_ON(irq_set_affinity(xgene_pmu->irq, &xgene_pmu->cpu)); 1831 1832 return 0; 1833 } 1834 1835 static int xgene_pmu_probe(struct platform_device *pdev) 1836 { 1837 const struct xgene_pmu_data *dev_data; 1838 struct xgene_pmu *xgene_pmu; 1839 int irq, rc; 1840 int version; 1841 1842 /* Install a hook to update the reader CPU in case it goes offline */ 1843 rc = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_APM_XGENE_ONLINE, 1844 "CPUHP_AP_PERF_ARM_APM_XGENE_ONLINE", 1845 xgene_pmu_online_cpu, 1846 xgene_pmu_offline_cpu); 1847 if (rc) 1848 return rc; 1849 1850 xgene_pmu = devm_kzalloc(&pdev->dev, sizeof(*xgene_pmu), GFP_KERNEL); 1851 if (!xgene_pmu) 1852 return -ENOMEM; 1853 xgene_pmu->dev = &pdev->dev; 1854 platform_set_drvdata(pdev, xgene_pmu); 1855 1856 dev_data = device_get_match_data(&pdev->dev); 1857 if (!dev_data) 1858 return -ENODEV; 1859 version = dev_data->id; 1860 1861 if (version == PCP_PMU_V3) 1862 xgene_pmu->ops = &xgene_pmu_v3_ops; 1863 else 1864 xgene_pmu->ops = &xgene_pmu_ops; 1865 1866 INIT_LIST_HEAD(&xgene_pmu->l3cpmus); 1867 INIT_LIST_HEAD(&xgene_pmu->iobpmus); 1868 INIT_LIST_HEAD(&xgene_pmu->mcbpmus); 1869 INIT_LIST_HEAD(&xgene_pmu->mcpmus); 1870 1871 xgene_pmu->version = version; 1872 dev_info(&pdev->dev, "X-Gene PMU version %d\n", xgene_pmu->version); 1873 1874 xgene_pmu->pcppmu_csr = devm_platform_ioremap_resource(pdev, 0); 1875 if (IS_ERR(xgene_pmu->pcppmu_csr)) { 1876 dev_err(&pdev->dev, "ioremap failed for PCP PMU resource\n"); 1877 return PTR_ERR(xgene_pmu->pcppmu_csr); 1878 } 1879 1880 irq = platform_get_irq(pdev, 0); 1881 if (irq < 0) 1882 return -EINVAL; 1883 1884 rc = devm_request_irq(&pdev->dev, irq, xgene_pmu_isr, 1885 IRQF_NOBALANCING | IRQF_NO_THREAD, 1886 dev_name(&pdev->dev), xgene_pmu); 1887 if (rc) { 1888 dev_err(&pdev->dev, "Could not request IRQ %d\n", irq); 1889 return rc; 1890 } 1891 1892 xgene_pmu->irq = irq; 1893 1894 raw_spin_lock_init(&xgene_pmu->lock); 1895 1896 /* Check for active MCBs and MCUs */ 1897 rc = xgene_pmu_probe_active_mcb_mcu_l3c(xgene_pmu, pdev); 1898 if (rc) { 1899 dev_warn(&pdev->dev, "Unknown MCB/MCU active status\n"); 1900 xgene_pmu->mcb_active_mask = 0x1; 1901 xgene_pmu->mc_active_mask = 0x1; 1902 } 1903 1904 /* Add this instance to the list used by the hotplug callback */ 1905 rc = cpuhp_state_add_instance(CPUHP_AP_PERF_ARM_APM_XGENE_ONLINE, 1906 &xgene_pmu->node); 1907 if (rc) { 1908 dev_err(&pdev->dev, "Error %d registering hotplug", rc); 1909 return rc; 1910 } 1911 1912 /* Walk through the tree for all PMU perf devices */ 1913 rc = xgene_pmu_probe_pmu_dev(xgene_pmu, pdev); 1914 if (rc) { 1915 dev_err(&pdev->dev, "No PMU perf devices found!\n"); 1916 goto out_unregister; 1917 } 1918 1919 /* Enable interrupt */ 1920 xgene_pmu->ops->unmask_int(xgene_pmu); 1921 1922 return 0; 1923 1924 out_unregister: 1925 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_APM_XGENE_ONLINE, 1926 &xgene_pmu->node); 1927 return rc; 1928 } 1929 1930 static void 1931 xgene_pmu_dev_cleanup(struct xgene_pmu *xgene_pmu, struct list_head *pmus) 1932 { 1933 struct xgene_pmu_dev_ctx *ctx; 1934 1935 list_for_each_entry(ctx, pmus, next) { 1936 perf_pmu_unregister(&ctx->pmu_dev->pmu); 1937 } 1938 } 1939 1940 static void xgene_pmu_remove(struct platform_device *pdev) 1941 { 1942 struct xgene_pmu *xgene_pmu = dev_get_drvdata(&pdev->dev); 1943 1944 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->l3cpmus); 1945 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->iobpmus); 1946 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcbpmus); 1947 xgene_pmu_dev_cleanup(xgene_pmu, &xgene_pmu->mcpmus); 1948 cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_APM_XGENE_ONLINE, 1949 &xgene_pmu->node); 1950 } 1951 1952 static struct platform_driver xgene_pmu_driver = { 1953 .probe = xgene_pmu_probe, 1954 .remove_new = xgene_pmu_remove, 1955 .driver = { 1956 .name = "xgene-pmu", 1957 .of_match_table = xgene_pmu_of_match, 1958 .acpi_match_table = ACPI_PTR(xgene_pmu_acpi_match), 1959 .suppress_bind_attrs = true, 1960 }, 1961 }; 1962 1963 builtin_platform_driver(xgene_pmu_driver); 1964