1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Alibaba DDR Sub-System Driveway PMU driver 4 * 5 * Copyright (C) 2022 Alibaba Inc 6 */ 7 8 #define ALI_DRW_PMUNAME "ali_drw" 9 #define ALI_DRW_DRVNAME ALI_DRW_PMUNAME "_pmu" 10 #define pr_fmt(fmt) ALI_DRW_DRVNAME ": " fmt 11 12 #include <linux/acpi.h> 13 #include <linux/bitfield.h> 14 #include <linux/bitmap.h> 15 #include <linux/bitops.h> 16 #include <linux/cpuhotplug.h> 17 #include <linux/cpumask.h> 18 #include <linux/device.h> 19 #include <linux/errno.h> 20 #include <linux/interrupt.h> 21 #include <linux/irq.h> 22 #include <linux/kernel.h> 23 #include <linux/list.h> 24 #include <linux/module.h> 25 #include <linux/mutex.h> 26 #include <linux/perf_event.h> 27 #include <linux/platform_device.h> 28 #include <linux/printk.h> 29 #include <linux/rculist.h> 30 #include <linux/refcount.h> 31 32 33 #define ALI_DRW_PMU_COMMON_MAX_COUNTERS 16 34 #define ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE 19 35 36 #define ALI_DRW_PMU_PA_SHIFT 12 37 #define ALI_DRW_PMU_CNT_INIT 0x00000000 38 #define ALI_DRW_CNT_MAX_PERIOD 0xffffffff 39 #define ALI_DRW_PMU_CYCLE_EVT_ID 0x80 40 41 #define ALI_DRW_PMU_CNT_CTRL 0xC00 42 #define ALI_DRW_PMU_CNT_RST BIT(2) 43 #define ALI_DRW_PMU_CNT_STOP BIT(1) 44 #define ALI_DRW_PMU_CNT_START BIT(0) 45 46 #define ALI_DRW_PMU_CNT_STATE 0xC04 47 #define ALI_DRW_PMU_TEST_CTRL 0xC08 48 #define ALI_DRW_PMU_CNT_PRELOAD 0xC0C 49 50 #define ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK GENMASK(23, 0) 51 #define ALI_DRW_PMU_CYCLE_CNT_LOW_MASK GENMASK(31, 0) 52 #define ALI_DRW_PMU_CYCLE_CNT_HIGH 0xC10 53 #define ALI_DRW_PMU_CYCLE_CNT_LOW 0xC14 54 55 /* PMU EVENT SEL 0-3 are paired in 32-bit registers on a 4-byte stride */ 56 #define ALI_DRW_PMU_EVENT_SEL0 0xC68 57 /* counter 0-3 use sel0, counter 4-7 use sel1...*/ 58 #define ALI_DRW_PMU_EVENT_SELn(n) \ 59 (ALI_DRW_PMU_EVENT_SEL0 + (n / 4) * 0x4) 60 #define ALI_DRW_PMCOM_CNT_EN BIT(7) 61 #define ALI_DRW_PMCOM_CNT_EVENT_MASK GENMASK(5, 0) 62 #define ALI_DRW_PMCOM_CNT_EVENT_OFFSET(n) \ 63 (8 * (n % 4)) 64 65 /* PMU COMMON COUNTER 0-15, are paired in 32-bit registers on a 4-byte stride */ 66 #define ALI_DRW_PMU_COMMON_COUNTER0 0xC78 67 #define ALI_DRW_PMU_COMMON_COUNTERn(n) \ 68 (ALI_DRW_PMU_COMMON_COUNTER0 + 0x4 * (n)) 69 70 #define ALI_DRW_PMU_OV_INTR_ENABLE_CTL 0xCB8 71 #define ALI_DRW_PMU_OV_INTR_DISABLE_CTL 0xCBC 72 #define ALI_DRW_PMU_OV_INTR_ENABLE_STATUS 0xCC0 73 #define ALI_DRW_PMU_OV_INTR_CLR 0xCC4 74 #define ALI_DRW_PMU_OV_INTR_STATUS 0xCC8 75 #define ALI_DRW_PMCOM_CNT_OV_INTR_MASK GENMASK(23, 8) 76 #define ALI_DRW_PMBW_CNT_OV_INTR_MASK GENMASK(7, 0) 77 #define ALI_DRW_PMU_OV_INTR_MASK GENMASK_ULL(63, 0) 78 79 static int ali_drw_cpuhp_state_num; 80 81 static LIST_HEAD(ali_drw_pmu_irqs); 82 static DEFINE_MUTEX(ali_drw_pmu_irqs_lock); 83 84 struct ali_drw_pmu_irq { 85 struct hlist_node node; 86 struct list_head irqs_node; 87 struct list_head pmus_node; 88 int irq_num; 89 int cpu; 90 refcount_t refcount; 91 }; 92 93 struct ali_drw_pmu { 94 void __iomem *cfg_base; 95 struct device *dev; 96 97 struct list_head pmus_node; 98 struct ali_drw_pmu_irq *irq; 99 int irq_num; 100 int cpu; 101 DECLARE_BITMAP(used_mask, ALI_DRW_PMU_COMMON_MAX_COUNTERS); 102 struct perf_event *events[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; 103 int evtids[ALI_DRW_PMU_COMMON_MAX_COUNTERS]; 104 105 struct pmu pmu; 106 }; 107 108 #define to_ali_drw_pmu(p) (container_of(p, struct ali_drw_pmu, pmu)) 109 110 #define DRW_CONFIG_EVENTID GENMASK(7, 0) 111 #define GET_DRW_EVENTID(event) FIELD_GET(DRW_CONFIG_EVENTID, (event)->attr.config) 112 113 static ssize_t ali_drw_pmu_format_show(struct device *dev, 114 struct device_attribute *attr, char *buf) 115 { 116 struct dev_ext_attribute *eattr; 117 118 eattr = container_of(attr, struct dev_ext_attribute, attr); 119 120 return sprintf(buf, "%s\n", (char *)eattr->var); 121 } 122 123 /* 124 * PMU event attributes 125 */ 126 static ssize_t ali_drw_pmu_event_show(struct device *dev, 127 struct device_attribute *attr, char *page) 128 { 129 struct dev_ext_attribute *eattr; 130 131 eattr = container_of(attr, struct dev_ext_attribute, attr); 132 133 return sprintf(page, "config=0x%lx\n", (unsigned long)eattr->var); 134 } 135 136 #define ALI_DRW_PMU_ATTR(_name, _func, _config) \ 137 (&((struct dev_ext_attribute[]) { \ 138 { __ATTR(_name, 0444, _func, NULL), (void *)_config } \ 139 })[0].attr.attr) 140 141 #define ALI_DRW_PMU_FORMAT_ATTR(_name, _config) \ 142 ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_format_show, (void *)_config) 143 #define ALI_DRW_PMU_EVENT_ATTR(_name, _config) \ 144 ALI_DRW_PMU_ATTR(_name, ali_drw_pmu_event_show, (unsigned long)_config) 145 146 static struct attribute *ali_drw_pmu_events_attrs[] = { 147 ALI_DRW_PMU_EVENT_ATTR(hif_rd_or_wr, 0x0), 148 ALI_DRW_PMU_EVENT_ATTR(hif_wr, 0x1), 149 ALI_DRW_PMU_EVENT_ATTR(hif_rd, 0x2), 150 ALI_DRW_PMU_EVENT_ATTR(hif_rmw, 0x3), 151 ALI_DRW_PMU_EVENT_ATTR(hif_hi_pri_rd, 0x4), 152 ALI_DRW_PMU_EVENT_ATTR(dfi_wr_data_cycles, 0x7), 153 ALI_DRW_PMU_EVENT_ATTR(dfi_rd_data_cycles, 0x8), 154 ALI_DRW_PMU_EVENT_ATTR(hpr_xact_when_critical, 0x9), 155 ALI_DRW_PMU_EVENT_ATTR(lpr_xact_when_critical, 0xA), 156 ALI_DRW_PMU_EVENT_ATTR(wr_xact_when_critical, 0xB), 157 ALI_DRW_PMU_EVENT_ATTR(op_is_activate, 0xC), 158 ALI_DRW_PMU_EVENT_ATTR(op_is_rd_or_wr, 0xD), 159 ALI_DRW_PMU_EVENT_ATTR(op_is_rd_activate, 0xE), 160 ALI_DRW_PMU_EVENT_ATTR(op_is_rd, 0xF), 161 ALI_DRW_PMU_EVENT_ATTR(op_is_wr, 0x10), 162 ALI_DRW_PMU_EVENT_ATTR(op_is_mwr, 0x11), 163 ALI_DRW_PMU_EVENT_ATTR(op_is_precharge, 0x12), 164 ALI_DRW_PMU_EVENT_ATTR(precharge_for_rdwr, 0x13), 165 ALI_DRW_PMU_EVENT_ATTR(precharge_for_other, 0x14), 166 ALI_DRW_PMU_EVENT_ATTR(rdwr_transitions, 0x15), 167 ALI_DRW_PMU_EVENT_ATTR(write_combine, 0x16), 168 ALI_DRW_PMU_EVENT_ATTR(war_hazard, 0x17), 169 ALI_DRW_PMU_EVENT_ATTR(raw_hazard, 0x18), 170 ALI_DRW_PMU_EVENT_ATTR(waw_hazard, 0x19), 171 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk0, 0x1A), 172 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk1, 0x1B), 173 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk2, 0x1C), 174 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_selfref_rk3, 0x1D), 175 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk0, 0x1E), 176 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk1, 0x1F), 177 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk2, 0x20), 178 ALI_DRW_PMU_EVENT_ATTR(op_is_enter_powerdown_rk3, 0x21), 179 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk0, 0x26), 180 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk1, 0x27), 181 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk2, 0x28), 182 ALI_DRW_PMU_EVENT_ATTR(selfref_mode_rk3, 0x29), 183 ALI_DRW_PMU_EVENT_ATTR(op_is_refresh, 0x2A), 184 ALI_DRW_PMU_EVENT_ATTR(op_is_crit_ref, 0x2B), 185 ALI_DRW_PMU_EVENT_ATTR(op_is_load_mode, 0x2D), 186 ALI_DRW_PMU_EVENT_ATTR(op_is_zqcl, 0x2E), 187 ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_rd, 0x30), 188 ALI_DRW_PMU_EVENT_ATTR(visible_window_limit_reached_wr, 0x31), 189 ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mpc, 0x34), 190 ALI_DRW_PMU_EVENT_ATTR(op_is_dqsosc_mrr, 0x35), 191 ALI_DRW_PMU_EVENT_ATTR(op_is_tcr_mrr, 0x36), 192 ALI_DRW_PMU_EVENT_ATTR(op_is_zqstart, 0x37), 193 ALI_DRW_PMU_EVENT_ATTR(op_is_zqlatch, 0x38), 194 ALI_DRW_PMU_EVENT_ATTR(chi_txreq, 0x39), 195 ALI_DRW_PMU_EVENT_ATTR(chi_txdat, 0x3A), 196 ALI_DRW_PMU_EVENT_ATTR(chi_rxdat, 0x3B), 197 ALI_DRW_PMU_EVENT_ATTR(chi_rxrsp, 0x3C), 198 ALI_DRW_PMU_EVENT_ATTR(tsz_vio, 0x3D), 199 ALI_DRW_PMU_EVENT_ATTR(cycle, 0x80), 200 NULL, 201 }; 202 203 static struct attribute_group ali_drw_pmu_events_attr_group = { 204 .name = "events", 205 .attrs = ali_drw_pmu_events_attrs, 206 }; 207 208 static struct attribute *ali_drw_pmu_format_attr[] = { 209 ALI_DRW_PMU_FORMAT_ATTR(event, "config:0-7"), 210 NULL, 211 }; 212 213 static const struct attribute_group ali_drw_pmu_format_group = { 214 .name = "format", 215 .attrs = ali_drw_pmu_format_attr, 216 }; 217 218 static ssize_t ali_drw_pmu_cpumask_show(struct device *dev, 219 struct device_attribute *attr, 220 char *buf) 221 { 222 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(dev_get_drvdata(dev)); 223 224 return cpumap_print_to_pagebuf(true, buf, cpumask_of(drw_pmu->cpu)); 225 } 226 227 static struct device_attribute ali_drw_pmu_cpumask_attr = 228 __ATTR(cpumask, 0444, ali_drw_pmu_cpumask_show, NULL); 229 230 static struct attribute *ali_drw_pmu_cpumask_attrs[] = { 231 &ali_drw_pmu_cpumask_attr.attr, 232 NULL, 233 }; 234 235 static const struct attribute_group ali_drw_pmu_cpumask_attr_group = { 236 .attrs = ali_drw_pmu_cpumask_attrs, 237 }; 238 239 static const struct attribute_group *ali_drw_pmu_attr_groups[] = { 240 &ali_drw_pmu_events_attr_group, 241 &ali_drw_pmu_cpumask_attr_group, 242 &ali_drw_pmu_format_group, 243 NULL, 244 }; 245 246 /* find a counter for event, then in add func, hw.idx will equal to counter */ 247 static int ali_drw_get_counter_idx(struct perf_event *event) 248 { 249 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 250 int idx; 251 252 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; ++idx) { 253 if (!test_and_set_bit(idx, drw_pmu->used_mask)) 254 return idx; 255 } 256 257 /* The counters are all in use. */ 258 return -EBUSY; 259 } 260 261 static u64 ali_drw_pmu_read_counter(struct perf_event *event) 262 { 263 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 264 u64 cycle_high, cycle_low; 265 266 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { 267 cycle_high = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_HIGH); 268 cycle_high &= ALI_DRW_PMU_CYCLE_CNT_HIGH_MASK; 269 cycle_low = readl(drw_pmu->cfg_base + ALI_DRW_PMU_CYCLE_CNT_LOW); 270 cycle_low &= ALI_DRW_PMU_CYCLE_CNT_LOW_MASK; 271 return (cycle_high << 32 | cycle_low); 272 } 273 274 return readl(drw_pmu->cfg_base + 275 ALI_DRW_PMU_COMMON_COUNTERn(event->hw.idx)); 276 } 277 278 static void ali_drw_pmu_event_update(struct perf_event *event) 279 { 280 struct hw_perf_event *hwc = &event->hw; 281 u64 delta, prev, now; 282 283 do { 284 prev = local64_read(&hwc->prev_count); 285 now = ali_drw_pmu_read_counter(event); 286 } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev); 287 288 /* handle overflow. */ 289 delta = now - prev; 290 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) 291 delta &= ALI_DRW_PMU_OV_INTR_MASK; 292 else 293 delta &= ALI_DRW_CNT_MAX_PERIOD; 294 local64_add(delta, &event->count); 295 } 296 297 static void ali_drw_pmu_event_set_period(struct perf_event *event) 298 { 299 u64 pre_val; 300 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 301 302 /* set a preload counter for test purpose */ 303 writel(ALI_DRW_PMU_TEST_SEL_COMMON_COUNTER_BASE + event->hw.idx, 304 drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); 305 306 /* set conunter initial value */ 307 pre_val = ALI_DRW_PMU_CNT_INIT; 308 writel(pre_val, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); 309 local64_set(&event->hw.prev_count, pre_val); 310 311 /* set sel mode to zero to start test */ 312 writel(0x0, drw_pmu->cfg_base + ALI_DRW_PMU_TEST_CTRL); 313 } 314 315 static void ali_drw_pmu_enable_counter(struct perf_event *event) 316 { 317 u32 val, subval, reg, shift; 318 int counter = event->hw.idx; 319 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 320 321 reg = ALI_DRW_PMU_EVENT_SELn(counter); 322 val = readl(drw_pmu->cfg_base + reg); 323 subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 1) | 324 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, drw_pmu->evtids[counter]); 325 326 shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); 327 val &= ~(GENMASK(7, 0) << shift); 328 val |= subval << shift; 329 330 writel(val, drw_pmu->cfg_base + reg); 331 } 332 333 static void ali_drw_pmu_disable_counter(struct perf_event *event) 334 { 335 u32 val, reg, subval, shift; 336 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 337 int counter = event->hw.idx; 338 339 reg = ALI_DRW_PMU_EVENT_SELn(counter); 340 val = readl(drw_pmu->cfg_base + reg); 341 subval = FIELD_PREP(ALI_DRW_PMCOM_CNT_EN, 0) | 342 FIELD_PREP(ALI_DRW_PMCOM_CNT_EVENT_MASK, 0); 343 344 shift = ALI_DRW_PMCOM_CNT_EVENT_OFFSET(counter); 345 val &= ~(GENMASK(7, 0) << shift); 346 val |= subval << shift; 347 348 writel(val, drw_pmu->cfg_base + reg); 349 } 350 351 static irqreturn_t ali_drw_pmu_isr(int irq_num, void *data) 352 { 353 struct ali_drw_pmu_irq *irq = data; 354 struct ali_drw_pmu *drw_pmu; 355 irqreturn_t ret = IRQ_NONE; 356 357 rcu_read_lock(); 358 list_for_each_entry_rcu(drw_pmu, &irq->pmus_node, pmus_node) { 359 unsigned long status, clr_status; 360 struct perf_event *event; 361 unsigned int idx; 362 363 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { 364 event = drw_pmu->events[idx]; 365 if (!event) 366 continue; 367 ali_drw_pmu_disable_counter(event); 368 } 369 370 /* common counter intr status */ 371 status = readl(drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_STATUS); 372 status = FIELD_GET(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, status); 373 if (status) { 374 for_each_set_bit(idx, &status, 375 ALI_DRW_PMU_COMMON_MAX_COUNTERS) { 376 event = drw_pmu->events[idx]; 377 if (WARN_ON_ONCE(!event)) 378 continue; 379 ali_drw_pmu_event_update(event); 380 ali_drw_pmu_event_set_period(event); 381 } 382 383 /* clear common counter intr status */ 384 clr_status = FIELD_PREP(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 1); 385 writel(clr_status, 386 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); 387 } 388 389 for (idx = 0; idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS; idx++) { 390 event = drw_pmu->events[idx]; 391 if (!event) 392 continue; 393 if (!(event->hw.state & PERF_HES_STOPPED)) 394 ali_drw_pmu_enable_counter(event); 395 } 396 if (status) 397 ret = IRQ_HANDLED; 398 } 399 rcu_read_unlock(); 400 return ret; 401 } 402 403 static struct ali_drw_pmu_irq *__ali_drw_pmu_init_irq(struct platform_device 404 *pdev, int irq_num) 405 { 406 int ret; 407 struct ali_drw_pmu_irq *irq; 408 409 list_for_each_entry(irq, &ali_drw_pmu_irqs, irqs_node) { 410 if (irq->irq_num == irq_num 411 && refcount_inc_not_zero(&irq->refcount)) 412 return irq; 413 } 414 415 irq = kzalloc(sizeof(*irq), GFP_KERNEL); 416 if (!irq) 417 return ERR_PTR(-ENOMEM); 418 419 INIT_LIST_HEAD(&irq->pmus_node); 420 421 /* Pick one CPU to be the preferred one to use */ 422 irq->cpu = smp_processor_id(); 423 refcount_set(&irq->refcount, 1); 424 425 /* 426 * FIXME: one of DDRSS Driveway PMU overflow interrupt shares the same 427 * irq number with MPAM ERR_IRQ. To register DDRSS PMU and MPAM drivers 428 * successfully, add IRQF_SHARED flag. Howerer, PMU interrupt should not 429 * share with other component. 430 */ 431 ret = devm_request_irq(&pdev->dev, irq_num, ali_drw_pmu_isr, 432 IRQF_SHARED, dev_name(&pdev->dev), irq); 433 if (ret < 0) { 434 dev_err(&pdev->dev, 435 "Fail to request IRQ:%d ret:%d\n", irq_num, ret); 436 goto out_free; 437 } 438 439 ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu)); 440 if (ret) 441 goto out_free; 442 443 ret = cpuhp_state_add_instance_nocalls(ali_drw_cpuhp_state_num, 444 &irq->node); 445 if (ret) 446 goto out_free; 447 448 irq->irq_num = irq_num; 449 list_add(&irq->irqs_node, &ali_drw_pmu_irqs); 450 451 return irq; 452 453 out_free: 454 kfree(irq); 455 return ERR_PTR(ret); 456 } 457 458 static int ali_drw_pmu_init_irq(struct ali_drw_pmu *drw_pmu, 459 struct platform_device *pdev) 460 { 461 int irq_num; 462 struct ali_drw_pmu_irq *irq; 463 464 /* Read and init IRQ */ 465 irq_num = platform_get_irq(pdev, 0); 466 if (irq_num < 0) 467 return irq_num; 468 469 mutex_lock(&ali_drw_pmu_irqs_lock); 470 irq = __ali_drw_pmu_init_irq(pdev, irq_num); 471 mutex_unlock(&ali_drw_pmu_irqs_lock); 472 473 if (IS_ERR(irq)) 474 return PTR_ERR(irq); 475 476 drw_pmu->irq = irq; 477 478 mutex_lock(&ali_drw_pmu_irqs_lock); 479 list_add_rcu(&drw_pmu->pmus_node, &irq->pmus_node); 480 mutex_unlock(&ali_drw_pmu_irqs_lock); 481 482 return 0; 483 } 484 485 static void ali_drw_pmu_uninit_irq(struct ali_drw_pmu *drw_pmu) 486 { 487 struct ali_drw_pmu_irq *irq = drw_pmu->irq; 488 489 mutex_lock(&ali_drw_pmu_irqs_lock); 490 list_del_rcu(&drw_pmu->pmus_node); 491 492 if (!refcount_dec_and_test(&irq->refcount)) { 493 mutex_unlock(&ali_drw_pmu_irqs_lock); 494 return; 495 } 496 497 list_del(&irq->irqs_node); 498 mutex_unlock(&ali_drw_pmu_irqs_lock); 499 500 WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL)); 501 cpuhp_state_remove_instance_nocalls(ali_drw_cpuhp_state_num, 502 &irq->node); 503 kfree(irq); 504 } 505 506 static int ali_drw_pmu_event_init(struct perf_event *event) 507 { 508 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 509 struct hw_perf_event *hwc = &event->hw; 510 struct perf_event *sibling; 511 struct device *dev = drw_pmu->pmu.dev; 512 513 if (event->attr.type != event->pmu->type) 514 return -ENOENT; 515 516 if (is_sampling_event(event)) { 517 dev_err(dev, "Sampling not supported!\n"); 518 return -EOPNOTSUPP; 519 } 520 521 if (event->attach_state & PERF_ATTACH_TASK) { 522 dev_err(dev, "Per-task counter cannot allocate!\n"); 523 return -EOPNOTSUPP; 524 } 525 526 event->cpu = drw_pmu->cpu; 527 if (event->cpu < 0) { 528 dev_err(dev, "Per-task mode not supported!\n"); 529 return -EOPNOTSUPP; 530 } 531 532 if (event->group_leader != event && 533 !is_software_event(event->group_leader)) { 534 dev_err(dev, "driveway only allow one event!\n"); 535 return -EINVAL; 536 } 537 538 for_each_sibling_event(sibling, event->group_leader) { 539 if (sibling != event && !is_software_event(sibling)) { 540 dev_err(dev, "driveway event not allowed!\n"); 541 return -EINVAL; 542 } 543 } 544 545 /* reset all the pmu counters */ 546 writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 547 548 hwc->idx = -1; 549 550 return 0; 551 } 552 553 static void ali_drw_pmu_start(struct perf_event *event, int flags) 554 { 555 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 556 557 event->hw.state = 0; 558 559 if (GET_DRW_EVENTID(event) == ALI_DRW_PMU_CYCLE_EVT_ID) { 560 writel(ALI_DRW_PMU_CNT_START, 561 drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 562 return; 563 } 564 565 ali_drw_pmu_event_set_period(event); 566 if (flags & PERF_EF_RELOAD) { 567 unsigned long prev_raw_count = 568 local64_read(&event->hw.prev_count); 569 writel(prev_raw_count, 570 drw_pmu->cfg_base + ALI_DRW_PMU_CNT_PRELOAD); 571 } 572 573 ali_drw_pmu_enable_counter(event); 574 575 writel(ALI_DRW_PMU_CNT_START, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 576 } 577 578 static void ali_drw_pmu_stop(struct perf_event *event, int flags) 579 { 580 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 581 582 if (event->hw.state & PERF_HES_STOPPED) 583 return; 584 585 if (GET_DRW_EVENTID(event) != ALI_DRW_PMU_CYCLE_EVT_ID) 586 ali_drw_pmu_disable_counter(event); 587 588 writel(ALI_DRW_PMU_CNT_STOP, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 589 590 ali_drw_pmu_event_update(event); 591 event->hw.state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 592 } 593 594 static int ali_drw_pmu_add(struct perf_event *event, int flags) 595 { 596 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 597 struct hw_perf_event *hwc = &event->hw; 598 int idx = -1; 599 int evtid; 600 601 evtid = GET_DRW_EVENTID(event); 602 603 if (evtid != ALI_DRW_PMU_CYCLE_EVT_ID) { 604 idx = ali_drw_get_counter_idx(event); 605 if (idx < 0) 606 return idx; 607 drw_pmu->events[idx] = event; 608 drw_pmu->evtids[idx] = evtid; 609 } 610 hwc->idx = idx; 611 612 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 613 614 if (flags & PERF_EF_START) 615 ali_drw_pmu_start(event, PERF_EF_RELOAD); 616 617 /* Propagate our changes to the userspace mapping. */ 618 perf_event_update_userpage(event); 619 620 return 0; 621 } 622 623 static void ali_drw_pmu_del(struct perf_event *event, int flags) 624 { 625 struct ali_drw_pmu *drw_pmu = to_ali_drw_pmu(event->pmu); 626 struct hw_perf_event *hwc = &event->hw; 627 int idx = hwc->idx; 628 629 ali_drw_pmu_stop(event, PERF_EF_UPDATE); 630 631 if (idx >= 0 && idx < ALI_DRW_PMU_COMMON_MAX_COUNTERS) { 632 drw_pmu->events[idx] = NULL; 633 drw_pmu->evtids[idx] = 0; 634 clear_bit(idx, drw_pmu->used_mask); 635 } 636 637 perf_event_update_userpage(event); 638 } 639 640 static void ali_drw_pmu_read(struct perf_event *event) 641 { 642 ali_drw_pmu_event_update(event); 643 } 644 645 static int ali_drw_pmu_probe(struct platform_device *pdev) 646 { 647 struct ali_drw_pmu *drw_pmu; 648 struct resource *res; 649 char *name; 650 int ret; 651 652 drw_pmu = devm_kzalloc(&pdev->dev, sizeof(*drw_pmu), GFP_KERNEL); 653 if (!drw_pmu) 654 return -ENOMEM; 655 656 drw_pmu->dev = &pdev->dev; 657 platform_set_drvdata(pdev, drw_pmu); 658 659 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 660 drw_pmu->cfg_base = devm_ioremap_resource(&pdev->dev, res); 661 if (IS_ERR(drw_pmu->cfg_base)) 662 return PTR_ERR(drw_pmu->cfg_base); 663 664 name = devm_kasprintf(drw_pmu->dev, GFP_KERNEL, "ali_drw_%llx", 665 (u64) (res->start >> ALI_DRW_PMU_PA_SHIFT)); 666 if (!name) 667 return -ENOMEM; 668 669 writel(ALI_DRW_PMU_CNT_RST, drw_pmu->cfg_base + ALI_DRW_PMU_CNT_CTRL); 670 671 /* enable the generation of interrupt by all common counters */ 672 writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 673 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_ENABLE_CTL); 674 675 /* clearing interrupt status */ 676 writel(0xffffff, drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_CLR); 677 678 drw_pmu->cpu = smp_processor_id(); 679 680 ret = ali_drw_pmu_init_irq(drw_pmu, pdev); 681 if (ret) 682 return ret; 683 684 drw_pmu->pmu = (struct pmu) { 685 .module = THIS_MODULE, 686 .task_ctx_nr = perf_invalid_context, 687 .event_init = ali_drw_pmu_event_init, 688 .add = ali_drw_pmu_add, 689 .del = ali_drw_pmu_del, 690 .start = ali_drw_pmu_start, 691 .stop = ali_drw_pmu_stop, 692 .read = ali_drw_pmu_read, 693 .attr_groups = ali_drw_pmu_attr_groups, 694 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 695 }; 696 697 ret = perf_pmu_register(&drw_pmu->pmu, name, -1); 698 if (ret) { 699 dev_err(drw_pmu->dev, "DRW Driveway PMU PMU register failed!\n"); 700 ali_drw_pmu_uninit_irq(drw_pmu); 701 } 702 703 return ret; 704 } 705 706 static int ali_drw_pmu_remove(struct platform_device *pdev) 707 { 708 struct ali_drw_pmu *drw_pmu = platform_get_drvdata(pdev); 709 710 /* disable the generation of interrupt by all common counters */ 711 writel(ALI_DRW_PMCOM_CNT_OV_INTR_MASK, 712 drw_pmu->cfg_base + ALI_DRW_PMU_OV_INTR_DISABLE_CTL); 713 714 ali_drw_pmu_uninit_irq(drw_pmu); 715 perf_pmu_unregister(&drw_pmu->pmu); 716 717 return 0; 718 } 719 720 static int ali_drw_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 721 { 722 struct ali_drw_pmu_irq *irq; 723 struct ali_drw_pmu *drw_pmu; 724 unsigned int target; 725 int ret; 726 cpumask_t node_online_cpus; 727 728 irq = hlist_entry_safe(node, struct ali_drw_pmu_irq, node); 729 if (cpu != irq->cpu) 730 return 0; 731 732 ret = cpumask_and(&node_online_cpus, 733 cpumask_of_node(cpu_to_node(cpu)), cpu_online_mask); 734 if (ret) 735 target = cpumask_any_but(&node_online_cpus, cpu); 736 else 737 target = cpumask_any_but(cpu_online_mask, cpu); 738 739 if (target >= nr_cpu_ids) 740 return 0; 741 742 /* We're only reading, but this isn't the place to be involving RCU */ 743 mutex_lock(&ali_drw_pmu_irqs_lock); 744 list_for_each_entry(drw_pmu, &irq->pmus_node, pmus_node) 745 perf_pmu_migrate_context(&drw_pmu->pmu, irq->cpu, target); 746 mutex_unlock(&ali_drw_pmu_irqs_lock); 747 748 WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target))); 749 irq->cpu = target; 750 751 return 0; 752 } 753 754 /* 755 * Due to historical reasons, the HID used in the production environment is 756 * ARMHD700, so we leave ARMHD700 as Compatible ID. 757 */ 758 static const struct acpi_device_id ali_drw_acpi_match[] = { 759 {"BABA5000", 0}, 760 {"ARMHD700", 0}, 761 {} 762 }; 763 764 MODULE_DEVICE_TABLE(acpi, ali_drw_acpi_match); 765 766 static struct platform_driver ali_drw_pmu_driver = { 767 .driver = { 768 .name = "ali_drw_pmu", 769 .acpi_match_table = ali_drw_acpi_match, 770 }, 771 .probe = ali_drw_pmu_probe, 772 .remove = ali_drw_pmu_remove, 773 }; 774 775 static int __init ali_drw_pmu_init(void) 776 { 777 int ret; 778 779 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 780 "ali_drw_pmu:online", 781 NULL, ali_drw_pmu_offline_cpu); 782 783 if (ret < 0) { 784 pr_err("DRW Driveway PMU: setup hotplug failed, ret = %d\n", 785 ret); 786 return ret; 787 } 788 ali_drw_cpuhp_state_num = ret; 789 790 ret = platform_driver_register(&ali_drw_pmu_driver); 791 if (ret) 792 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num); 793 794 return ret; 795 } 796 797 static void __exit ali_drw_pmu_exit(void) 798 { 799 platform_driver_unregister(&ali_drw_pmu_driver); 800 cpuhp_remove_multi_state(ali_drw_cpuhp_state_num); 801 } 802 803 module_init(ali_drw_pmu_init); 804 module_exit(ali_drw_pmu_exit); 805 806 MODULE_AUTHOR("Hongbo Yao <yaohongbo@linux.alibaba.com>"); 807 MODULE_AUTHOR("Neng Chen <nengchen@linux.alibaba.com>"); 808 MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>"); 809 MODULE_DESCRIPTION("Alibaba DDR Sub-System Driveway PMU driver"); 810 MODULE_LICENSE("GPL v2"); 811