1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell CN10K DRAM Subsystem (DSS) Performance Monitor Driver 3 * 4 * Copyright (C) 2021 Marvell. 5 */ 6 7 #include <linux/init.h> 8 #include <linux/io.h> 9 #include <linux/module.h> 10 #include <linux/of.h> 11 #include <linux/of_address.h> 12 #include <linux/of_device.h> 13 #include <linux/perf_event.h> 14 #include <linux/hrtimer.h> 15 16 /* Performance Counters Operating Mode Control Registers */ 17 #define DDRC_PERF_CNT_OP_MODE_CTRL 0x8020 18 #define OP_MODE_CTRL_VAL_MANNUAL 0x1 19 20 /* Performance Counters Start Operation Control Registers */ 21 #define DDRC_PERF_CNT_START_OP_CTRL 0x8028 22 #define START_OP_CTRL_VAL_START 0x1ULL 23 #define START_OP_CTRL_VAL_ACTIVE 0x2 24 25 /* Performance Counters End Operation Control Registers */ 26 #define DDRC_PERF_CNT_END_OP_CTRL 0x8030 27 #define END_OP_CTRL_VAL_END 0x1ULL 28 29 /* Performance Counters End Status Registers */ 30 #define DDRC_PERF_CNT_END_STATUS 0x8038 31 #define END_STATUS_VAL_END_TIMER_MODE_END 0x1 32 33 /* Performance Counters Configuration Registers */ 34 #define DDRC_PERF_CFG_BASE 0x8040 35 36 /* 8 Generic event counter + 2 fixed event counters */ 37 #define DDRC_PERF_NUM_GEN_COUNTERS 8 38 #define DDRC_PERF_NUM_FIX_COUNTERS 2 39 #define DDRC_PERF_READ_COUNTER_IDX DDRC_PERF_NUM_GEN_COUNTERS 40 #define DDRC_PERF_WRITE_COUNTER_IDX (DDRC_PERF_NUM_GEN_COUNTERS + 1) 41 #define DDRC_PERF_NUM_COUNTERS (DDRC_PERF_NUM_GEN_COUNTERS + \ 42 DDRC_PERF_NUM_FIX_COUNTERS) 43 44 /* Generic event counter registers */ 45 #define DDRC_PERF_CFG(n) (DDRC_PERF_CFG_BASE + 8 * (n)) 46 #define EVENT_ENABLE BIT_ULL(63) 47 48 /* Two dedicated event counters for DDR reads and writes */ 49 #define EVENT_DDR_READS 101 50 #define EVENT_DDR_WRITES 100 51 52 /* 53 * programmable events IDs in programmable event counters. 54 * DO NOT change these event-id numbers, they are used to 55 * program event bitmap in h/w. 56 */ 57 #define EVENT_OP_IS_ZQLATCH 55 58 #define EVENT_OP_IS_ZQSTART 54 59 #define EVENT_OP_IS_TCR_MRR 53 60 #define EVENT_OP_IS_DQSOSC_MRR 52 61 #define EVENT_OP_IS_DQSOSC_MPC 51 62 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_WR 50 63 #define EVENT_VISIBLE_WIN_LIMIT_REACHED_RD 49 64 #define EVENT_BSM_STARVATION 48 65 #define EVENT_BSM_ALLOC 47 66 #define EVENT_LPR_REQ_WITH_NOCREDIT 46 67 #define EVENT_HPR_REQ_WITH_NOCREDIT 45 68 #define EVENT_OP_IS_ZQCS 44 69 #define EVENT_OP_IS_ZQCL 43 70 #define EVENT_OP_IS_LOAD_MODE 42 71 #define EVENT_OP_IS_SPEC_REF 41 72 #define EVENT_OP_IS_CRIT_REF 40 73 #define EVENT_OP_IS_REFRESH 39 74 #define EVENT_OP_IS_ENTER_MPSM 35 75 #define EVENT_OP_IS_ENTER_POWERDOWN 31 76 #define EVENT_OP_IS_ENTER_SELFREF 27 77 #define EVENT_WAW_HAZARD 26 78 #define EVENT_RAW_HAZARD 25 79 #define EVENT_WAR_HAZARD 24 80 #define EVENT_WRITE_COMBINE 23 81 #define EVENT_RDWR_TRANSITIONS 22 82 #define EVENT_PRECHARGE_FOR_OTHER 21 83 #define EVENT_PRECHARGE_FOR_RDWR 20 84 #define EVENT_OP_IS_PRECHARGE 19 85 #define EVENT_OP_IS_MWR 18 86 #define EVENT_OP_IS_WR 17 87 #define EVENT_OP_IS_RD 16 88 #define EVENT_OP_IS_RD_ACTIVATE 15 89 #define EVENT_OP_IS_RD_OR_WR 14 90 #define EVENT_OP_IS_ACTIVATE 13 91 #define EVENT_WR_XACT_WHEN_CRITICAL 12 92 #define EVENT_LPR_XACT_WHEN_CRITICAL 11 93 #define EVENT_HPR_XACT_WHEN_CRITICAL 10 94 #define EVENT_DFI_RD_DATA_CYCLES 9 95 #define EVENT_DFI_WR_DATA_CYCLES 8 96 #define EVENT_ACT_BYPASS 7 97 #define EVENT_READ_BYPASS 6 98 #define EVENT_HIF_HI_PRI_RD 5 99 #define EVENT_HIF_RMW 4 100 #define EVENT_HIF_RD 3 101 #define EVENT_HIF_WR 2 102 #define EVENT_HIF_RD_OR_WR 1 103 104 /* Event counter value registers */ 105 #define DDRC_PERF_CNT_VALUE_BASE 0x8080 106 #define DDRC_PERF_CNT_VALUE(n) (DDRC_PERF_CNT_VALUE_BASE + 8 * (n)) 107 108 /* Fixed event counter enable/disable register */ 109 #define DDRC_PERF_CNT_FREERUN_EN 0x80C0 110 #define DDRC_PERF_FREERUN_WRITE_EN 0x1 111 #define DDRC_PERF_FREERUN_READ_EN 0x2 112 113 /* Fixed event counter control register */ 114 #define DDRC_PERF_CNT_FREERUN_CTRL 0x80C8 115 #define DDRC_FREERUN_WRITE_CNT_CLR 0x1 116 #define DDRC_FREERUN_READ_CNT_CLR 0x2 117 118 /* Fixed event counter value register */ 119 #define DDRC_PERF_CNT_VALUE_WR_OP 0x80D0 120 #define DDRC_PERF_CNT_VALUE_RD_OP 0x80D8 121 #define DDRC_PERF_CNT_VALUE_OVERFLOW BIT_ULL(48) 122 #define DDRC_PERF_CNT_MAX_VALUE GENMASK_ULL(48, 0) 123 124 struct cn10k_ddr_pmu { 125 struct pmu pmu; 126 void __iomem *base; 127 unsigned int cpu; 128 struct device *dev; 129 int active_events; 130 struct perf_event *events[DDRC_PERF_NUM_COUNTERS]; 131 struct hrtimer hrtimer; 132 struct hlist_node node; 133 }; 134 135 #define to_cn10k_ddr_pmu(p) container_of(p, struct cn10k_ddr_pmu, pmu) 136 137 static ssize_t cn10k_ddr_pmu_event_show(struct device *dev, 138 struct device_attribute *attr, 139 char *page) 140 { 141 struct perf_pmu_events_attr *pmu_attr; 142 143 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 144 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); 145 146 } 147 148 #define CN10K_DDR_PMU_EVENT_ATTR(_name, _id) \ 149 PMU_EVENT_ATTR_ID(_name, cn10k_ddr_pmu_event_show, _id) 150 151 static struct attribute *cn10k_ddr_perf_events_attrs[] = { 152 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_or_wr_access, EVENT_HIF_RD_OR_WR), 153 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_wr_access, EVENT_HIF_WR), 154 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rd_access, EVENT_HIF_RD), 155 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_rmw_access, EVENT_HIF_RMW), 156 CN10K_DDR_PMU_EVENT_ATTR(ddr_hif_pri_rdaccess, EVENT_HIF_HI_PRI_RD), 157 CN10K_DDR_PMU_EVENT_ATTR(ddr_rd_bypass_access, EVENT_READ_BYPASS), 158 CN10K_DDR_PMU_EVENT_ATTR(ddr_act_bypass_access, EVENT_ACT_BYPASS), 159 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_wr_data_access, EVENT_DFI_WR_DATA_CYCLES), 160 CN10K_DDR_PMU_EVENT_ATTR(ddr_dif_rd_data_access, EVENT_DFI_RD_DATA_CYCLES), 161 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpri_sched_rd_crit_access, 162 EVENT_HPR_XACT_WHEN_CRITICAL), 163 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpri_sched_rd_crit_access, 164 EVENT_LPR_XACT_WHEN_CRITICAL), 165 CN10K_DDR_PMU_EVENT_ATTR(ddr_wr_trxn_crit_access, 166 EVENT_WR_XACT_WHEN_CRITICAL), 167 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_active_access, EVENT_OP_IS_ACTIVATE), 168 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_or_wr_access, EVENT_OP_IS_RD_OR_WR), 169 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_rd_active_access, EVENT_OP_IS_RD_ACTIVATE), 170 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_read, EVENT_OP_IS_RD), 171 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_write, EVENT_OP_IS_WR), 172 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_mwr, EVENT_OP_IS_MWR), 173 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge, EVENT_OP_IS_PRECHARGE), 174 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_rdwr, EVENT_PRECHARGE_FOR_RDWR), 175 CN10K_DDR_PMU_EVENT_ATTR(ddr_precharge_for_other, 176 EVENT_PRECHARGE_FOR_OTHER), 177 CN10K_DDR_PMU_EVENT_ATTR(ddr_rdwr_transitions, EVENT_RDWR_TRANSITIONS), 178 CN10K_DDR_PMU_EVENT_ATTR(ddr_write_combine, EVENT_WRITE_COMBINE), 179 CN10K_DDR_PMU_EVENT_ATTR(ddr_war_hazard, EVENT_WAR_HAZARD), 180 CN10K_DDR_PMU_EVENT_ATTR(ddr_raw_hazard, EVENT_RAW_HAZARD), 181 CN10K_DDR_PMU_EVENT_ATTR(ddr_waw_hazard, EVENT_WAW_HAZARD), 182 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_selfref, EVENT_OP_IS_ENTER_SELFREF), 183 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_powerdown, EVENT_OP_IS_ENTER_POWERDOWN), 184 CN10K_DDR_PMU_EVENT_ATTR(ddr_enter_mpsm, EVENT_OP_IS_ENTER_MPSM), 185 CN10K_DDR_PMU_EVENT_ATTR(ddr_refresh, EVENT_OP_IS_REFRESH), 186 CN10K_DDR_PMU_EVENT_ATTR(ddr_crit_ref, EVENT_OP_IS_CRIT_REF), 187 CN10K_DDR_PMU_EVENT_ATTR(ddr_spec_ref, EVENT_OP_IS_SPEC_REF), 188 CN10K_DDR_PMU_EVENT_ATTR(ddr_load_mode, EVENT_OP_IS_LOAD_MODE), 189 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqcl, EVENT_OP_IS_ZQCL), 190 CN10K_DDR_PMU_EVENT_ATTR(ddr_cam_wr_access, EVENT_OP_IS_ZQCS), 191 CN10K_DDR_PMU_EVENT_ATTR(ddr_hpr_req_with_nocredit, 192 EVENT_HPR_REQ_WITH_NOCREDIT), 193 CN10K_DDR_PMU_EVENT_ATTR(ddr_lpr_req_with_nocredit, 194 EVENT_LPR_REQ_WITH_NOCREDIT), 195 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_alloc, EVENT_BSM_ALLOC), 196 CN10K_DDR_PMU_EVENT_ATTR(ddr_bsm_starvation, EVENT_BSM_STARVATION), 197 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_rd, 198 EVENT_VISIBLE_WIN_LIMIT_REACHED_RD), 199 CN10K_DDR_PMU_EVENT_ATTR(ddr_win_limit_reached_wr, 200 EVENT_VISIBLE_WIN_LIMIT_REACHED_WR), 201 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mpc, EVENT_OP_IS_DQSOSC_MPC), 202 CN10K_DDR_PMU_EVENT_ATTR(ddr_dqsosc_mrr, EVENT_OP_IS_DQSOSC_MRR), 203 CN10K_DDR_PMU_EVENT_ATTR(ddr_tcr_mrr, EVENT_OP_IS_TCR_MRR), 204 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqstart, EVENT_OP_IS_ZQSTART), 205 CN10K_DDR_PMU_EVENT_ATTR(ddr_zqlatch, EVENT_OP_IS_ZQLATCH), 206 /* Free run event counters */ 207 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_reads, EVENT_DDR_READS), 208 CN10K_DDR_PMU_EVENT_ATTR(ddr_ddr_writes, EVENT_DDR_WRITES), 209 NULL 210 }; 211 212 static struct attribute_group cn10k_ddr_perf_events_attr_group = { 213 .name = "events", 214 .attrs = cn10k_ddr_perf_events_attrs, 215 }; 216 217 PMU_FORMAT_ATTR(event, "config:0-8"); 218 219 static struct attribute *cn10k_ddr_perf_format_attrs[] = { 220 &format_attr_event.attr, 221 NULL, 222 }; 223 224 static struct attribute_group cn10k_ddr_perf_format_attr_group = { 225 .name = "format", 226 .attrs = cn10k_ddr_perf_format_attrs, 227 }; 228 229 static ssize_t cn10k_ddr_perf_cpumask_show(struct device *dev, 230 struct device_attribute *attr, 231 char *buf) 232 { 233 struct cn10k_ddr_pmu *pmu = dev_get_drvdata(dev); 234 235 return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu)); 236 } 237 238 static struct device_attribute cn10k_ddr_perf_cpumask_attr = 239 __ATTR(cpumask, 0444, cn10k_ddr_perf_cpumask_show, NULL); 240 241 static struct attribute *cn10k_ddr_perf_cpumask_attrs[] = { 242 &cn10k_ddr_perf_cpumask_attr.attr, 243 NULL, 244 }; 245 246 static struct attribute_group cn10k_ddr_perf_cpumask_attr_group = { 247 .attrs = cn10k_ddr_perf_cpumask_attrs, 248 }; 249 250 static const struct attribute_group *cn10k_attr_groups[] = { 251 &cn10k_ddr_perf_events_attr_group, 252 &cn10k_ddr_perf_format_attr_group, 253 &cn10k_ddr_perf_cpumask_attr_group, 254 NULL, 255 }; 256 257 /* Default poll timeout is 100 sec, which is very sufficient for 258 * 48 bit counter incremented max at 5.6 GT/s, which may take many 259 * hours to overflow. 260 */ 261 static unsigned long cn10k_ddr_pmu_poll_period_sec = 100; 262 module_param_named(poll_period_sec, cn10k_ddr_pmu_poll_period_sec, ulong, 0644); 263 264 static ktime_t cn10k_ddr_pmu_timer_period(void) 265 { 266 return ms_to_ktime((u64)cn10k_ddr_pmu_poll_period_sec * USEC_PER_SEC); 267 } 268 269 static int ddr_perf_get_event_bitmap(int eventid, u64 *event_bitmap) 270 { 271 switch (eventid) { 272 case EVENT_HIF_RD_OR_WR ... EVENT_WAW_HAZARD: 273 case EVENT_OP_IS_REFRESH ... EVENT_OP_IS_ZQLATCH: 274 *event_bitmap = (1ULL << (eventid - 1)); 275 break; 276 case EVENT_OP_IS_ENTER_SELFREF: 277 case EVENT_OP_IS_ENTER_POWERDOWN: 278 case EVENT_OP_IS_ENTER_MPSM: 279 *event_bitmap = (0xFULL << (eventid - 1)); 280 break; 281 default: 282 pr_err("%s Invalid eventid %d\n", __func__, eventid); 283 return -EINVAL; 284 } 285 286 return 0; 287 } 288 289 static int cn10k_ddr_perf_alloc_counter(struct cn10k_ddr_pmu *pmu, 290 struct perf_event *event) 291 { 292 u8 config = event->attr.config; 293 int i; 294 295 /* DDR read free-run counter index */ 296 if (config == EVENT_DDR_READS) { 297 pmu->events[DDRC_PERF_READ_COUNTER_IDX] = event; 298 return DDRC_PERF_READ_COUNTER_IDX; 299 } 300 301 /* DDR write free-run counter index */ 302 if (config == EVENT_DDR_WRITES) { 303 pmu->events[DDRC_PERF_WRITE_COUNTER_IDX] = event; 304 return DDRC_PERF_WRITE_COUNTER_IDX; 305 } 306 307 /* Allocate DDR generic counters */ 308 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { 309 if (pmu->events[i] == NULL) { 310 pmu->events[i] = event; 311 return i; 312 } 313 } 314 315 return -ENOENT; 316 } 317 318 static void cn10k_ddr_perf_free_counter(struct cn10k_ddr_pmu *pmu, int counter) 319 { 320 pmu->events[counter] = NULL; 321 } 322 323 static int cn10k_ddr_perf_event_init(struct perf_event *event) 324 { 325 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 326 struct hw_perf_event *hwc = &event->hw; 327 328 if (event->attr.type != event->pmu->type) 329 return -ENOENT; 330 331 if (is_sampling_event(event)) { 332 dev_info(pmu->dev, "Sampling not supported!\n"); 333 return -EOPNOTSUPP; 334 } 335 336 if (event->cpu < 0) { 337 dev_warn(pmu->dev, "Can't provide per-task data!\n"); 338 return -EOPNOTSUPP; 339 } 340 341 /* We must NOT create groups containing mixed PMUs */ 342 if (event->group_leader->pmu != event->pmu && 343 !is_software_event(event->group_leader)) 344 return -EINVAL; 345 346 /* Set ownership of event to one CPU, same event can not be observed 347 * on multiple cpus at same time. 348 */ 349 event->cpu = pmu->cpu; 350 hwc->idx = -1; 351 return 0; 352 } 353 354 static void cn10k_ddr_perf_counter_enable(struct cn10k_ddr_pmu *pmu, 355 int counter, bool enable) 356 { 357 u32 reg; 358 u64 val; 359 360 if (counter > DDRC_PERF_NUM_COUNTERS) { 361 pr_err("Error: unsupported counter %d\n", counter); 362 return; 363 } 364 365 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) { 366 reg = DDRC_PERF_CFG(counter); 367 val = readq_relaxed(pmu->base + reg); 368 369 if (enable) 370 val |= EVENT_ENABLE; 371 else 372 val &= ~EVENT_ENABLE; 373 374 writeq_relaxed(val, pmu->base + reg); 375 } else { 376 val = readq_relaxed(pmu->base + DDRC_PERF_CNT_FREERUN_EN); 377 if (enable) { 378 if (counter == DDRC_PERF_READ_COUNTER_IDX) 379 val |= DDRC_PERF_FREERUN_READ_EN; 380 else 381 val |= DDRC_PERF_FREERUN_WRITE_EN; 382 } else { 383 if (counter == DDRC_PERF_READ_COUNTER_IDX) 384 val &= ~DDRC_PERF_FREERUN_READ_EN; 385 else 386 val &= ~DDRC_PERF_FREERUN_WRITE_EN; 387 } 388 writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_EN); 389 } 390 } 391 392 static u64 cn10k_ddr_perf_read_counter(struct cn10k_ddr_pmu *pmu, int counter) 393 { 394 u64 val; 395 396 if (counter == DDRC_PERF_READ_COUNTER_IDX) 397 return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_RD_OP); 398 399 if (counter == DDRC_PERF_WRITE_COUNTER_IDX) 400 return readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE_WR_OP); 401 402 val = readq_relaxed(pmu->base + DDRC_PERF_CNT_VALUE(counter)); 403 return val; 404 } 405 406 static void cn10k_ddr_perf_event_update(struct perf_event *event) 407 { 408 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 409 struct hw_perf_event *hwc = &event->hw; 410 u64 prev_count, new_count, mask; 411 412 do { 413 prev_count = local64_read(&hwc->prev_count); 414 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx); 415 } while (local64_xchg(&hwc->prev_count, new_count) != prev_count); 416 417 mask = DDRC_PERF_CNT_MAX_VALUE; 418 419 local64_add((new_count - prev_count) & mask, &event->count); 420 } 421 422 static void cn10k_ddr_perf_event_start(struct perf_event *event, int flags) 423 { 424 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 425 struct hw_perf_event *hwc = &event->hw; 426 int counter = hwc->idx; 427 428 local64_set(&hwc->prev_count, 0); 429 430 cn10k_ddr_perf_counter_enable(pmu, counter, true); 431 432 hwc->state = 0; 433 } 434 435 static int cn10k_ddr_perf_event_add(struct perf_event *event, int flags) 436 { 437 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 438 struct hw_perf_event *hwc = &event->hw; 439 u8 config = event->attr.config; 440 int counter, ret; 441 u32 reg_offset; 442 u64 val; 443 444 counter = cn10k_ddr_perf_alloc_counter(pmu, event); 445 if (counter < 0) 446 return -EAGAIN; 447 448 pmu->active_events++; 449 hwc->idx = counter; 450 451 if (pmu->active_events == 1) 452 hrtimer_start(&pmu->hrtimer, cn10k_ddr_pmu_timer_period(), 453 HRTIMER_MODE_REL_PINNED); 454 455 if (counter < DDRC_PERF_NUM_GEN_COUNTERS) { 456 /* Generic counters, configure event id */ 457 reg_offset = DDRC_PERF_CFG(counter); 458 ret = ddr_perf_get_event_bitmap(config, &val); 459 if (ret) 460 return ret; 461 462 writeq_relaxed(val, pmu->base + reg_offset); 463 } else { 464 /* fixed event counter, clear counter value */ 465 if (counter == DDRC_PERF_READ_COUNTER_IDX) 466 val = DDRC_FREERUN_READ_CNT_CLR; 467 else 468 val = DDRC_FREERUN_WRITE_CNT_CLR; 469 470 writeq_relaxed(val, pmu->base + DDRC_PERF_CNT_FREERUN_CTRL); 471 } 472 473 hwc->state |= PERF_HES_STOPPED; 474 475 if (flags & PERF_EF_START) 476 cn10k_ddr_perf_event_start(event, flags); 477 478 return 0; 479 } 480 481 static void cn10k_ddr_perf_event_stop(struct perf_event *event, int flags) 482 { 483 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 484 struct hw_perf_event *hwc = &event->hw; 485 int counter = hwc->idx; 486 487 cn10k_ddr_perf_counter_enable(pmu, counter, false); 488 489 if (flags & PERF_EF_UPDATE) 490 cn10k_ddr_perf_event_update(event); 491 492 hwc->state |= PERF_HES_STOPPED; 493 } 494 495 static void cn10k_ddr_perf_event_del(struct perf_event *event, int flags) 496 { 497 struct cn10k_ddr_pmu *pmu = to_cn10k_ddr_pmu(event->pmu); 498 struct hw_perf_event *hwc = &event->hw; 499 int counter = hwc->idx; 500 501 cn10k_ddr_perf_event_stop(event, PERF_EF_UPDATE); 502 503 cn10k_ddr_perf_free_counter(pmu, counter); 504 pmu->active_events--; 505 hwc->idx = -1; 506 507 /* Cancel timer when no events to capture */ 508 if (pmu->active_events == 0) 509 hrtimer_cancel(&pmu->hrtimer); 510 } 511 512 static void cn10k_ddr_perf_pmu_enable(struct pmu *pmu) 513 { 514 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu); 515 516 writeq_relaxed(START_OP_CTRL_VAL_START, ddr_pmu->base + 517 DDRC_PERF_CNT_START_OP_CTRL); 518 } 519 520 static void cn10k_ddr_perf_pmu_disable(struct pmu *pmu) 521 { 522 struct cn10k_ddr_pmu *ddr_pmu = to_cn10k_ddr_pmu(pmu); 523 524 writeq_relaxed(END_OP_CTRL_VAL_END, ddr_pmu->base + 525 DDRC_PERF_CNT_END_OP_CTRL); 526 } 527 528 static void cn10k_ddr_perf_event_update_all(struct cn10k_ddr_pmu *pmu) 529 { 530 struct hw_perf_event *hwc; 531 int i; 532 533 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { 534 if (pmu->events[i] == NULL) 535 continue; 536 537 cn10k_ddr_perf_event_update(pmu->events[i]); 538 } 539 540 /* Reset previous count as h/w counter are reset */ 541 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { 542 if (pmu->events[i] == NULL) 543 continue; 544 545 hwc = &pmu->events[i]->hw; 546 local64_set(&hwc->prev_count, 0); 547 } 548 } 549 550 static irqreturn_t cn10k_ddr_pmu_overflow_handler(struct cn10k_ddr_pmu *pmu) 551 { 552 struct perf_event *event; 553 struct hw_perf_event *hwc; 554 u64 prev_count, new_count; 555 u64 value; 556 int i; 557 558 event = pmu->events[DDRC_PERF_READ_COUNTER_IDX]; 559 if (event) { 560 hwc = &event->hw; 561 prev_count = local64_read(&hwc->prev_count); 562 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx); 563 564 /* Overflow condition is when new count less than 565 * previous count 566 */ 567 if (new_count < prev_count) 568 cn10k_ddr_perf_event_update(event); 569 } 570 571 event = pmu->events[DDRC_PERF_WRITE_COUNTER_IDX]; 572 if (event) { 573 hwc = &event->hw; 574 prev_count = local64_read(&hwc->prev_count); 575 new_count = cn10k_ddr_perf_read_counter(pmu, hwc->idx); 576 577 /* Overflow condition is when new count less than 578 * previous count 579 */ 580 if (new_count < prev_count) 581 cn10k_ddr_perf_event_update(event); 582 } 583 584 for (i = 0; i < DDRC_PERF_NUM_GEN_COUNTERS; i++) { 585 if (pmu->events[i] == NULL) 586 continue; 587 588 value = cn10k_ddr_perf_read_counter(pmu, i); 589 if (value == DDRC_PERF_CNT_MAX_VALUE) { 590 pr_info("Counter-(%d) reached max value\n", i); 591 cn10k_ddr_perf_event_update_all(pmu); 592 cn10k_ddr_perf_pmu_disable(&pmu->pmu); 593 cn10k_ddr_perf_pmu_enable(&pmu->pmu); 594 } 595 } 596 597 return IRQ_HANDLED; 598 } 599 600 static enum hrtimer_restart cn10k_ddr_pmu_timer_handler(struct hrtimer *hrtimer) 601 { 602 struct cn10k_ddr_pmu *pmu = container_of(hrtimer, struct cn10k_ddr_pmu, 603 hrtimer); 604 unsigned long flags; 605 606 local_irq_save(flags); 607 cn10k_ddr_pmu_overflow_handler(pmu); 608 local_irq_restore(flags); 609 610 hrtimer_forward_now(hrtimer, cn10k_ddr_pmu_timer_period()); 611 return HRTIMER_RESTART; 612 } 613 614 static int cn10k_ddr_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 615 { 616 struct cn10k_ddr_pmu *pmu = hlist_entry_safe(node, struct cn10k_ddr_pmu, 617 node); 618 unsigned int target; 619 620 if (cpu != pmu->cpu) 621 return 0; 622 623 target = cpumask_any_but(cpu_online_mask, cpu); 624 if (target >= nr_cpu_ids) 625 return 0; 626 627 perf_pmu_migrate_context(&pmu->pmu, cpu, target); 628 pmu->cpu = target; 629 return 0; 630 } 631 632 static int cn10k_ddr_perf_probe(struct platform_device *pdev) 633 { 634 struct cn10k_ddr_pmu *ddr_pmu; 635 struct resource *res; 636 void __iomem *base; 637 char *name; 638 int ret; 639 640 ddr_pmu = devm_kzalloc(&pdev->dev, sizeof(*ddr_pmu), GFP_KERNEL); 641 if (!ddr_pmu) 642 return -ENOMEM; 643 644 ddr_pmu->dev = &pdev->dev; 645 platform_set_drvdata(pdev, ddr_pmu); 646 647 base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 648 if (IS_ERR(base)) 649 return PTR_ERR(base); 650 651 ddr_pmu->base = base; 652 653 /* Setup the PMU counter to work in manual mode */ 654 writeq_relaxed(OP_MODE_CTRL_VAL_MANNUAL, ddr_pmu->base + 655 DDRC_PERF_CNT_OP_MODE_CTRL); 656 657 ddr_pmu->pmu = (struct pmu) { 658 .module = THIS_MODULE, 659 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 660 .task_ctx_nr = perf_invalid_context, 661 .attr_groups = cn10k_attr_groups, 662 .event_init = cn10k_ddr_perf_event_init, 663 .add = cn10k_ddr_perf_event_add, 664 .del = cn10k_ddr_perf_event_del, 665 .start = cn10k_ddr_perf_event_start, 666 .stop = cn10k_ddr_perf_event_stop, 667 .read = cn10k_ddr_perf_event_update, 668 .pmu_enable = cn10k_ddr_perf_pmu_enable, 669 .pmu_disable = cn10k_ddr_perf_pmu_disable, 670 }; 671 672 /* Choose this cpu to collect perf data */ 673 ddr_pmu->cpu = raw_smp_processor_id(); 674 675 name = devm_kasprintf(ddr_pmu->dev, GFP_KERNEL, "mrvl_ddr_pmu_%llx", 676 res->start); 677 if (!name) 678 return -ENOMEM; 679 680 hrtimer_init(&ddr_pmu->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 681 ddr_pmu->hrtimer.function = cn10k_ddr_pmu_timer_handler; 682 683 cpuhp_state_add_instance_nocalls( 684 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, 685 &ddr_pmu->node); 686 687 ret = perf_pmu_register(&ddr_pmu->pmu, name, -1); 688 if (ret) 689 goto error; 690 691 pr_info("CN10K DDR PMU Driver for ddrc@%llx\n", res->start); 692 return 0; 693 error: 694 cpuhp_state_remove_instance_nocalls( 695 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, 696 &ddr_pmu->node); 697 return ret; 698 } 699 700 static int cn10k_ddr_perf_remove(struct platform_device *pdev) 701 { 702 struct cn10k_ddr_pmu *ddr_pmu = platform_get_drvdata(pdev); 703 704 cpuhp_state_remove_instance_nocalls( 705 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, 706 &ddr_pmu->node); 707 708 perf_pmu_unregister(&ddr_pmu->pmu); 709 return 0; 710 } 711 712 #ifdef CONFIG_OF 713 static const struct of_device_id cn10k_ddr_pmu_of_match[] = { 714 { .compatible = "marvell,cn10k-ddr-pmu", }, 715 { }, 716 }; 717 MODULE_DEVICE_TABLE(of, cn10k_ddr_pmu_of_match); 718 #endif 719 720 static struct platform_driver cn10k_ddr_pmu_driver = { 721 .driver = { 722 .name = "cn10k-ddr-pmu", 723 .of_match_table = of_match_ptr(cn10k_ddr_pmu_of_match), 724 .suppress_bind_attrs = true, 725 }, 726 .probe = cn10k_ddr_perf_probe, 727 .remove = cn10k_ddr_perf_remove, 728 }; 729 730 static int __init cn10k_ddr_pmu_init(void) 731 { 732 int ret; 733 734 ret = cpuhp_setup_state_multi( 735 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE, 736 "perf/marvell/cn10k/ddr:online", NULL, 737 cn10k_ddr_pmu_offline_cpu); 738 if (ret) 739 return ret; 740 741 ret = platform_driver_register(&cn10k_ddr_pmu_driver); 742 if (ret) 743 cpuhp_remove_multi_state( 744 CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE); 745 return ret; 746 } 747 748 static void __exit cn10k_ddr_pmu_exit(void) 749 { 750 platform_driver_unregister(&cn10k_ddr_pmu_driver); 751 cpuhp_remove_multi_state(CPUHP_AP_PERF_ARM_MARVELL_CN10K_DDR_ONLINE); 752 } 753 754 module_init(cn10k_ddr_pmu_init); 755 module_exit(cn10k_ddr_pmu_exit); 756 757 MODULE_AUTHOR("Bharat Bhushan <bbhushan2@marvell.com>"); 758 MODULE_LICENSE("GPL v2"); 759