1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell CN10K LLC-TAD perf driver 3 * 4 * Copyright (C) 2021 Marvell 5 */ 6 7 #define pr_fmt(fmt) "tad_pmu: " fmt 8 9 #include <linux/io.h> 10 #include <linux/module.h> 11 #include <linux/of.h> 12 #include <linux/cpuhotplug.h> 13 #include <linux/perf_event.h> 14 #include <linux/platform_device.h> 15 #include <linux/acpi.h> 16 17 #define TAD_PFC_OFFSET 0x800 18 #define TAD_PFC(counter) (TAD_PFC_OFFSET | (counter << 3)) 19 #define TAD_PRF_OFFSET 0x900 20 #define TAD_PRF(counter) (TAD_PRF_OFFSET | (counter << 3)) 21 #define TAD_PRF_CNTSEL_MASK 0xFF 22 #define TAD_MAX_COUNTERS 8 23 24 #define to_tad_pmu(p) (container_of(p, struct tad_pmu, pmu)) 25 26 struct tad_region { 27 void __iomem *base; 28 }; 29 30 struct tad_pmu { 31 struct pmu pmu; 32 struct tad_region *regions; 33 u32 region_cnt; 34 unsigned int cpu; 35 struct hlist_node node; 36 struct perf_event *events[TAD_MAX_COUNTERS]; 37 DECLARE_BITMAP(counters_map, TAD_MAX_COUNTERS); 38 }; 39 40 enum mrvl_tad_pmu_version { 41 TAD_PMU_V1 = 1, 42 TAD_PMU_V2, 43 }; 44 45 struct tad_pmu_data { 46 int id; 47 }; 48 49 static int tad_pmu_cpuhp_state; 50 51 static void tad_pmu_event_counter_read(struct perf_event *event) 52 { 53 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 54 struct hw_perf_event *hwc = &event->hw; 55 u32 counter_idx = hwc->idx; 56 u64 prev, new; 57 int i; 58 59 do { 60 prev = local64_read(&hwc->prev_count); 61 for (i = 0, new = 0; i < tad_pmu->region_cnt; i++) 62 new += readq(tad_pmu->regions[i].base + 63 TAD_PFC(counter_idx)); 64 } while (local64_cmpxchg(&hwc->prev_count, prev, new) != prev); 65 66 local64_add(new - prev, &event->count); 67 } 68 69 static void tad_pmu_event_counter_stop(struct perf_event *event, int flags) 70 { 71 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 72 struct hw_perf_event *hwc = &event->hw; 73 u32 counter_idx = hwc->idx; 74 int i; 75 76 /* TAD()_PFC() stop counting on the write 77 * which sets TAD()_PRF()[CNTSEL] == 0 78 */ 79 for (i = 0; i < tad_pmu->region_cnt; i++) { 80 writeq_relaxed(0, tad_pmu->regions[i].base + 81 TAD_PRF(counter_idx)); 82 } 83 84 tad_pmu_event_counter_read(event); 85 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 86 } 87 88 static void tad_pmu_event_counter_start(struct perf_event *event, int flags) 89 { 90 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 91 struct hw_perf_event *hwc = &event->hw; 92 u32 event_idx = event->attr.config; 93 u32 counter_idx = hwc->idx; 94 u64 reg_val; 95 int i; 96 97 hwc->state = 0; 98 99 /* Typically TAD_PFC() are zeroed to start counting */ 100 for (i = 0; i < tad_pmu->region_cnt; i++) 101 writeq_relaxed(0, tad_pmu->regions[i].base + 102 TAD_PFC(counter_idx)); 103 104 /* TAD()_PFC() start counting on the write 105 * which sets TAD()_PRF()[CNTSEL] != 0 106 */ 107 for (i = 0; i < tad_pmu->region_cnt; i++) { 108 reg_val = event_idx & 0xFF; 109 writeq_relaxed(reg_val, tad_pmu->regions[i].base + 110 TAD_PRF(counter_idx)); 111 } 112 } 113 114 static void tad_pmu_event_counter_del(struct perf_event *event, int flags) 115 { 116 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 117 struct hw_perf_event *hwc = &event->hw; 118 int idx = hwc->idx; 119 120 tad_pmu_event_counter_stop(event, flags | PERF_EF_UPDATE); 121 tad_pmu->events[idx] = NULL; 122 clear_bit(idx, tad_pmu->counters_map); 123 } 124 125 static int tad_pmu_event_counter_add(struct perf_event *event, int flags) 126 { 127 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 128 struct hw_perf_event *hwc = &event->hw; 129 int idx; 130 131 /* Get a free counter for this event */ 132 idx = find_first_zero_bit(tad_pmu->counters_map, TAD_MAX_COUNTERS); 133 if (idx == TAD_MAX_COUNTERS) 134 return -EAGAIN; 135 136 set_bit(idx, tad_pmu->counters_map); 137 138 hwc->idx = idx; 139 hwc->state = PERF_HES_STOPPED; 140 tad_pmu->events[idx] = event; 141 142 if (flags & PERF_EF_START) 143 tad_pmu_event_counter_start(event, flags); 144 145 return 0; 146 } 147 148 static int tad_pmu_event_init(struct perf_event *event) 149 { 150 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 151 152 if (event->attr.type != event->pmu->type) 153 return -ENOENT; 154 155 if (!event->attr.disabled) 156 return -EINVAL; 157 158 if (event->state != PERF_EVENT_STATE_OFF) 159 return -EINVAL; 160 161 event->cpu = tad_pmu->cpu; 162 event->hw.idx = -1; 163 event->hw.config_base = event->attr.config; 164 165 return 0; 166 } 167 168 static ssize_t tad_pmu_event_show(struct device *dev, 169 struct device_attribute *attr, char *page) 170 { 171 struct perf_pmu_events_attr *pmu_attr; 172 173 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 174 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); 175 } 176 177 #define TAD_PMU_EVENT_ATTR(name, config) \ 178 PMU_EVENT_ATTR_ID(name, tad_pmu_event_show, config) 179 180 static struct attribute *tad_pmu_event_attrs[] = { 181 TAD_PMU_EVENT_ATTR(tad_none, 0x0), 182 TAD_PMU_EVENT_ATTR(tad_req_msh_in_any, 0x1), 183 TAD_PMU_EVENT_ATTR(tad_req_msh_in_mn, 0x2), 184 TAD_PMU_EVENT_ATTR(tad_req_msh_in_exlmn, 0x3), 185 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_any, 0x4), 186 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_mn, 0x5), 187 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_exlmn, 0x6), 188 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_dss, 0x7), 189 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_retry_dss, 0x8), 190 TAD_PMU_EVENT_ATTR(tad_dat_msh_in_any, 0x9), 191 TAD_PMU_EVENT_ATTR(tad_dat_msh_in_dss, 0xa), 192 TAD_PMU_EVENT_ATTR(tad_req_msh_out_any, 0xb), 193 TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_rd, 0xc), 194 TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_wr, 0xd), 195 TAD_PMU_EVENT_ATTR(tad_req_msh_out_evict, 0xe), 196 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_any, 0xf), 197 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_exlmn, 0x10), 198 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_mn, 0x11), 199 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_exlmn, 0x12), 200 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_mn, 0x13), 201 TAD_PMU_EVENT_ATTR(tad_snp_msh_out_any, 0x14), 202 TAD_PMU_EVENT_ATTR(tad_snp_msh_out_mn, 0x15), 203 TAD_PMU_EVENT_ATTR(tad_snp_msh_out_exlmn, 0x16), 204 TAD_PMU_EVENT_ATTR(tad_dat_msh_out_any, 0x17), 205 TAD_PMU_EVENT_ATTR(tad_dat_msh_out_fill, 0x18), 206 TAD_PMU_EVENT_ATTR(tad_dat_msh_out_dss, 0x19), 207 TAD_PMU_EVENT_ATTR(tad_alloc_dtg, 0x1a), 208 TAD_PMU_EVENT_ATTR(tad_alloc_ltg, 0x1b), 209 TAD_PMU_EVENT_ATTR(tad_alloc_any, 0x1c), 210 TAD_PMU_EVENT_ATTR(tad_hit_dtg, 0x1d), 211 TAD_PMU_EVENT_ATTR(tad_hit_ltg, 0x1e), 212 TAD_PMU_EVENT_ATTR(tad_hit_any, 0x1f), 213 TAD_PMU_EVENT_ATTR(tad_tag_rd, 0x20), 214 TAD_PMU_EVENT_ATTR(tad_dat_rd, 0x21), 215 TAD_PMU_EVENT_ATTR(tad_dat_rd_byp, 0x22), 216 TAD_PMU_EVENT_ATTR(tad_ifb_occ, 0x23), 217 TAD_PMU_EVENT_ATTR(tad_req_occ, 0x24), 218 NULL 219 }; 220 221 static const struct attribute_group tad_pmu_events_attr_group = { 222 .name = "events", 223 .attrs = tad_pmu_event_attrs, 224 }; 225 226 static struct attribute *ody_tad_pmu_event_attrs[] = { 227 TAD_PMU_EVENT_ATTR(tad_req_msh_in_exlmn, 0x3), 228 TAD_PMU_EVENT_ATTR(tad_alloc_dtg, 0x1a), 229 TAD_PMU_EVENT_ATTR(tad_alloc_ltg, 0x1b), 230 TAD_PMU_EVENT_ATTR(tad_alloc_any, 0x1c), 231 TAD_PMU_EVENT_ATTR(tad_hit_dtg, 0x1d), 232 TAD_PMU_EVENT_ATTR(tad_hit_ltg, 0x1e), 233 TAD_PMU_EVENT_ATTR(tad_hit_any, 0x1f), 234 TAD_PMU_EVENT_ATTR(tad_tag_rd, 0x20), 235 TAD_PMU_EVENT_ATTR(tad_tot_cycle, 0xFF), 236 NULL 237 }; 238 239 static const struct attribute_group ody_tad_pmu_events_attr_group = { 240 .name = "events", 241 .attrs = ody_tad_pmu_event_attrs, 242 }; 243 244 PMU_FORMAT_ATTR(event, "config:0-7"); 245 246 static struct attribute *tad_pmu_format_attrs[] = { 247 &format_attr_event.attr, 248 NULL 249 }; 250 251 static struct attribute_group tad_pmu_format_attr_group = { 252 .name = "format", 253 .attrs = tad_pmu_format_attrs, 254 }; 255 256 static ssize_t tad_pmu_cpumask_show(struct device *dev, 257 struct device_attribute *attr, char *buf) 258 { 259 struct tad_pmu *tad_pmu = to_tad_pmu(dev_get_drvdata(dev)); 260 261 return cpumap_print_to_pagebuf(true, buf, cpumask_of(tad_pmu->cpu)); 262 } 263 264 static DEVICE_ATTR(cpumask, 0444, tad_pmu_cpumask_show, NULL); 265 266 static struct attribute *tad_pmu_cpumask_attrs[] = { 267 &dev_attr_cpumask.attr, 268 NULL 269 }; 270 271 static struct attribute_group tad_pmu_cpumask_attr_group = { 272 .attrs = tad_pmu_cpumask_attrs, 273 }; 274 275 static const struct attribute_group *tad_pmu_attr_groups[] = { 276 &tad_pmu_events_attr_group, 277 &tad_pmu_format_attr_group, 278 &tad_pmu_cpumask_attr_group, 279 NULL 280 }; 281 282 static const struct attribute_group *ody_tad_pmu_attr_groups[] = { 283 &ody_tad_pmu_events_attr_group, 284 &tad_pmu_format_attr_group, 285 &tad_pmu_cpumask_attr_group, 286 NULL 287 }; 288 289 static int tad_pmu_probe(struct platform_device *pdev) 290 { 291 const struct tad_pmu_data *dev_data; 292 struct device *dev = &pdev->dev; 293 struct tad_region *regions; 294 struct tad_pmu *tad_pmu; 295 struct resource *res; 296 u32 tad_pmu_page_size; 297 u32 tad_page_size; 298 u32 tad_cnt; 299 int version; 300 int i, ret; 301 char *name; 302 303 tad_pmu = devm_kzalloc(&pdev->dev, sizeof(*tad_pmu), GFP_KERNEL); 304 if (!tad_pmu) 305 return -ENOMEM; 306 307 platform_set_drvdata(pdev, tad_pmu); 308 309 dev_data = device_get_match_data(&pdev->dev); 310 if (!dev_data) { 311 dev_err(&pdev->dev, "Error: No device match data found\n"); 312 return -ENODEV; 313 } 314 version = dev_data->id; 315 316 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 317 if (!res) { 318 dev_err(&pdev->dev, "Mem resource not found\n"); 319 return -ENODEV; 320 } 321 322 ret = device_property_read_u32(dev, "marvell,tad-page-size", 323 &tad_page_size); 324 if (ret) { 325 dev_err(&pdev->dev, "Can't find tad-page-size property\n"); 326 return ret; 327 } 328 329 ret = device_property_read_u32(dev, "marvell,tad-pmu-page-size", 330 &tad_pmu_page_size); 331 if (ret) { 332 dev_err(&pdev->dev, "Can't find tad-pmu-page-size property\n"); 333 return ret; 334 } 335 336 ret = device_property_read_u32(dev, "marvell,tad-cnt", &tad_cnt); 337 if (ret) { 338 dev_err(&pdev->dev, "Can't find tad-cnt property\n"); 339 return ret; 340 } 341 342 regions = devm_kcalloc(&pdev->dev, tad_cnt, 343 sizeof(*regions), GFP_KERNEL); 344 if (!regions) 345 return -ENOMEM; 346 347 /* ioremap the distributed TAD pmu regions */ 348 for (i = 0; i < tad_cnt && res->start < res->end; i++) { 349 regions[i].base = devm_ioremap(&pdev->dev, 350 res->start, 351 tad_pmu_page_size); 352 if (!regions[i].base) { 353 dev_err(&pdev->dev, "TAD%d ioremap fail\n", i); 354 return -ENOMEM; 355 } 356 res->start += tad_page_size; 357 } 358 359 tad_pmu->regions = regions; 360 tad_pmu->region_cnt = tad_cnt; 361 362 tad_pmu->pmu = (struct pmu) { 363 364 .module = THIS_MODULE, 365 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | 366 PERF_PMU_CAP_NO_INTERRUPT, 367 .task_ctx_nr = perf_invalid_context, 368 369 .event_init = tad_pmu_event_init, 370 .add = tad_pmu_event_counter_add, 371 .del = tad_pmu_event_counter_del, 372 .start = tad_pmu_event_counter_start, 373 .stop = tad_pmu_event_counter_stop, 374 .read = tad_pmu_event_counter_read, 375 }; 376 377 if (version == TAD_PMU_V1) 378 tad_pmu->pmu.attr_groups = tad_pmu_attr_groups; 379 else 380 tad_pmu->pmu.attr_groups = ody_tad_pmu_attr_groups; 381 382 tad_pmu->cpu = raw_smp_processor_id(); 383 384 /* Register pmu instance for cpu hotplug */ 385 ret = cpuhp_state_add_instance_nocalls(tad_pmu_cpuhp_state, 386 &tad_pmu->node); 387 if (ret) { 388 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret); 389 return ret; 390 } 391 392 name = "tad"; 393 ret = perf_pmu_register(&tad_pmu->pmu, name, -1); 394 if (ret) 395 cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state, 396 &tad_pmu->node); 397 398 return ret; 399 } 400 401 static void tad_pmu_remove(struct platform_device *pdev) 402 { 403 struct tad_pmu *pmu = platform_get_drvdata(pdev); 404 405 cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state, 406 &pmu->node); 407 perf_pmu_unregister(&pmu->pmu); 408 } 409 410 #if defined(CONFIG_OF) || defined(CONFIG_ACPI) 411 static const struct tad_pmu_data tad_pmu_data = { 412 .id = TAD_PMU_V1, 413 }; 414 #endif 415 416 #ifdef CONFIG_ACPI 417 static const struct tad_pmu_data tad_pmu_v2_data = { 418 .id = TAD_PMU_V2, 419 }; 420 #endif 421 422 #ifdef CONFIG_OF 423 static const struct of_device_id tad_pmu_of_match[] = { 424 { .compatible = "marvell,cn10k-tad-pmu", .data = &tad_pmu_data }, 425 {}, 426 }; 427 #endif 428 429 #ifdef CONFIG_ACPI 430 static const struct acpi_device_id tad_pmu_acpi_match[] = { 431 {"MRVL000B", (kernel_ulong_t)&tad_pmu_data}, 432 {"MRVL000D", (kernel_ulong_t)&tad_pmu_v2_data}, 433 {}, 434 }; 435 MODULE_DEVICE_TABLE(acpi, tad_pmu_acpi_match); 436 #endif 437 438 static struct platform_driver tad_pmu_driver = { 439 .driver = { 440 .name = "cn10k_tad_pmu", 441 .of_match_table = of_match_ptr(tad_pmu_of_match), 442 .acpi_match_table = ACPI_PTR(tad_pmu_acpi_match), 443 .suppress_bind_attrs = true, 444 }, 445 .probe = tad_pmu_probe, 446 .remove = tad_pmu_remove, 447 }; 448 449 static int tad_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 450 { 451 struct tad_pmu *pmu = hlist_entry_safe(node, struct tad_pmu, node); 452 unsigned int target; 453 454 if (cpu != pmu->cpu) 455 return 0; 456 457 target = cpumask_any_but(cpu_online_mask, cpu); 458 if (target >= nr_cpu_ids) 459 return 0; 460 461 perf_pmu_migrate_context(&pmu->pmu, cpu, target); 462 pmu->cpu = target; 463 464 return 0; 465 } 466 467 static int __init tad_pmu_init(void) 468 { 469 int ret; 470 471 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 472 "perf/cn10k/tadpmu:online", 473 NULL, 474 tad_pmu_offline_cpu); 475 if (ret < 0) 476 return ret; 477 tad_pmu_cpuhp_state = ret; 478 ret = platform_driver_register(&tad_pmu_driver); 479 if (ret) 480 cpuhp_remove_multi_state(tad_pmu_cpuhp_state); 481 482 return ret; 483 } 484 485 static void __exit tad_pmu_exit(void) 486 { 487 platform_driver_unregister(&tad_pmu_driver); 488 cpuhp_remove_multi_state(tad_pmu_cpuhp_state); 489 } 490 491 module_init(tad_pmu_init); 492 module_exit(tad_pmu_exit); 493 494 MODULE_DESCRIPTION("Marvell CN10K LLC-TAD Perf driver"); 495 MODULE_AUTHOR("Bhaskara Budiredla <bbudiredla@marvell.com>"); 496 MODULE_LICENSE("GPL v2"); 497