1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell CN10K LLC-TAD perf driver 3 * 4 * Copyright (C) 2021 Marvell 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11 #define pr_fmt(fmt) "tad_pmu: " fmt 12 13 #include <linux/module.h> 14 #include <linux/of.h> 15 #include <linux/of_address.h> 16 #include <linux/of_device.h> 17 #include <linux/cpuhotplug.h> 18 #include <linux/perf_event.h> 19 #include <linux/platform_device.h> 20 21 #define TAD_PFC_OFFSET 0x0 22 #define TAD_PFC(counter) (TAD_PFC_OFFSET | (counter << 3)) 23 #define TAD_PRF_OFFSET 0x100 24 #define TAD_PRF(counter) (TAD_PRF_OFFSET | (counter << 3)) 25 #define TAD_PRF_CNTSEL_MASK 0xFF 26 #define TAD_MAX_COUNTERS 8 27 28 #define to_tad_pmu(p) (container_of(p, struct tad_pmu, pmu)) 29 30 struct tad_region { 31 void __iomem *base; 32 }; 33 34 struct tad_pmu { 35 struct pmu pmu; 36 struct tad_region *regions; 37 u32 region_cnt; 38 unsigned int cpu; 39 struct hlist_node node; 40 struct perf_event *events[TAD_MAX_COUNTERS]; 41 DECLARE_BITMAP(counters_map, TAD_MAX_COUNTERS); 42 }; 43 44 static int tad_pmu_cpuhp_state; 45 46 static void tad_pmu_event_counter_read(struct perf_event *event) 47 { 48 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 49 struct hw_perf_event *hwc = &event->hw; 50 u32 counter_idx = hwc->idx; 51 u64 prev, new; 52 int i; 53 54 do { 55 prev = local64_read(&hwc->prev_count); 56 for (i = 0, new = 0; i < tad_pmu->region_cnt; i++) 57 new += readq(tad_pmu->regions[i].base + 58 TAD_PFC(counter_idx)); 59 } while (local64_cmpxchg(&hwc->prev_count, prev, new) != prev); 60 61 local64_add(new - prev, &event->count); 62 } 63 64 static void tad_pmu_event_counter_stop(struct perf_event *event, int flags) 65 { 66 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 67 struct hw_perf_event *hwc = &event->hw; 68 u32 counter_idx = hwc->idx; 69 int i; 70 71 /* TAD()_PFC() stop counting on the write 72 * which sets TAD()_PRF()[CNTSEL] == 0 73 */ 74 for (i = 0; i < tad_pmu->region_cnt; i++) { 75 writeq_relaxed(0, tad_pmu->regions[i].base + 76 TAD_PRF(counter_idx)); 77 } 78 79 tad_pmu_event_counter_read(event); 80 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 81 } 82 83 static void tad_pmu_event_counter_start(struct perf_event *event, int flags) 84 { 85 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 86 struct hw_perf_event *hwc = &event->hw; 87 u32 event_idx = event->attr.config; 88 u32 counter_idx = hwc->idx; 89 u64 reg_val; 90 int i; 91 92 hwc->state = 0; 93 94 /* Typically TAD_PFC() are zeroed to start counting */ 95 for (i = 0; i < tad_pmu->region_cnt; i++) 96 writeq_relaxed(0, tad_pmu->regions[i].base + 97 TAD_PFC(counter_idx)); 98 99 /* TAD()_PFC() start counting on the write 100 * which sets TAD()_PRF()[CNTSEL] != 0 101 */ 102 for (i = 0; i < tad_pmu->region_cnt; i++) { 103 reg_val = readq_relaxed(tad_pmu->regions[i].base + 104 TAD_PRF(counter_idx)); 105 reg_val |= (event_idx & 0xFF); 106 writeq_relaxed(reg_val, tad_pmu->regions[i].base + 107 TAD_PRF(counter_idx)); 108 } 109 } 110 111 static void tad_pmu_event_counter_del(struct perf_event *event, int flags) 112 { 113 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 114 struct hw_perf_event *hwc = &event->hw; 115 int idx = hwc->idx; 116 117 tad_pmu_event_counter_stop(event, flags | PERF_EF_UPDATE); 118 tad_pmu->events[idx] = NULL; 119 clear_bit(idx, tad_pmu->counters_map); 120 } 121 122 static int tad_pmu_event_counter_add(struct perf_event *event, int flags) 123 { 124 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 125 struct hw_perf_event *hwc = &event->hw; 126 int idx; 127 128 /* Get a free counter for this event */ 129 idx = find_first_zero_bit(tad_pmu->counters_map, TAD_MAX_COUNTERS); 130 if (idx == TAD_MAX_COUNTERS) 131 return -EAGAIN; 132 133 set_bit(idx, tad_pmu->counters_map); 134 135 hwc->idx = idx; 136 hwc->state = PERF_HES_STOPPED; 137 tad_pmu->events[idx] = event; 138 139 if (flags & PERF_EF_START) 140 tad_pmu_event_counter_start(event, flags); 141 142 return 0; 143 } 144 145 static int tad_pmu_event_init(struct perf_event *event) 146 { 147 struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu); 148 149 if (!event->attr.disabled) 150 return -EINVAL; 151 152 if (event->attr.type != event->pmu->type) 153 return -ENOENT; 154 155 if (event->state != PERF_EVENT_STATE_OFF) 156 return -EINVAL; 157 158 event->cpu = tad_pmu->cpu; 159 event->hw.idx = -1; 160 event->hw.config_base = event->attr.config; 161 162 return 0; 163 } 164 165 static ssize_t tad_pmu_event_show(struct device *dev, 166 struct device_attribute *attr, char *page) 167 { 168 struct perf_pmu_events_attr *pmu_attr; 169 170 pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr); 171 return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); 172 } 173 174 #define TAD_PMU_EVENT_ATTR(name, config) \ 175 PMU_EVENT_ATTR_ID(name, tad_pmu_event_show, config) 176 177 static struct attribute *tad_pmu_event_attrs[] = { 178 TAD_PMU_EVENT_ATTR(tad_none, 0x0), 179 TAD_PMU_EVENT_ATTR(tad_req_msh_in_any, 0x1), 180 TAD_PMU_EVENT_ATTR(tad_req_msh_in_mn, 0x2), 181 TAD_PMU_EVENT_ATTR(tad_req_msh_in_exlmn, 0x3), 182 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_any, 0x4), 183 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_mn, 0x5), 184 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_exlmn, 0x6), 185 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_dss, 0x7), 186 TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_retry_dss, 0x8), 187 TAD_PMU_EVENT_ATTR(tad_dat_msh_in_any, 0x9), 188 TAD_PMU_EVENT_ATTR(tad_dat_msh_in_dss, 0xa), 189 TAD_PMU_EVENT_ATTR(tad_req_msh_out_any, 0xb), 190 TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_rd, 0xc), 191 TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_wr, 0xd), 192 TAD_PMU_EVENT_ATTR(tad_req_msh_out_evict, 0xe), 193 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_any, 0xf), 194 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_exlmn, 0x10), 195 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_mn, 0x11), 196 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_exlmn, 0x12), 197 TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_mn, 0x13), 198 TAD_PMU_EVENT_ATTR(tad_snp_msh_out_any, 0x14), 199 TAD_PMU_EVENT_ATTR(tad_snp_msh_out_mn, 0x15), 200 TAD_PMU_EVENT_ATTR(tad_snp_msh_out_exlmn, 0x16), 201 TAD_PMU_EVENT_ATTR(tad_dat_msh_out_any, 0x17), 202 TAD_PMU_EVENT_ATTR(tad_dat_msh_out_fill, 0x18), 203 TAD_PMU_EVENT_ATTR(tad_dat_msh_out_dss, 0x19), 204 TAD_PMU_EVENT_ATTR(tad_alloc_dtg, 0x1a), 205 TAD_PMU_EVENT_ATTR(tad_alloc_ltg, 0x1b), 206 TAD_PMU_EVENT_ATTR(tad_alloc_any, 0x1c), 207 TAD_PMU_EVENT_ATTR(tad_hit_dtg, 0x1d), 208 TAD_PMU_EVENT_ATTR(tad_hit_ltg, 0x1e), 209 TAD_PMU_EVENT_ATTR(tad_hit_any, 0x1f), 210 TAD_PMU_EVENT_ATTR(tad_tag_rd, 0x20), 211 TAD_PMU_EVENT_ATTR(tad_dat_rd, 0x21), 212 TAD_PMU_EVENT_ATTR(tad_dat_rd_byp, 0x22), 213 TAD_PMU_EVENT_ATTR(tad_ifb_occ, 0x23), 214 TAD_PMU_EVENT_ATTR(tad_req_occ, 0x24), 215 NULL 216 }; 217 218 static const struct attribute_group tad_pmu_events_attr_group = { 219 .name = "events", 220 .attrs = tad_pmu_event_attrs, 221 }; 222 223 PMU_FORMAT_ATTR(event, "config:0-7"); 224 225 static struct attribute *tad_pmu_format_attrs[] = { 226 &format_attr_event.attr, 227 NULL 228 }; 229 230 static struct attribute_group tad_pmu_format_attr_group = { 231 .name = "format", 232 .attrs = tad_pmu_format_attrs, 233 }; 234 235 static ssize_t tad_pmu_cpumask_show(struct device *dev, 236 struct device_attribute *attr, char *buf) 237 { 238 struct tad_pmu *tad_pmu = to_tad_pmu(dev_get_drvdata(dev)); 239 240 return cpumap_print_to_pagebuf(true, buf, cpumask_of(tad_pmu->cpu)); 241 } 242 243 static DEVICE_ATTR(cpumask, 0444, tad_pmu_cpumask_show, NULL); 244 245 static struct attribute *tad_pmu_cpumask_attrs[] = { 246 &dev_attr_cpumask.attr, 247 NULL 248 }; 249 250 static struct attribute_group tad_pmu_cpumask_attr_group = { 251 .attrs = tad_pmu_cpumask_attrs, 252 }; 253 254 static const struct attribute_group *tad_pmu_attr_groups[] = { 255 &tad_pmu_events_attr_group, 256 &tad_pmu_format_attr_group, 257 &tad_pmu_cpumask_attr_group, 258 NULL 259 }; 260 261 static int tad_pmu_probe(struct platform_device *pdev) 262 { 263 struct device_node *node = pdev->dev.of_node; 264 struct tad_region *regions; 265 struct tad_pmu *tad_pmu; 266 struct resource *res; 267 u32 tad_pmu_page_size; 268 u32 tad_page_size; 269 u32 tad_cnt; 270 int i, ret; 271 char *name; 272 273 tad_pmu = devm_kzalloc(&pdev->dev, sizeof(*tad_pmu), GFP_KERNEL); 274 if (!tad_pmu) 275 return -ENOMEM; 276 277 platform_set_drvdata(pdev, tad_pmu); 278 279 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 280 if (!res) { 281 dev_err(&pdev->dev, "Mem resource not found\n"); 282 return -ENODEV; 283 } 284 285 ret = of_property_read_u32(node, "marvell,tad-page-size", 286 &tad_page_size); 287 if (ret) { 288 dev_err(&pdev->dev, "Can't find tad-page-size property\n"); 289 return ret; 290 } 291 292 ret = of_property_read_u32(node, "marvell,tad-pmu-page-size", 293 &tad_pmu_page_size); 294 if (ret) { 295 dev_err(&pdev->dev, "Can't find tad-pmu-page-size property\n"); 296 return ret; 297 } 298 299 ret = of_property_read_u32(node, "marvell,tad-cnt", &tad_cnt); 300 if (ret) { 301 dev_err(&pdev->dev, "Can't find tad-cnt property\n"); 302 return ret; 303 } 304 305 regions = devm_kcalloc(&pdev->dev, tad_cnt, 306 sizeof(*regions), GFP_KERNEL); 307 if (!regions) 308 return -ENOMEM; 309 310 /* ioremap the distributed TAD pmu regions */ 311 for (i = 0; i < tad_cnt && res->start < res->end; i++) { 312 regions[i].base = devm_ioremap(&pdev->dev, 313 res->start, 314 tad_pmu_page_size); 315 if (!regions[i].base) { 316 dev_err(&pdev->dev, "TAD%d ioremap fail\n", i); 317 return -ENOMEM; 318 } 319 res->start += tad_page_size; 320 } 321 322 tad_pmu->regions = regions; 323 tad_pmu->region_cnt = tad_cnt; 324 325 tad_pmu->pmu = (struct pmu) { 326 327 .module = THIS_MODULE, 328 .attr_groups = tad_pmu_attr_groups, 329 .capabilities = PERF_PMU_CAP_NO_EXCLUDE | 330 PERF_PMU_CAP_NO_INTERRUPT, 331 .task_ctx_nr = perf_invalid_context, 332 333 .event_init = tad_pmu_event_init, 334 .add = tad_pmu_event_counter_add, 335 .del = tad_pmu_event_counter_del, 336 .start = tad_pmu_event_counter_start, 337 .stop = tad_pmu_event_counter_stop, 338 .read = tad_pmu_event_counter_read, 339 }; 340 341 tad_pmu->cpu = raw_smp_processor_id(); 342 343 /* Register pmu instance for cpu hotplug */ 344 ret = cpuhp_state_add_instance_nocalls(tad_pmu_cpuhp_state, 345 &tad_pmu->node); 346 if (ret) { 347 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret); 348 return ret; 349 } 350 351 name = "tad"; 352 ret = perf_pmu_register(&tad_pmu->pmu, name, -1); 353 if (ret) 354 cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state, 355 &tad_pmu->node); 356 357 return ret; 358 } 359 360 static int tad_pmu_remove(struct platform_device *pdev) 361 { 362 struct tad_pmu *pmu = platform_get_drvdata(pdev); 363 364 cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state, 365 &pmu->node); 366 perf_pmu_unregister(&pmu->pmu); 367 368 return 0; 369 } 370 371 #ifdef CONFIG_OF 372 static const struct of_device_id tad_pmu_of_match[] = { 373 { .compatible = "marvell,cn10k-tad-pmu", }, 374 {}, 375 }; 376 #endif 377 378 static struct platform_driver tad_pmu_driver = { 379 .driver = { 380 .name = "cn10k_tad_pmu", 381 .of_match_table = of_match_ptr(tad_pmu_of_match), 382 .suppress_bind_attrs = true, 383 }, 384 .probe = tad_pmu_probe, 385 .remove = tad_pmu_remove, 386 }; 387 388 static int tad_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) 389 { 390 struct tad_pmu *pmu = hlist_entry_safe(node, struct tad_pmu, node); 391 unsigned int target; 392 393 if (cpu != pmu->cpu) 394 return 0; 395 396 target = cpumask_any_but(cpu_online_mask, cpu); 397 if (target >= nr_cpu_ids) 398 return 0; 399 400 perf_pmu_migrate_context(&pmu->pmu, cpu, target); 401 pmu->cpu = target; 402 403 return 0; 404 } 405 406 static int __init tad_pmu_init(void) 407 { 408 int ret; 409 410 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, 411 "perf/cn10k/tadpmu:online", 412 NULL, 413 tad_pmu_offline_cpu); 414 if (ret < 0) 415 return ret; 416 tad_pmu_cpuhp_state = ret; 417 return platform_driver_register(&tad_pmu_driver); 418 } 419 420 static void __exit tad_pmu_exit(void) 421 { 422 platform_driver_unregister(&tad_pmu_driver); 423 cpuhp_remove_multi_state(tad_pmu_cpuhp_state); 424 } 425 426 module_init(tad_pmu_init); 427 module_exit(tad_pmu_exit); 428 429 MODULE_DESCRIPTION("Marvell CN10K LLC-TAD Perf driver"); 430 MODULE_AUTHOR("Bhaskara Budiredla <bbudiredla@marvell.com>"); 431 MODULE_LICENSE("GPL v2"); 432