1 /* 2 * Copyright (C) 2013 Advanced Micro Devices, Inc. 3 * 4 * Author: Steven Kinney <Steven.Kinney@amd.com> 5 * Author: Suravee Suthikulpanit <Suraveee.Suthikulpanit@amd.com> 6 * 7 * Perf: amd_iommu - AMD IOMMU Performance Counter PMU implementation 8 * 9 * This program is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License version 2 as 11 * published by the Free Software Foundation. 12 */ 13 14 #define pr_fmt(fmt) "perf/amd_iommu: " fmt 15 16 #include <linux/perf_event.h> 17 #include <linux/init.h> 18 #include <linux/cpumask.h> 19 #include <linux/slab.h> 20 21 #include "../perf_event.h" 22 #include "iommu.h" 23 24 #define COUNTER_SHIFT 16 25 26 /* iommu pmu conf masks */ 27 #define GET_CSOURCE(x) ((x)->conf & 0xFFULL) 28 #define GET_DEVID(x) (((x)->conf >> 8) & 0xFFFFULL) 29 #define GET_DOMID(x) (((x)->conf >> 24) & 0xFFFFULL) 30 #define GET_PASID(x) (((x)->conf >> 40) & 0xFFFFFULL) 31 32 /* iommu pmu conf1 masks */ 33 #define GET_DEVID_MASK(x) ((x)->conf1 & 0xFFFFULL) 34 #define GET_DOMID_MASK(x) (((x)->conf1 >> 16) & 0xFFFFULL) 35 #define GET_PASID_MASK(x) (((x)->conf1 >> 32) & 0xFFFFFULL) 36 37 #define IOMMU_NAME_SIZE 16 38 39 struct perf_amd_iommu { 40 struct list_head list; 41 struct pmu pmu; 42 struct amd_iommu *iommu; 43 char name[IOMMU_NAME_SIZE]; 44 u8 max_banks; 45 u8 max_counters; 46 u64 cntr_assign_mask; 47 raw_spinlock_t lock; 48 }; 49 50 static LIST_HEAD(perf_amd_iommu_list); 51 52 /*--------------------------------------------- 53 * sysfs format attributes 54 *---------------------------------------------*/ 55 PMU_FORMAT_ATTR(csource, "config:0-7"); 56 PMU_FORMAT_ATTR(devid, "config:8-23"); 57 PMU_FORMAT_ATTR(domid, "config:24-39"); 58 PMU_FORMAT_ATTR(pasid, "config:40-59"); 59 PMU_FORMAT_ATTR(devid_mask, "config1:0-15"); 60 PMU_FORMAT_ATTR(domid_mask, "config1:16-31"); 61 PMU_FORMAT_ATTR(pasid_mask, "config1:32-51"); 62 63 static struct attribute *iommu_format_attrs[] = { 64 &format_attr_csource.attr, 65 &format_attr_devid.attr, 66 &format_attr_pasid.attr, 67 &format_attr_domid.attr, 68 &format_attr_devid_mask.attr, 69 &format_attr_pasid_mask.attr, 70 &format_attr_domid_mask.attr, 71 NULL, 72 }; 73 74 static struct attribute_group amd_iommu_format_group = { 75 .name = "format", 76 .attrs = iommu_format_attrs, 77 }; 78 79 /*--------------------------------------------- 80 * sysfs events attributes 81 *---------------------------------------------*/ 82 static struct attribute_group amd_iommu_events_group = { 83 .name = "events", 84 }; 85 86 struct amd_iommu_event_desc { 87 struct kobj_attribute attr; 88 const char *event; 89 }; 90 91 static ssize_t _iommu_event_show(struct kobject *kobj, 92 struct kobj_attribute *attr, char *buf) 93 { 94 struct amd_iommu_event_desc *event = 95 container_of(attr, struct amd_iommu_event_desc, attr); 96 return sprintf(buf, "%s\n", event->event); 97 } 98 99 #define AMD_IOMMU_EVENT_DESC(_name, _event) \ 100 { \ 101 .attr = __ATTR(_name, 0444, _iommu_event_show, NULL), \ 102 .event = _event, \ 103 } 104 105 static struct amd_iommu_event_desc amd_iommu_v2_event_descs[] = { 106 AMD_IOMMU_EVENT_DESC(mem_pass_untrans, "csource=0x01"), 107 AMD_IOMMU_EVENT_DESC(mem_pass_pretrans, "csource=0x02"), 108 AMD_IOMMU_EVENT_DESC(mem_pass_excl, "csource=0x03"), 109 AMD_IOMMU_EVENT_DESC(mem_target_abort, "csource=0x04"), 110 AMD_IOMMU_EVENT_DESC(mem_trans_total, "csource=0x05"), 111 AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pte_hit, "csource=0x06"), 112 AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pte_mis, "csource=0x07"), 113 AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pde_hit, "csource=0x08"), 114 AMD_IOMMU_EVENT_DESC(mem_iommu_tlb_pde_mis, "csource=0x09"), 115 AMD_IOMMU_EVENT_DESC(mem_dte_hit, "csource=0x0a"), 116 AMD_IOMMU_EVENT_DESC(mem_dte_mis, "csource=0x0b"), 117 AMD_IOMMU_EVENT_DESC(page_tbl_read_tot, "csource=0x0c"), 118 AMD_IOMMU_EVENT_DESC(page_tbl_read_nst, "csource=0x0d"), 119 AMD_IOMMU_EVENT_DESC(page_tbl_read_gst, "csource=0x0e"), 120 AMD_IOMMU_EVENT_DESC(int_dte_hit, "csource=0x0f"), 121 AMD_IOMMU_EVENT_DESC(int_dte_mis, "csource=0x10"), 122 AMD_IOMMU_EVENT_DESC(cmd_processed, "csource=0x11"), 123 AMD_IOMMU_EVENT_DESC(cmd_processed_inv, "csource=0x12"), 124 AMD_IOMMU_EVENT_DESC(tlb_inv, "csource=0x13"), 125 AMD_IOMMU_EVENT_DESC(ign_rd_wr_mmio_1ff8h, "csource=0x14"), 126 AMD_IOMMU_EVENT_DESC(vapic_int_non_guest, "csource=0x15"), 127 AMD_IOMMU_EVENT_DESC(vapic_int_guest, "csource=0x16"), 128 AMD_IOMMU_EVENT_DESC(smi_recv, "csource=0x17"), 129 AMD_IOMMU_EVENT_DESC(smi_blk, "csource=0x18"), 130 { /* end: all zeroes */ }, 131 }; 132 133 /*--------------------------------------------- 134 * sysfs cpumask attributes 135 *---------------------------------------------*/ 136 static cpumask_t iommu_cpumask; 137 138 static ssize_t _iommu_cpumask_show(struct device *dev, 139 struct device_attribute *attr, 140 char *buf) 141 { 142 return cpumap_print_to_pagebuf(true, buf, &iommu_cpumask); 143 } 144 static DEVICE_ATTR(cpumask, S_IRUGO, _iommu_cpumask_show, NULL); 145 146 static struct attribute *iommu_cpumask_attrs[] = { 147 &dev_attr_cpumask.attr, 148 NULL, 149 }; 150 151 static struct attribute_group amd_iommu_cpumask_group = { 152 .attrs = iommu_cpumask_attrs, 153 }; 154 155 /*---------------------------------------------*/ 156 157 static int get_next_avail_iommu_bnk_cntr(struct perf_event *event) 158 { 159 struct perf_amd_iommu *piommu = container_of(event->pmu, struct perf_amd_iommu, pmu); 160 int max_cntrs = piommu->max_counters; 161 int max_banks = piommu->max_banks; 162 u32 shift, bank, cntr; 163 unsigned long flags; 164 int retval; 165 166 raw_spin_lock_irqsave(&piommu->lock, flags); 167 168 for (bank = 0, shift = 0; bank < max_banks; bank++) { 169 for (cntr = 0; cntr < max_cntrs; cntr++) { 170 shift = bank + (bank*3) + cntr; 171 if (piommu->cntr_assign_mask & BIT_ULL(shift)) { 172 continue; 173 } else { 174 piommu->cntr_assign_mask |= BIT_ULL(shift); 175 event->hw.iommu_bank = bank; 176 event->hw.iommu_cntr = cntr; 177 retval = 0; 178 goto out; 179 } 180 } 181 } 182 retval = -ENOSPC; 183 out: 184 raw_spin_unlock_irqrestore(&piommu->lock, flags); 185 return retval; 186 } 187 188 static int clear_avail_iommu_bnk_cntr(struct perf_amd_iommu *perf_iommu, 189 u8 bank, u8 cntr) 190 { 191 unsigned long flags; 192 int max_banks, max_cntrs; 193 int shift = 0; 194 195 max_banks = perf_iommu->max_banks; 196 max_cntrs = perf_iommu->max_counters; 197 198 if ((bank > max_banks) || (cntr > max_cntrs)) 199 return -EINVAL; 200 201 shift = bank + cntr + (bank*3); 202 203 raw_spin_lock_irqsave(&perf_iommu->lock, flags); 204 perf_iommu->cntr_assign_mask &= ~(1ULL<<shift); 205 raw_spin_unlock_irqrestore(&perf_iommu->lock, flags); 206 207 return 0; 208 } 209 210 static int perf_iommu_event_init(struct perf_event *event) 211 { 212 struct hw_perf_event *hwc = &event->hw; 213 214 /* test the event attr type check for PMU enumeration */ 215 if (event->attr.type != event->pmu->type) 216 return -ENOENT; 217 218 /* 219 * IOMMU counters are shared across all cores. 220 * Therefore, it does not support per-process mode. 221 * Also, it does not support event sampling mode. 222 */ 223 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK) 224 return -EINVAL; 225 226 if (event->cpu < 0) 227 return -EINVAL; 228 229 /* update the hw_perf_event struct with the iommu config data */ 230 hwc->conf = event->attr.config; 231 hwc->conf1 = event->attr.config1; 232 233 return 0; 234 } 235 236 static inline struct amd_iommu *perf_event_2_iommu(struct perf_event *ev) 237 { 238 return (container_of(ev->pmu, struct perf_amd_iommu, pmu))->iommu; 239 } 240 241 static void perf_iommu_enable_event(struct perf_event *ev) 242 { 243 struct amd_iommu *iommu = perf_event_2_iommu(ev); 244 struct hw_perf_event *hwc = &ev->hw; 245 u8 bank = hwc->iommu_bank; 246 u8 cntr = hwc->iommu_cntr; 247 u64 reg = 0ULL; 248 249 reg = GET_CSOURCE(hwc); 250 amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_COUNTER_SRC_REG, ®); 251 252 reg = GET_DEVID_MASK(hwc); 253 reg = GET_DEVID(hwc) | (reg << 32); 254 if (reg) 255 reg |= BIT(31); 256 amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DEVID_MATCH_REG, ®); 257 258 reg = GET_PASID_MASK(hwc); 259 reg = GET_PASID(hwc) | (reg << 32); 260 if (reg) 261 reg |= BIT(31); 262 amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_PASID_MATCH_REG, ®); 263 264 reg = GET_DOMID_MASK(hwc); 265 reg = GET_DOMID(hwc) | (reg << 32); 266 if (reg) 267 reg |= BIT(31); 268 amd_iommu_pc_set_reg(iommu, bank, cntr, IOMMU_PC_DOMID_MATCH_REG, ®); 269 } 270 271 static void perf_iommu_disable_event(struct perf_event *event) 272 { 273 struct amd_iommu *iommu = perf_event_2_iommu(event); 274 struct hw_perf_event *hwc = &event->hw; 275 u64 reg = 0ULL; 276 277 amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr, 278 IOMMU_PC_COUNTER_SRC_REG, ®); 279 } 280 281 static void perf_iommu_start(struct perf_event *event, int flags) 282 { 283 struct hw_perf_event *hwc = &event->hw; 284 285 if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED))) 286 return; 287 288 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 289 hwc->state = 0; 290 291 if (flags & PERF_EF_RELOAD) { 292 u64 prev_raw_count = local64_read(&hwc->prev_count); 293 struct amd_iommu *iommu = perf_event_2_iommu(event); 294 295 amd_iommu_pc_set_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr, 296 IOMMU_PC_COUNTER_REG, &prev_raw_count); 297 } 298 299 perf_iommu_enable_event(event); 300 perf_event_update_userpage(event); 301 302 } 303 304 static void perf_iommu_read(struct perf_event *event) 305 { 306 u64 count, prev, delta; 307 struct hw_perf_event *hwc = &event->hw; 308 struct amd_iommu *iommu = perf_event_2_iommu(event); 309 310 if (amd_iommu_pc_get_reg(iommu, hwc->iommu_bank, hwc->iommu_cntr, 311 IOMMU_PC_COUNTER_REG, &count)) 312 return; 313 314 /* IOMMU pc counter register is only 48 bits */ 315 count &= GENMASK_ULL(47, 0); 316 317 prev = local64_read(&hwc->prev_count); 318 if (local64_cmpxchg(&hwc->prev_count, prev, count) != prev) 319 return; 320 321 /* Handle 48-bit counter overflow */ 322 delta = (count << COUNTER_SHIFT) - (prev << COUNTER_SHIFT); 323 delta >>= COUNTER_SHIFT; 324 local64_add(delta, &event->count); 325 } 326 327 static void perf_iommu_stop(struct perf_event *event, int flags) 328 { 329 struct hw_perf_event *hwc = &event->hw; 330 331 if (hwc->state & PERF_HES_UPTODATE) 332 return; 333 334 perf_iommu_disable_event(event); 335 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); 336 hwc->state |= PERF_HES_STOPPED; 337 338 if (hwc->state & PERF_HES_UPTODATE) 339 return; 340 341 perf_iommu_read(event); 342 hwc->state |= PERF_HES_UPTODATE; 343 } 344 345 static int perf_iommu_add(struct perf_event *event, int flags) 346 { 347 int retval; 348 349 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 350 351 /* request an iommu bank/counter */ 352 retval = get_next_avail_iommu_bnk_cntr(event); 353 if (retval) 354 return retval; 355 356 if (flags & PERF_EF_START) 357 perf_iommu_start(event, PERF_EF_RELOAD); 358 359 return 0; 360 } 361 362 static void perf_iommu_del(struct perf_event *event, int flags) 363 { 364 struct hw_perf_event *hwc = &event->hw; 365 struct perf_amd_iommu *perf_iommu = 366 container_of(event->pmu, struct perf_amd_iommu, pmu); 367 368 perf_iommu_stop(event, PERF_EF_UPDATE); 369 370 /* clear the assigned iommu bank/counter */ 371 clear_avail_iommu_bnk_cntr(perf_iommu, 372 hwc->iommu_bank, hwc->iommu_cntr); 373 374 perf_event_update_userpage(event); 375 } 376 377 static __init int _init_events_attrs(void) 378 { 379 int i = 0, j; 380 struct attribute **attrs; 381 382 while (amd_iommu_v2_event_descs[i].attr.attr.name) 383 i++; 384 385 attrs = kcalloc(i + 1, sizeof(struct attribute **), GFP_KERNEL); 386 if (!attrs) 387 return -ENOMEM; 388 389 for (j = 0; j < i; j++) 390 attrs[j] = &amd_iommu_v2_event_descs[j].attr.attr; 391 392 amd_iommu_events_group.attrs = attrs; 393 return 0; 394 } 395 396 static const struct attribute_group *amd_iommu_attr_groups[] = { 397 &amd_iommu_format_group, 398 &amd_iommu_cpumask_group, 399 &amd_iommu_events_group, 400 NULL, 401 }; 402 403 static const struct pmu iommu_pmu __initconst = { 404 .event_init = perf_iommu_event_init, 405 .add = perf_iommu_add, 406 .del = perf_iommu_del, 407 .start = perf_iommu_start, 408 .stop = perf_iommu_stop, 409 .read = perf_iommu_read, 410 .task_ctx_nr = perf_invalid_context, 411 .attr_groups = amd_iommu_attr_groups, 412 .capabilities = PERF_PMU_CAP_NO_EXCLUDE, 413 }; 414 415 static __init int init_one_iommu(unsigned int idx) 416 { 417 struct perf_amd_iommu *perf_iommu; 418 int ret; 419 420 perf_iommu = kzalloc(sizeof(struct perf_amd_iommu), GFP_KERNEL); 421 if (!perf_iommu) 422 return -ENOMEM; 423 424 raw_spin_lock_init(&perf_iommu->lock); 425 426 perf_iommu->pmu = iommu_pmu; 427 perf_iommu->iommu = get_amd_iommu(idx); 428 perf_iommu->max_banks = amd_iommu_pc_get_max_banks(idx); 429 perf_iommu->max_counters = amd_iommu_pc_get_max_counters(idx); 430 431 if (!perf_iommu->iommu || 432 !perf_iommu->max_banks || 433 !perf_iommu->max_counters) { 434 kfree(perf_iommu); 435 return -EINVAL; 436 } 437 438 snprintf(perf_iommu->name, IOMMU_NAME_SIZE, "amd_iommu_%u", idx); 439 440 ret = perf_pmu_register(&perf_iommu->pmu, perf_iommu->name, -1); 441 if (!ret) { 442 pr_info("Detected AMD IOMMU #%d (%d banks, %d counters/bank).\n", 443 idx, perf_iommu->max_banks, perf_iommu->max_counters); 444 list_add_tail(&perf_iommu->list, &perf_amd_iommu_list); 445 } else { 446 pr_warn("Error initializing IOMMU %d.\n", idx); 447 kfree(perf_iommu); 448 } 449 return ret; 450 } 451 452 static __init int amd_iommu_pc_init(void) 453 { 454 unsigned int i, cnt = 0; 455 int ret; 456 457 /* Make sure the IOMMU PC resource is available */ 458 if (!amd_iommu_pc_supported()) 459 return -ENODEV; 460 461 ret = _init_events_attrs(); 462 if (ret) 463 return ret; 464 465 /* 466 * An IOMMU PMU is specific to an IOMMU, and can function independently. 467 * So we go through all IOMMUs and ignore the one that fails init 468 * unless all IOMMU are failing. 469 */ 470 for (i = 0; i < amd_iommu_get_num_iommus(); i++) { 471 ret = init_one_iommu(i); 472 if (!ret) 473 cnt++; 474 } 475 476 if (!cnt) { 477 kfree(amd_iommu_events_group.attrs); 478 return -ENODEV; 479 } 480 481 /* Init cpumask attributes to only core 0 */ 482 cpumask_set_cpu(0, &iommu_cpumask); 483 return 0; 484 } 485 486 device_initcall(amd_iommu_pc_init); 487