1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Linux performance counter support for LoongArch. 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 * 7 * Derived from MIPS: 8 * Copyright (C) 2010 MIPS Technologies, Inc. 9 * Copyright (C) 2011 Cavium Networks, Inc. 10 * Author: Deng-Cheng Zhu 11 */ 12 13 #include <linux/cpumask.h> 14 #include <linux/interrupt.h> 15 #include <linux/smp.h> 16 #include <linux/kernel.h> 17 #include <linux/perf_event.h> 18 #include <linux/uaccess.h> 19 #include <linux/sched/task_stack.h> 20 21 #include <asm/irq.h> 22 #include <asm/irq_regs.h> 23 #include <asm/stacktrace.h> 24 #include <asm/unwind.h> 25 26 /* 27 * Get the return address for a single stackframe and return a pointer to the 28 * next frame tail. 29 */ 30 static unsigned long 31 user_backtrace(struct perf_callchain_entry_ctx *entry, unsigned long fp) 32 { 33 unsigned long err; 34 unsigned long __user *user_frame_tail; 35 struct stack_frame buftail; 36 37 user_frame_tail = (unsigned long __user *)(fp - sizeof(struct stack_frame)); 38 39 /* Also check accessibility of one struct frame_tail beyond */ 40 if (!access_ok(user_frame_tail, sizeof(buftail))) 41 return 0; 42 43 pagefault_disable(); 44 err = __copy_from_user_inatomic(&buftail, user_frame_tail, sizeof(buftail)); 45 pagefault_enable(); 46 47 if (err || (unsigned long)user_frame_tail >= buftail.fp) 48 return 0; 49 50 perf_callchain_store(entry, buftail.ra); 51 52 return buftail.fp; 53 } 54 55 void perf_callchain_user(struct perf_callchain_entry_ctx *entry, 56 struct pt_regs *regs) 57 { 58 unsigned long fp; 59 60 if (perf_guest_state()) { 61 /* We don't support guest os callchain now */ 62 return; 63 } 64 65 perf_callchain_store(entry, regs->csr_era); 66 67 fp = regs->regs[22]; 68 69 while (entry->nr < entry->max_stack && fp && !((unsigned long)fp & 0xf)) 70 fp = user_backtrace(entry, fp); 71 } 72 73 void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, 74 struct pt_regs *regs) 75 { 76 struct unwind_state state; 77 unsigned long addr; 78 79 for (unwind_start(&state, current, regs); 80 !unwind_done(&state); unwind_next_frame(&state)) { 81 addr = unwind_get_return_address(&state); 82 if (!addr || perf_callchain_store(entry, addr)) 83 return; 84 } 85 } 86 87 #define LOONGARCH_MAX_HWEVENTS 32 88 89 struct cpu_hw_events { 90 /* Array of events on this cpu. */ 91 struct perf_event *events[LOONGARCH_MAX_HWEVENTS]; 92 93 /* 94 * Set the bit (indexed by the counter number) when the counter 95 * is used for an event. 96 */ 97 unsigned long used_mask[BITS_TO_LONGS(LOONGARCH_MAX_HWEVENTS)]; 98 99 /* 100 * Software copy of the control register for each performance counter. 101 */ 102 unsigned int saved_ctrl[LOONGARCH_MAX_HWEVENTS]; 103 }; 104 static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { 105 .saved_ctrl = {0}, 106 }; 107 108 /* The description of LoongArch performance events. */ 109 struct loongarch_perf_event { 110 unsigned int event_id; 111 }; 112 113 static struct loongarch_perf_event raw_event; 114 static DEFINE_MUTEX(raw_event_mutex); 115 116 #define C(x) PERF_COUNT_HW_CACHE_##x 117 #define HW_OP_UNSUPPORTED 0xffffffff 118 #define CACHE_OP_UNSUPPORTED 0xffffffff 119 120 #define PERF_MAP_ALL_UNSUPPORTED \ 121 [0 ... PERF_COUNT_HW_MAX - 1] = {HW_OP_UNSUPPORTED} 122 123 #define PERF_CACHE_MAP_ALL_UNSUPPORTED \ 124 [0 ... C(MAX) - 1] = { \ 125 [0 ... C(OP_MAX) - 1] = { \ 126 [0 ... C(RESULT_MAX) - 1] = {CACHE_OP_UNSUPPORTED}, \ 127 }, \ 128 } 129 130 struct loongarch_pmu { 131 u64 max_period; 132 u64 valid_count; 133 u64 overflow; 134 const char *name; 135 unsigned int num_counters; 136 u64 (*read_counter)(unsigned int idx); 137 void (*write_counter)(unsigned int idx, u64 val); 138 const struct loongarch_perf_event *(*map_raw_event)(u64 config); 139 const struct loongarch_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; 140 const struct loongarch_perf_event (*cache_event_map) 141 [PERF_COUNT_HW_CACHE_MAX] 142 [PERF_COUNT_HW_CACHE_OP_MAX] 143 [PERF_COUNT_HW_CACHE_RESULT_MAX]; 144 }; 145 146 static struct loongarch_pmu loongarch_pmu; 147 148 #define M_PERFCTL_EVENT(event) (event & CSR_PERFCTRL_EVENT) 149 150 #define M_PERFCTL_COUNT_EVENT_WHENEVER (CSR_PERFCTRL_PLV0 | \ 151 CSR_PERFCTRL_PLV1 | \ 152 CSR_PERFCTRL_PLV2 | \ 153 CSR_PERFCTRL_PLV3 | \ 154 CSR_PERFCTRL_IE) 155 156 #define M_PERFCTL_CONFIG_MASK 0x1f0000 157 158 static void pause_local_counters(void); 159 static void resume_local_counters(void); 160 161 static u64 loongarch_pmu_read_counter(unsigned int idx) 162 { 163 u64 val = -1; 164 165 switch (idx) { 166 case 0: 167 val = read_csr_perfcntr0(); 168 break; 169 case 1: 170 val = read_csr_perfcntr1(); 171 break; 172 case 2: 173 val = read_csr_perfcntr2(); 174 break; 175 case 3: 176 val = read_csr_perfcntr3(); 177 break; 178 default: 179 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); 180 return 0; 181 } 182 183 return val; 184 } 185 186 static void loongarch_pmu_write_counter(unsigned int idx, u64 val) 187 { 188 switch (idx) { 189 case 0: 190 write_csr_perfcntr0(val); 191 return; 192 case 1: 193 write_csr_perfcntr1(val); 194 return; 195 case 2: 196 write_csr_perfcntr2(val); 197 return; 198 case 3: 199 write_csr_perfcntr3(val); 200 return; 201 default: 202 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); 203 return; 204 } 205 } 206 207 static unsigned int loongarch_pmu_read_control(unsigned int idx) 208 { 209 unsigned int val = -1; 210 211 switch (idx) { 212 case 0: 213 val = read_csr_perfctrl0(); 214 break; 215 case 1: 216 val = read_csr_perfctrl1(); 217 break; 218 case 2: 219 val = read_csr_perfctrl2(); 220 break; 221 case 3: 222 val = read_csr_perfctrl3(); 223 break; 224 default: 225 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); 226 return 0; 227 } 228 229 return val; 230 } 231 232 static void loongarch_pmu_write_control(unsigned int idx, unsigned int val) 233 { 234 switch (idx) { 235 case 0: 236 write_csr_perfctrl0(val); 237 return; 238 case 1: 239 write_csr_perfctrl1(val); 240 return; 241 case 2: 242 write_csr_perfctrl2(val); 243 return; 244 case 3: 245 write_csr_perfctrl3(val); 246 return; 247 default: 248 WARN_ONCE(1, "Invalid performance counter number (%d)\n", idx); 249 return; 250 } 251 } 252 253 static int loongarch_pmu_alloc_counter(struct cpu_hw_events *cpuc, struct hw_perf_event *hwc) 254 { 255 int i; 256 257 for (i = 0; i < loongarch_pmu.num_counters; i++) { 258 if (!test_and_set_bit(i, cpuc->used_mask)) 259 return i; 260 } 261 262 return -EAGAIN; 263 } 264 265 static void loongarch_pmu_enable_event(struct hw_perf_event *evt, int idx) 266 { 267 unsigned int cpu; 268 struct perf_event *event = container_of(evt, struct perf_event, hw); 269 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 270 271 WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters); 272 273 /* Make sure interrupt enabled. */ 274 cpuc->saved_ctrl[idx] = M_PERFCTL_EVENT(evt->event_base) | 275 (evt->config_base & M_PERFCTL_CONFIG_MASK) | CSR_PERFCTRL_IE; 276 277 cpu = (event->cpu >= 0) ? event->cpu : smp_processor_id(); 278 279 /* 280 * We do not actually let the counter run. Leave it until start(). 281 */ 282 pr_debug("Enabling perf counter for CPU%d\n", cpu); 283 } 284 285 static void loongarch_pmu_disable_event(int idx) 286 { 287 unsigned long flags; 288 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 289 290 WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters); 291 292 local_irq_save(flags); 293 cpuc->saved_ctrl[idx] = loongarch_pmu_read_control(idx) & 294 ~M_PERFCTL_COUNT_EVENT_WHENEVER; 295 loongarch_pmu_write_control(idx, cpuc->saved_ctrl[idx]); 296 local_irq_restore(flags); 297 } 298 299 static int loongarch_pmu_event_set_period(struct perf_event *event, 300 struct hw_perf_event *hwc, 301 int idx) 302 { 303 int ret = 0; 304 u64 left = local64_read(&hwc->period_left); 305 u64 period = hwc->sample_period; 306 307 if (unlikely((left + period) & (1ULL << 63))) { 308 /* left underflowed by more than period. */ 309 left = period; 310 local64_set(&hwc->period_left, left); 311 hwc->last_period = period; 312 ret = 1; 313 } else if (unlikely((left + period) <= period)) { 314 /* left underflowed by less than period. */ 315 left += period; 316 local64_set(&hwc->period_left, left); 317 hwc->last_period = period; 318 ret = 1; 319 } 320 321 if (left > loongarch_pmu.max_period) { 322 left = loongarch_pmu.max_period; 323 local64_set(&hwc->period_left, left); 324 } 325 326 local64_set(&hwc->prev_count, loongarch_pmu.overflow - left); 327 328 loongarch_pmu.write_counter(idx, loongarch_pmu.overflow - left); 329 330 perf_event_update_userpage(event); 331 332 return ret; 333 } 334 335 static void loongarch_pmu_event_update(struct perf_event *event, 336 struct hw_perf_event *hwc, 337 int idx) 338 { 339 u64 delta; 340 u64 prev_raw_count, new_raw_count; 341 342 again: 343 prev_raw_count = local64_read(&hwc->prev_count); 344 new_raw_count = loongarch_pmu.read_counter(idx); 345 346 if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, 347 new_raw_count) != prev_raw_count) 348 goto again; 349 350 delta = new_raw_count - prev_raw_count; 351 352 local64_add(delta, &event->count); 353 local64_sub(delta, &hwc->period_left); 354 } 355 356 static void loongarch_pmu_start(struct perf_event *event, int flags) 357 { 358 struct hw_perf_event *hwc = &event->hw; 359 360 if (flags & PERF_EF_RELOAD) 361 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE)); 362 363 hwc->state = 0; 364 365 /* Set the period for the event. */ 366 loongarch_pmu_event_set_period(event, hwc, hwc->idx); 367 368 /* Enable the event. */ 369 loongarch_pmu_enable_event(hwc, hwc->idx); 370 } 371 372 static void loongarch_pmu_stop(struct perf_event *event, int flags) 373 { 374 struct hw_perf_event *hwc = &event->hw; 375 376 if (!(hwc->state & PERF_HES_STOPPED)) { 377 /* We are working on a local event. */ 378 loongarch_pmu_disable_event(hwc->idx); 379 barrier(); 380 loongarch_pmu_event_update(event, hwc, hwc->idx); 381 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; 382 } 383 } 384 385 static int loongarch_pmu_add(struct perf_event *event, int flags) 386 { 387 int idx, err = 0; 388 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 389 struct hw_perf_event *hwc = &event->hw; 390 391 perf_pmu_disable(event->pmu); 392 393 /* To look for a free counter for this event. */ 394 idx = loongarch_pmu_alloc_counter(cpuc, hwc); 395 if (idx < 0) { 396 err = idx; 397 goto out; 398 } 399 400 /* 401 * If there is an event in the counter we are going to use then 402 * make sure it is disabled. 403 */ 404 event->hw.idx = idx; 405 loongarch_pmu_disable_event(idx); 406 cpuc->events[idx] = event; 407 408 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; 409 if (flags & PERF_EF_START) 410 loongarch_pmu_start(event, PERF_EF_RELOAD); 411 412 /* Propagate our changes to the userspace mapping. */ 413 perf_event_update_userpage(event); 414 415 out: 416 perf_pmu_enable(event->pmu); 417 return err; 418 } 419 420 static void loongarch_pmu_del(struct perf_event *event, int flags) 421 { 422 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 423 struct hw_perf_event *hwc = &event->hw; 424 int idx = hwc->idx; 425 426 WARN_ON(idx < 0 || idx >= loongarch_pmu.num_counters); 427 428 loongarch_pmu_stop(event, PERF_EF_UPDATE); 429 cpuc->events[idx] = NULL; 430 clear_bit(idx, cpuc->used_mask); 431 432 perf_event_update_userpage(event); 433 } 434 435 static void loongarch_pmu_read(struct perf_event *event) 436 { 437 struct hw_perf_event *hwc = &event->hw; 438 439 /* Don't read disabled counters! */ 440 if (hwc->idx < 0) 441 return; 442 443 loongarch_pmu_event_update(event, hwc, hwc->idx); 444 } 445 446 static void loongarch_pmu_enable(struct pmu *pmu) 447 { 448 resume_local_counters(); 449 } 450 451 static void loongarch_pmu_disable(struct pmu *pmu) 452 { 453 pause_local_counters(); 454 } 455 456 static DEFINE_MUTEX(pmu_reserve_mutex); 457 static atomic_t active_events = ATOMIC_INIT(0); 458 459 static int get_pmc_irq(void) 460 { 461 struct irq_domain *d = irq_find_matching_fwnode(cpuintc_handle, DOMAIN_BUS_ANY); 462 463 if (d) 464 return irq_create_mapping(d, INT_PCOV); 465 466 return -EINVAL; 467 } 468 469 static void reset_counters(void *arg); 470 static int __hw_perf_event_init(struct perf_event *event); 471 472 static void hw_perf_event_destroy(struct perf_event *event) 473 { 474 if (atomic_dec_and_mutex_lock(&active_events, &pmu_reserve_mutex)) { 475 on_each_cpu(reset_counters, NULL, 1); 476 free_irq(get_pmc_irq(), &loongarch_pmu); 477 mutex_unlock(&pmu_reserve_mutex); 478 } 479 } 480 481 static void handle_associated_event(struct cpu_hw_events *cpuc, int idx, 482 struct perf_sample_data *data, struct pt_regs *regs) 483 { 484 struct perf_event *event = cpuc->events[idx]; 485 struct hw_perf_event *hwc = &event->hw; 486 487 loongarch_pmu_event_update(event, hwc, idx); 488 data->period = event->hw.last_period; 489 if (!loongarch_pmu_event_set_period(event, hwc, idx)) 490 return; 491 492 if (perf_event_overflow(event, data, regs)) 493 loongarch_pmu_disable_event(idx); 494 } 495 496 static irqreturn_t pmu_handle_irq(int irq, void *dev) 497 { 498 int n; 499 int handled = IRQ_NONE; 500 uint64_t counter; 501 struct pt_regs *regs; 502 struct perf_sample_data data; 503 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 504 505 /* 506 * First we pause the local counters, so that when we are locked 507 * here, the counters are all paused. When it gets locked due to 508 * perf_disable(), the timer interrupt handler will be delayed. 509 * 510 * See also loongarch_pmu_start(). 511 */ 512 pause_local_counters(); 513 514 regs = get_irq_regs(); 515 516 perf_sample_data_init(&data, 0, 0); 517 518 for (n = 0; n < loongarch_pmu.num_counters; n++) { 519 if (test_bit(n, cpuc->used_mask)) { 520 counter = loongarch_pmu.read_counter(n); 521 if (counter & loongarch_pmu.overflow) { 522 handle_associated_event(cpuc, n, &data, regs); 523 handled = IRQ_HANDLED; 524 } 525 } 526 } 527 528 resume_local_counters(); 529 530 /* 531 * Do all the work for the pending perf events. We can do this 532 * in here because the performance counter interrupt is a regular 533 * interrupt, not NMI. 534 */ 535 if (handled == IRQ_HANDLED) 536 irq_work_run(); 537 538 return handled; 539 } 540 541 static int loongarch_pmu_event_init(struct perf_event *event) 542 { 543 int r, irq; 544 unsigned long flags; 545 546 /* does not support taken branch sampling */ 547 if (has_branch_stack(event)) 548 return -EOPNOTSUPP; 549 550 switch (event->attr.type) { 551 case PERF_TYPE_RAW: 552 case PERF_TYPE_HARDWARE: 553 case PERF_TYPE_HW_CACHE: 554 break; 555 556 default: 557 /* Init it to avoid false validate_group */ 558 event->hw.event_base = 0xffffffff; 559 return -ENOENT; 560 } 561 562 if (event->cpu >= 0 && !cpu_online(event->cpu)) 563 return -ENODEV; 564 565 irq = get_pmc_irq(); 566 flags = IRQF_PERCPU | IRQF_NOBALANCING | IRQF_NO_THREAD | IRQF_NO_SUSPEND | IRQF_SHARED; 567 if (!atomic_inc_not_zero(&active_events)) { 568 mutex_lock(&pmu_reserve_mutex); 569 if (atomic_read(&active_events) == 0) { 570 r = request_irq(irq, pmu_handle_irq, flags, "Perf_PMU", &loongarch_pmu); 571 if (r < 0) { 572 mutex_unlock(&pmu_reserve_mutex); 573 pr_warn("PMU IRQ request failed\n"); 574 return -ENODEV; 575 } 576 } 577 atomic_inc(&active_events); 578 mutex_unlock(&pmu_reserve_mutex); 579 } 580 581 return __hw_perf_event_init(event); 582 } 583 584 static struct pmu pmu = { 585 .pmu_enable = loongarch_pmu_enable, 586 .pmu_disable = loongarch_pmu_disable, 587 .event_init = loongarch_pmu_event_init, 588 .add = loongarch_pmu_add, 589 .del = loongarch_pmu_del, 590 .start = loongarch_pmu_start, 591 .stop = loongarch_pmu_stop, 592 .read = loongarch_pmu_read, 593 }; 594 595 static unsigned int loongarch_pmu_perf_event_encode(const struct loongarch_perf_event *pev) 596 { 597 return M_PERFCTL_EVENT(pev->event_id); 598 } 599 600 static const struct loongarch_perf_event *loongarch_pmu_map_general_event(int idx) 601 { 602 const struct loongarch_perf_event *pev; 603 604 pev = &(*loongarch_pmu.general_event_map)[idx]; 605 606 if (pev->event_id == HW_OP_UNSUPPORTED) 607 return ERR_PTR(-ENOENT); 608 609 return pev; 610 } 611 612 static const struct loongarch_perf_event *loongarch_pmu_map_cache_event(u64 config) 613 { 614 unsigned int cache_type, cache_op, cache_result; 615 const struct loongarch_perf_event *pev; 616 617 cache_type = (config >> 0) & 0xff; 618 if (cache_type >= PERF_COUNT_HW_CACHE_MAX) 619 return ERR_PTR(-EINVAL); 620 621 cache_op = (config >> 8) & 0xff; 622 if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) 623 return ERR_PTR(-EINVAL); 624 625 cache_result = (config >> 16) & 0xff; 626 if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) 627 return ERR_PTR(-EINVAL); 628 629 pev = &((*loongarch_pmu.cache_event_map) 630 [cache_type] 631 [cache_op] 632 [cache_result]); 633 634 if (pev->event_id == CACHE_OP_UNSUPPORTED) 635 return ERR_PTR(-ENOENT); 636 637 return pev; 638 } 639 640 static int validate_group(struct perf_event *event) 641 { 642 struct cpu_hw_events fake_cpuc; 643 struct perf_event *sibling, *leader = event->group_leader; 644 645 memset(&fake_cpuc, 0, sizeof(fake_cpuc)); 646 647 if (loongarch_pmu_alloc_counter(&fake_cpuc, &leader->hw) < 0) 648 return -EINVAL; 649 650 for_each_sibling_event(sibling, leader) { 651 if (loongarch_pmu_alloc_counter(&fake_cpuc, &sibling->hw) < 0) 652 return -EINVAL; 653 } 654 655 if (loongarch_pmu_alloc_counter(&fake_cpuc, &event->hw) < 0) 656 return -EINVAL; 657 658 return 0; 659 } 660 661 static void reset_counters(void *arg) 662 { 663 int n; 664 int counters = loongarch_pmu.num_counters; 665 666 for (n = 0; n < counters; n++) { 667 loongarch_pmu_write_control(n, 0); 668 loongarch_pmu.write_counter(n, 0); 669 } 670 } 671 672 static const struct loongarch_perf_event loongson_event_map[PERF_COUNT_HW_MAX] = { 673 PERF_MAP_ALL_UNSUPPORTED, 674 [PERF_COUNT_HW_CPU_CYCLES] = { 0x00 }, 675 [PERF_COUNT_HW_INSTRUCTIONS] = { 0x01 }, 676 [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x08 }, 677 [PERF_COUNT_HW_CACHE_MISSES] = { 0x09 }, 678 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x02 }, 679 [PERF_COUNT_HW_BRANCH_MISSES] = { 0x03 }, 680 }; 681 682 static const struct loongarch_perf_event loongson_cache_map 683 [PERF_COUNT_HW_CACHE_MAX] 684 [PERF_COUNT_HW_CACHE_OP_MAX] 685 [PERF_COUNT_HW_CACHE_RESULT_MAX] = { 686 PERF_CACHE_MAP_ALL_UNSUPPORTED, 687 [C(L1D)] = { 688 /* 689 * Like some other architectures (e.g. ARM), the performance 690 * counters don't differentiate between read and write 691 * accesses/misses, so this isn't strictly correct, but it's the 692 * best we can do. Writes and reads get combined. 693 */ 694 [C(OP_READ)] = { 695 [C(RESULT_ACCESS)] = { 0x8 }, 696 [C(RESULT_MISS)] = { 0x9 }, 697 }, 698 [C(OP_WRITE)] = { 699 [C(RESULT_ACCESS)] = { 0x8 }, 700 [C(RESULT_MISS)] = { 0x9 }, 701 }, 702 [C(OP_PREFETCH)] = { 703 [C(RESULT_ACCESS)] = { 0xaa }, 704 [C(RESULT_MISS)] = { 0xa9 }, 705 }, 706 }, 707 [C(L1I)] = { 708 [C(OP_READ)] = { 709 [C(RESULT_ACCESS)] = { 0x6 }, 710 [C(RESULT_MISS)] = { 0x7 }, 711 }, 712 }, 713 [C(LL)] = { 714 [C(OP_READ)] = { 715 [C(RESULT_ACCESS)] = { 0xc }, 716 [C(RESULT_MISS)] = { 0xd }, 717 }, 718 [C(OP_WRITE)] = { 719 [C(RESULT_ACCESS)] = { 0xc }, 720 [C(RESULT_MISS)] = { 0xd }, 721 }, 722 }, 723 [C(ITLB)] = { 724 [C(OP_READ)] = { 725 [C(RESULT_MISS)] = { 0x3b }, 726 }, 727 }, 728 [C(DTLB)] = { 729 [C(OP_READ)] = { 730 [C(RESULT_ACCESS)] = { 0x4 }, 731 [C(RESULT_MISS)] = { 0x3c }, 732 }, 733 [C(OP_WRITE)] = { 734 [C(RESULT_ACCESS)] = { 0x4 }, 735 [C(RESULT_MISS)] = { 0x3c }, 736 }, 737 }, 738 [C(BPU)] = { 739 /* Using the same code for *HW_BRANCH* */ 740 [C(OP_READ)] = { 741 [C(RESULT_ACCESS)] = { 0x02 }, 742 [C(RESULT_MISS)] = { 0x03 }, 743 }, 744 }, 745 }; 746 747 static int __hw_perf_event_init(struct perf_event *event) 748 { 749 int err; 750 struct hw_perf_event *hwc = &event->hw; 751 struct perf_event_attr *attr = &event->attr; 752 const struct loongarch_perf_event *pev; 753 754 /* Returning LoongArch event descriptor for generic perf event. */ 755 if (PERF_TYPE_HARDWARE == event->attr.type) { 756 if (event->attr.config >= PERF_COUNT_HW_MAX) 757 return -EINVAL; 758 pev = loongarch_pmu_map_general_event(event->attr.config); 759 } else if (PERF_TYPE_HW_CACHE == event->attr.type) { 760 pev = loongarch_pmu_map_cache_event(event->attr.config); 761 } else if (PERF_TYPE_RAW == event->attr.type) { 762 /* We are working on the global raw event. */ 763 mutex_lock(&raw_event_mutex); 764 pev = loongarch_pmu.map_raw_event(event->attr.config); 765 } else { 766 /* The event type is not (yet) supported. */ 767 return -EOPNOTSUPP; 768 } 769 770 if (IS_ERR(pev)) { 771 if (PERF_TYPE_RAW == event->attr.type) 772 mutex_unlock(&raw_event_mutex); 773 return PTR_ERR(pev); 774 } 775 776 /* 777 * We allow max flexibility on how each individual counter shared 778 * by the single CPU operates (the mode exclusion and the range). 779 */ 780 hwc->config_base = CSR_PERFCTRL_IE; 781 782 hwc->event_base = loongarch_pmu_perf_event_encode(pev); 783 if (PERF_TYPE_RAW == event->attr.type) 784 mutex_unlock(&raw_event_mutex); 785 786 if (!attr->exclude_user) { 787 hwc->config_base |= CSR_PERFCTRL_PLV3; 788 hwc->config_base |= CSR_PERFCTRL_PLV2; 789 } 790 if (!attr->exclude_kernel) { 791 hwc->config_base |= CSR_PERFCTRL_PLV0; 792 } 793 if (!attr->exclude_hv) { 794 hwc->config_base |= CSR_PERFCTRL_PLV1; 795 } 796 797 hwc->config_base &= M_PERFCTL_CONFIG_MASK; 798 /* 799 * The event can belong to another cpu. We do not assign a local 800 * counter for it for now. 801 */ 802 hwc->idx = -1; 803 hwc->config = 0; 804 805 if (!hwc->sample_period) { 806 hwc->sample_period = loongarch_pmu.max_period; 807 hwc->last_period = hwc->sample_period; 808 local64_set(&hwc->period_left, hwc->sample_period); 809 } 810 811 err = 0; 812 if (event->group_leader != event) 813 err = validate_group(event); 814 815 event->destroy = hw_perf_event_destroy; 816 817 if (err) 818 event->destroy(event); 819 820 return err; 821 } 822 823 static void pause_local_counters(void) 824 { 825 unsigned long flags; 826 int ctr = loongarch_pmu.num_counters; 827 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 828 829 local_irq_save(flags); 830 do { 831 ctr--; 832 cpuc->saved_ctrl[ctr] = loongarch_pmu_read_control(ctr); 833 loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr] & 834 ~M_PERFCTL_COUNT_EVENT_WHENEVER); 835 } while (ctr > 0); 836 local_irq_restore(flags); 837 } 838 839 static void resume_local_counters(void) 840 { 841 int ctr = loongarch_pmu.num_counters; 842 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events); 843 844 do { 845 ctr--; 846 loongarch_pmu_write_control(ctr, cpuc->saved_ctrl[ctr]); 847 } while (ctr > 0); 848 } 849 850 static const struct loongarch_perf_event *loongarch_pmu_map_raw_event(u64 config) 851 { 852 raw_event.event_id = M_PERFCTL_EVENT(config); 853 854 return &raw_event; 855 } 856 857 static int __init init_hw_perf_events(void) 858 { 859 int counters; 860 861 if (!cpu_has_pmp) 862 return -ENODEV; 863 864 pr_info("Performance counters: "); 865 counters = ((read_cpucfg(LOONGARCH_CPUCFG6) & CPUCFG6_PMNUM) >> 4) + 1; 866 867 loongarch_pmu.num_counters = counters; 868 loongarch_pmu.max_period = (1ULL << 63) - 1; 869 loongarch_pmu.valid_count = (1ULL << 63) - 1; 870 loongarch_pmu.overflow = 1ULL << 63; 871 loongarch_pmu.name = "loongarch/loongson64"; 872 loongarch_pmu.read_counter = loongarch_pmu_read_counter; 873 loongarch_pmu.write_counter = loongarch_pmu_write_counter; 874 loongarch_pmu.map_raw_event = loongarch_pmu_map_raw_event; 875 loongarch_pmu.general_event_map = &loongson_event_map; 876 loongarch_pmu.cache_event_map = &loongson_cache_map; 877 878 on_each_cpu(reset_counters, NULL, 1); 879 880 pr_cont("%s PMU enabled, %d %d-bit counters available to each CPU.\n", 881 loongarch_pmu.name, counters, 64); 882 883 perf_pmu_register(&pmu, "cpu", PERF_TYPE_RAW); 884 885 return 0; 886 } 887 early_initcall(init_hw_perf_events); 888