lbr.c (530bfff6480307d210734222a54d56af7f908957) lbr.c (f42be8651a7a9d5cb165e5d176fc0b09621b4f4d)
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/perf_event.h>
3#include <linux/types.h>
4
5#include <asm/perf_event.h>
6#include <asm/msr.h>
7#include <asm/insn.h>
8

--- 341 unchanged lines hidden (view full) ---

350 }
351
352 wrmsrl(x86_pmu.lbr_tos, tos);
353
354 if (cpuc->lbr_select)
355 wrmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel);
356}
357
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/perf_event.h>
3#include <linux/types.h>
4
5#include <asm/perf_event.h>
6#include <asm/msr.h>
7#include <asm/insn.h>
8

--- 341 unchanged lines hidden (view full) ---

350 }
351
352 wrmsrl(x86_pmu.lbr_tos, tos);
353
354 if (cpuc->lbr_select)
355 wrmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel);
356}
357
358static __always_inline bool
359lbr_is_reset_in_cstate(struct x86_perf_task_context *task_ctx)
358static __always_inline bool lbr_is_reset_in_cstate(void *ctx)
360{
359{
361 return !rdlbr_from(task_ctx->tos);
360 return !rdlbr_from(((struct x86_perf_task_context *)ctx)->tos);
362}
363
361}
362
364static void __intel_pmu_lbr_restore(struct x86_perf_task_context *task_ctx)
363static void __intel_pmu_lbr_restore(void *ctx)
365{
366 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
367
364{
365 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
366
368 if (task_ctx->opt.lbr_callstack_users == 0 ||
369 task_ctx->opt.lbr_stack_state == LBR_NONE) {
367 if (task_context_opt(ctx)->lbr_callstack_users == 0 ||
368 task_context_opt(ctx)->lbr_stack_state == LBR_NONE) {
370 intel_pmu_lbr_reset();
371 return;
372 }
373
374 /*
375 * Does not restore the LBR registers, if
376 * - No one else touched them, and
377 * - Was not cleared in Cstate
378 */
369 intel_pmu_lbr_reset();
370 return;
371 }
372
373 /*
374 * Does not restore the LBR registers, if
375 * - No one else touched them, and
376 * - Was not cleared in Cstate
377 */
379 if ((task_ctx == cpuc->last_task_ctx) &&
380 (task_ctx->opt.log_id == cpuc->last_log_id) &&
381 !lbr_is_reset_in_cstate(task_ctx)) {
382 task_ctx->opt.lbr_stack_state = LBR_NONE;
378 if ((ctx == cpuc->last_task_ctx) &&
379 (task_context_opt(ctx)->log_id == cpuc->last_log_id) &&
380 !lbr_is_reset_in_cstate(ctx)) {
381 task_context_opt(ctx)->lbr_stack_state = LBR_NONE;
383 return;
384 }
385
382 return;
383 }
384
386 x86_pmu.lbr_restore(task_ctx);
385 x86_pmu.lbr_restore(ctx);
387
386
388 task_ctx->opt.lbr_stack_state = LBR_NONE;
387 task_context_opt(ctx)->lbr_stack_state = LBR_NONE;
389}
390
391void intel_pmu_lbr_save(void *ctx)
392{
393 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
394 struct x86_perf_task_context *task_ctx = ctx;
395 unsigned lbr_idx, mask;
396 u64 tos, from;

--- 13 unchanged lines hidden (view full) ---

410 }
411 task_ctx->valid_lbrs = i;
412 task_ctx->tos = tos;
413
414 if (cpuc->lbr_select)
415 rdmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel);
416}
417
388}
389
390void intel_pmu_lbr_save(void *ctx)
391{
392 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
393 struct x86_perf_task_context *task_ctx = ctx;
394 unsigned lbr_idx, mask;
395 u64 tos, from;

--- 13 unchanged lines hidden (view full) ---

409 }
410 task_ctx->valid_lbrs = i;
411 task_ctx->tos = tos;
412
413 if (cpuc->lbr_select)
414 rdmsrl(MSR_LBR_SELECT, task_ctx->lbr_sel);
415}
416
418static void __intel_pmu_lbr_save(struct x86_perf_task_context *task_ctx)
417static void __intel_pmu_lbr_save(void *ctx)
419{
420 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
421
418{
419 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
420
422 if (task_ctx->opt.lbr_callstack_users == 0) {
423 task_ctx->opt.lbr_stack_state = LBR_NONE;
421 if (task_context_opt(ctx)->lbr_callstack_users == 0) {
422 task_context_opt(ctx)->lbr_stack_state = LBR_NONE;
424 return;
425 }
426
423 return;
424 }
425
427 x86_pmu.lbr_save(task_ctx);
426 x86_pmu.lbr_save(ctx);
428
427
429 task_ctx->opt.lbr_stack_state = LBR_VALID;
428 task_context_opt(ctx)->lbr_stack_state = LBR_VALID;
430
429
431 cpuc->last_task_ctx = task_ctx;
432 cpuc->last_log_id = ++task_ctx->opt.log_id;
430 cpuc->last_task_ctx = ctx;
431 cpuc->last_log_id = ++task_context_opt(ctx)->log_id;
433}
434
435void intel_pmu_lbr_swap_task_ctx(struct perf_event_context *prev,
436 struct perf_event_context *next)
437{
432}
433
434void intel_pmu_lbr_swap_task_ctx(struct perf_event_context *prev,
435 struct perf_event_context *next)
436{
438 struct x86_perf_task_context *prev_ctx_data, *next_ctx_data;
437 void *prev_ctx_data, *next_ctx_data;
439
440 swap(prev->task_ctx_data, next->task_ctx_data);
441
442 /*
443 * Architecture specific synchronization makes sense in
444 * case both prev->task_ctx_data and next->task_ctx_data
445 * pointers are allocated.
446 */
447
448 prev_ctx_data = next->task_ctx_data;
449 next_ctx_data = prev->task_ctx_data;
450
451 if (!prev_ctx_data || !next_ctx_data)
452 return;
453
438
439 swap(prev->task_ctx_data, next->task_ctx_data);
440
441 /*
442 * Architecture specific synchronization makes sense in
443 * case both prev->task_ctx_data and next->task_ctx_data
444 * pointers are allocated.
445 */
446
447 prev_ctx_data = next->task_ctx_data;
448 next_ctx_data = prev->task_ctx_data;
449
450 if (!prev_ctx_data || !next_ctx_data)
451 return;
452
454 swap(prev_ctx_data->opt.lbr_callstack_users,
455 next_ctx_data->opt.lbr_callstack_users);
453 swap(task_context_opt(prev_ctx_data)->lbr_callstack_users,
454 task_context_opt(next_ctx_data)->lbr_callstack_users);
456}
457
458void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in)
459{
460 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
455}
456
457void intel_pmu_lbr_sched_task(struct perf_event_context *ctx, bool sched_in)
458{
459 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
461 struct x86_perf_task_context *task_ctx;
460 void *task_ctx;
462
463 if (!cpuc->lbr_users)
464 return;
465
466 /*
467 * If LBR callstack feature is enabled and the stack was saved when
468 * the task was scheduled out, restore the stack. Otherwise flush
469 * the LBR stack.

--- 20 unchanged lines hidden (view full) ---

490static inline bool branch_user_callstack(unsigned br_sel)
491{
492 return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK);
493}
494
495void intel_pmu_lbr_add(struct perf_event *event)
496{
497 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
461
462 if (!cpuc->lbr_users)
463 return;
464
465 /*
466 * If LBR callstack feature is enabled and the stack was saved when
467 * the task was scheduled out, restore the stack. Otherwise flush
468 * the LBR stack.

--- 20 unchanged lines hidden (view full) ---

489static inline bool branch_user_callstack(unsigned br_sel)
490{
491 return (br_sel & X86_BR_USER) && (br_sel & X86_BR_CALL_STACK);
492}
493
494void intel_pmu_lbr_add(struct perf_event *event)
495{
496 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
498 struct x86_perf_task_context *task_ctx;
499
500 if (!x86_pmu.lbr_nr)
501 return;
502
503 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT)
504 cpuc->lbr_select = 1;
505
506 cpuc->br_sel = event->hw.branch_reg.reg;
507
497
498 if (!x86_pmu.lbr_nr)
499 return;
500
501 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT)
502 cpuc->lbr_select = 1;
503
504 cpuc->br_sel = event->hw.branch_reg.reg;
505
508 if (branch_user_callstack(cpuc->br_sel) && event->ctx->task_ctx_data) {
509 task_ctx = event->ctx->task_ctx_data;
510 task_ctx->opt.lbr_callstack_users++;
511 }
506 if (branch_user_callstack(cpuc->br_sel) && event->ctx->task_ctx_data)
507 task_context_opt(event->ctx->task_ctx_data)->lbr_callstack_users++;
512
513 /*
514 * Request pmu::sched_task() callback, which will fire inside the
515 * regular perf event scheduling, so that call will:
516 *
517 * - restore or wipe; when LBR-callstack,
518 * - wipe; otherwise,
519 *

--- 14 unchanged lines hidden (view full) ---

534 perf_sched_cb_inc(event->ctx->pmu);
535 if (!cpuc->lbr_users++ && !event->total_time_running)
536 intel_pmu_lbr_reset();
537}
538
539void intel_pmu_lbr_del(struct perf_event *event)
540{
541 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
508
509 /*
510 * Request pmu::sched_task() callback, which will fire inside the
511 * regular perf event scheduling, so that call will:
512 *
513 * - restore or wipe; when LBR-callstack,
514 * - wipe; otherwise,
515 *

--- 14 unchanged lines hidden (view full) ---

530 perf_sched_cb_inc(event->ctx->pmu);
531 if (!cpuc->lbr_users++ && !event->total_time_running)
532 intel_pmu_lbr_reset();
533}
534
535void intel_pmu_lbr_del(struct perf_event *event)
536{
537 struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
542 struct x86_perf_task_context *task_ctx;
543
544 if (!x86_pmu.lbr_nr)
545 return;
546
547 if (branch_user_callstack(cpuc->br_sel) &&
538
539 if (!x86_pmu.lbr_nr)
540 return;
541
542 if (branch_user_callstack(cpuc->br_sel) &&
548 event->ctx->task_ctx_data) {
549 task_ctx = event->ctx->task_ctx_data;
550 task_ctx->opt.lbr_callstack_users--;
551 }
543 event->ctx->task_ctx_data)
544 task_context_opt(event->ctx->task_ctx_data)->lbr_callstack_users--;
552
553 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT)
554 cpuc->lbr_select = 0;
555
556 if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip > 0)
557 cpuc->lbr_pebs_users--;
558 cpuc->lbr_users--;
559 WARN_ON_ONCE(cpuc->lbr_users < 0);

--- 837 unchanged lines hidden --- 		545
546 if (event->hw.flags & PERF_X86_EVENT_LBR_SELECT)
547 cpuc->lbr_select = 0;
548
549 if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip > 0)
550 cpuc->lbr_pebs_users--;
551 cpuc->lbr_users--;
552 WARN_ON_ONCE(cpuc->lbr_users < 0);

--- 837 unchanged lines hidden ---