1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance events core code: 4 * 5 * Copyright (C) 2008 Linutronix GmbH, Thomas Gleixner <tglx@kernel.org> 6 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 7 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra 8 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 9 */ 10 11 #include <linux/fs.h> 12 #include <linux/mm.h> 13 #include <linux/cpu.h> 14 #include <linux/smp.h> 15 #include <linux/idr.h> 16 #include <linux/file.h> 17 #include <linux/poll.h> 18 #include <linux/slab.h> 19 #include <linux/hash.h> 20 #include <linux/tick.h> 21 #include <linux/sysfs.h> 22 #include <linux/dcache.h> 23 #include <linux/percpu.h> 24 #include <linux/ptrace.h> 25 #include <linux/reboot.h> 26 #include <linux/vmstat.h> 27 #include <linux/device.h> 28 #include <linux/export.h> 29 #include <linux/vmalloc.h> 30 #include <linux/hardirq.h> 31 #include <linux/hugetlb.h> 32 #include <linux/rculist.h> 33 #include <linux/uaccess.h> 34 #include <linux/syscalls.h> 35 #include <linux/anon_inodes.h> 36 #include <linux/kernel_stat.h> 37 #include <linux/cgroup.h> 38 #include <linux/perf_event.h> 39 #include <linux/trace_events.h> 40 #include <linux/hw_breakpoint.h> 41 #include <linux/mm_types.h> 42 #include <linux/module.h> 43 #include <linux/mman.h> 44 #include <linux/compat.h> 45 #include <linux/bpf.h> 46 #include <linux/filter.h> 47 #include <linux/namei.h> 48 #include <linux/parser.h> 49 #include <linux/sched/clock.h> 50 #include <linux/sched/mm.h> 51 #include <linux/proc_ns.h> 52 #include <linux/mount.h> 53 #include <linux/min_heap.h> 54 #include <linux/highmem.h> 55 #include <linux/pgtable.h> 56 #include <linux/buildid.h> 57 #include <linux/task_work.h> 58 #include <linux/percpu-rwsem.h> 59 #include <linux/unwind_deferred.h> 60 #include <linux/kvm_types.h> 61 #include <linux/seq_file.h> 62 63 #include "internal.h" 64 65 #include <asm/irq_regs.h> 66 67 typedef int (*remote_function_f)(void *); 68 69 struct remote_function_call { 70 struct task_struct *p; 71 remote_function_f func; 72 void *info; 73 int ret; 74 }; 75 76 static void remote_function(void *data) 77 { 78 struct remote_function_call *tfc = data; 79 struct task_struct *p = tfc->p; 80 81 if (p) { 82 /* -EAGAIN */ 83 if (task_cpu(p) != smp_processor_id()) 84 return; 85 86 /* 87 * Now that we're on right CPU with IRQs disabled, we can test 88 * if we hit the right task without races. 89 */ 90 91 tfc->ret = -ESRCH; /* No such (running) process */ 92 if (p != current) 93 return; 94 } 95 96 tfc->ret = tfc->func(tfc->info); 97 } 98 99 /** 100 * task_function_call - call a function on the cpu on which a task runs 101 * @p: the task to evaluate 102 * @func: the function to be called 103 * @info: the function call argument 104 * 105 * Calls the function @func when the task is currently running. This might 106 * be on the current CPU, which just calls the function directly. This will 107 * retry due to any failures in smp_call_function_single(), such as if the 108 * task_cpu() goes offline concurrently. 109 * 110 * returns @func return value or -ESRCH or -ENXIO when the process isn't running 111 */ 112 static int 113 task_function_call(struct task_struct *p, remote_function_f func, void *info) 114 { 115 struct remote_function_call data = { 116 .p = p, 117 .func = func, 118 .info = info, 119 .ret = -EAGAIN, 120 }; 121 int ret; 122 123 for (;;) { 124 ret = smp_call_function_single(task_cpu(p), remote_function, 125 &data, 1); 126 if (!ret) 127 ret = data.ret; 128 129 if (ret != -EAGAIN) 130 break; 131 132 cond_resched(); 133 } 134 135 return ret; 136 } 137 138 /** 139 * cpu_function_call - call a function on the cpu 140 * @cpu: target cpu to queue this function 141 * @func: the function to be called 142 * @info: the function call argument 143 * 144 * Calls the function @func on the remote cpu. 145 * 146 * returns: @func return value or -ENXIO when the cpu is offline 147 */ 148 static int cpu_function_call(int cpu, remote_function_f func, void *info) 149 { 150 struct remote_function_call data = { 151 .p = NULL, 152 .func = func, 153 .info = info, 154 .ret = -ENXIO, /* No such CPU */ 155 }; 156 157 smp_call_function_single(cpu, remote_function, &data, 1); 158 159 return data.ret; 160 } 161 162 enum event_type_t { 163 EVENT_FLEXIBLE = 0x01, 164 EVENT_PINNED = 0x02, 165 EVENT_TIME = 0x04, 166 EVENT_FROZEN = 0x08, 167 /* see ctx_resched() for details */ 168 EVENT_CPU = 0x10, 169 EVENT_CGROUP = 0x20, 170 171 /* 172 * EVENT_GUEST is set when scheduling in/out events between the host 173 * and a guest with a mediated vPMU. Among other things, EVENT_GUEST 174 * is used: 175 * 176 * - In for_each_epc() to skip PMUs that don't support events in a 177 * MEDIATED_VPMU guest, i.e. don't need to be context switched. 178 * - To indicate the start/end point of the events in a guest. Guest 179 * running time is deducted for host-only (exclude_guest) events. 180 */ 181 EVENT_GUEST = 0x40, 182 EVENT_FLAGS = EVENT_CGROUP | EVENT_GUEST, 183 /* compound helpers */ 184 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, 185 EVENT_TIME_FROZEN = EVENT_TIME | EVENT_FROZEN, 186 }; 187 188 static inline void __perf_ctx_lock(struct perf_event_context *ctx) 189 { 190 raw_spin_lock(&ctx->lock); 191 WARN_ON_ONCE(ctx->is_active & EVENT_FROZEN); 192 } 193 194 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, 195 struct perf_event_context *ctx) 196 { 197 __perf_ctx_lock(&cpuctx->ctx); 198 if (ctx) 199 __perf_ctx_lock(ctx); 200 } 201 202 static inline void __perf_ctx_unlock(struct perf_event_context *ctx) 203 { 204 /* 205 * If ctx_sched_in() didn't again set any ALL flags, clean up 206 * after ctx_sched_out() by clearing is_active. 207 */ 208 if (ctx->is_active & EVENT_FROZEN) { 209 if (!(ctx->is_active & EVENT_ALL)) 210 ctx->is_active = 0; 211 else 212 ctx->is_active &= ~EVENT_FROZEN; 213 } 214 raw_spin_unlock(&ctx->lock); 215 } 216 217 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, 218 struct perf_event_context *ctx) 219 { 220 if (ctx) 221 __perf_ctx_unlock(ctx); 222 __perf_ctx_unlock(&cpuctx->ctx); 223 } 224 225 typedef struct { 226 struct perf_cpu_context *cpuctx; 227 struct perf_event_context *ctx; 228 } class_perf_ctx_lock_t; 229 230 static inline void class_perf_ctx_lock_destructor(class_perf_ctx_lock_t *_T) 231 { perf_ctx_unlock(_T->cpuctx, _T->ctx); } 232 233 static inline class_perf_ctx_lock_t 234 class_perf_ctx_lock_constructor(struct perf_cpu_context *cpuctx, 235 struct perf_event_context *ctx) 236 { perf_ctx_lock(cpuctx, ctx); return (class_perf_ctx_lock_t){ cpuctx, ctx }; } 237 238 #define TASK_TOMBSTONE ((void *)-1L) 239 240 static bool is_kernel_event(struct perf_event *event) 241 { 242 return READ_ONCE(event->owner) == TASK_TOMBSTONE; 243 } 244 245 static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context); 246 247 struct perf_event_context *perf_cpu_task_ctx(void) 248 { 249 lockdep_assert_irqs_disabled(); 250 return this_cpu_ptr(&perf_cpu_context)->task_ctx; 251 } 252 253 /* 254 * On task ctx scheduling... 255 * 256 * When !ctx->nr_events a task context will not be scheduled. This means 257 * we can disable the scheduler hooks (for performance) without leaving 258 * pending task ctx state. 259 * 260 * This however results in two special cases: 261 * 262 * - removing the last event from a task ctx; this is relatively straight 263 * forward and is done in __perf_remove_from_context. 264 * 265 * - adding the first event to a task ctx; this is tricky because we cannot 266 * rely on ctx->is_active and therefore cannot use event_function_call(). 267 * See perf_install_in_context(). 268 * 269 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set. 270 */ 271 272 typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *, 273 struct perf_event_context *, void *); 274 275 struct event_function_struct { 276 struct perf_event *event; 277 event_f func; 278 void *data; 279 }; 280 281 static int event_function(void *info) 282 { 283 struct event_function_struct *efs = info; 284 struct perf_event *event = efs->event; 285 struct perf_event_context *ctx = event->ctx; 286 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 287 struct perf_event_context *task_ctx = cpuctx->task_ctx; 288 int ret = 0; 289 290 lockdep_assert_irqs_disabled(); 291 292 perf_ctx_lock(cpuctx, task_ctx); 293 /* 294 * Since we do the IPI call without holding ctx->lock things can have 295 * changed, double check we hit the task we set out to hit. 296 */ 297 if (ctx->task) { 298 if (ctx->task != current) { 299 ret = -ESRCH; 300 goto unlock; 301 } 302 303 /* 304 * We only use event_function_call() on established contexts, 305 * and event_function() is only ever called when active (or 306 * rather, we'll have bailed in task_function_call() or the 307 * above ctx->task != current test), therefore we must have 308 * ctx->is_active here. 309 */ 310 WARN_ON_ONCE(!ctx->is_active); 311 /* 312 * And since we have ctx->is_active, cpuctx->task_ctx must 313 * match. 314 */ 315 WARN_ON_ONCE(task_ctx != ctx); 316 } else { 317 WARN_ON_ONCE(&cpuctx->ctx != ctx); 318 } 319 320 efs->func(event, cpuctx, ctx, efs->data); 321 unlock: 322 perf_ctx_unlock(cpuctx, task_ctx); 323 324 return ret; 325 } 326 327 static void event_function_call(struct perf_event *event, event_f func, void *data) 328 { 329 struct perf_event_context *ctx = event->ctx; 330 struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */ 331 struct perf_cpu_context *cpuctx; 332 struct event_function_struct efs = { 333 .event = event, 334 .func = func, 335 .data = data, 336 }; 337 338 if (!event->parent) { 339 /* 340 * If this is a !child event, we must hold ctx::mutex to 341 * stabilize the event->ctx relation. See 342 * perf_event_ctx_lock(). 343 */ 344 lockdep_assert_held(&ctx->mutex); 345 } 346 347 if (!task) { 348 cpu_function_call(event->cpu, event_function, &efs); 349 return; 350 } 351 352 if (task == TASK_TOMBSTONE) 353 return; 354 355 again: 356 if (!task_function_call(task, event_function, &efs)) 357 return; 358 359 local_irq_disable(); 360 cpuctx = this_cpu_ptr(&perf_cpu_context); 361 perf_ctx_lock(cpuctx, ctx); 362 /* 363 * Reload the task pointer, it might have been changed by 364 * a concurrent perf_event_context_sched_out(). 365 */ 366 task = ctx->task; 367 if (task == TASK_TOMBSTONE) 368 goto unlock; 369 if (ctx->is_active) { 370 perf_ctx_unlock(cpuctx, ctx); 371 local_irq_enable(); 372 goto again; 373 } 374 func(event, NULL, ctx, data); 375 unlock: 376 perf_ctx_unlock(cpuctx, ctx); 377 local_irq_enable(); 378 } 379 380 /* 381 * Similar to event_function_call() + event_function(), but hard assumes IRQs 382 * are already disabled and we're on the right CPU. 383 */ 384 static void event_function_local(struct perf_event *event, event_f func, void *data) 385 { 386 struct perf_event_context *ctx = event->ctx; 387 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 388 struct task_struct *task = READ_ONCE(ctx->task); 389 struct perf_event_context *task_ctx = NULL; 390 391 lockdep_assert_irqs_disabled(); 392 393 if (task) { 394 if (task == TASK_TOMBSTONE) 395 return; 396 397 task_ctx = ctx; 398 } 399 400 perf_ctx_lock(cpuctx, task_ctx); 401 402 task = ctx->task; 403 if (task == TASK_TOMBSTONE) 404 goto unlock; 405 406 if (task) { 407 /* 408 * We must be either inactive or active and the right task, 409 * otherwise we're screwed, since we cannot IPI to somewhere 410 * else. 411 */ 412 if (ctx->is_active) { 413 if (WARN_ON_ONCE(task != current)) 414 goto unlock; 415 416 if (WARN_ON_ONCE(cpuctx->task_ctx != ctx)) 417 goto unlock; 418 } 419 } else { 420 WARN_ON_ONCE(&cpuctx->ctx != ctx); 421 } 422 423 func(event, cpuctx, ctx, data); 424 unlock: 425 perf_ctx_unlock(cpuctx, task_ctx); 426 } 427 428 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ 429 PERF_FLAG_FD_OUTPUT |\ 430 PERF_FLAG_PID_CGROUP |\ 431 PERF_FLAG_FD_CLOEXEC) 432 433 /* 434 * branch priv levels that need permission checks 435 */ 436 #define PERF_SAMPLE_BRANCH_PERM_PLM \ 437 (PERF_SAMPLE_BRANCH_KERNEL |\ 438 PERF_SAMPLE_BRANCH_HV) 439 440 /* 441 * perf_sched_events : >0 events exist 442 */ 443 444 static void perf_sched_delayed(struct work_struct *work); 445 DEFINE_STATIC_KEY_FALSE(perf_sched_events); 446 static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed); 447 static DEFINE_MUTEX(perf_sched_mutex); 448 static atomic_t perf_sched_count; 449 450 static DEFINE_PER_CPU(struct pmu_event_list, pmu_sb_events); 451 452 static atomic_t nr_mmap_events __read_mostly; 453 static atomic_t nr_comm_events __read_mostly; 454 static atomic_t nr_namespaces_events __read_mostly; 455 static atomic_t nr_task_events __read_mostly; 456 static atomic_t nr_freq_events __read_mostly; 457 static atomic_t nr_switch_events __read_mostly; 458 static atomic_t nr_ksymbol_events __read_mostly; 459 static atomic_t nr_bpf_events __read_mostly; 460 static atomic_t nr_cgroup_events __read_mostly; 461 static atomic_t nr_text_poke_events __read_mostly; 462 static atomic_t nr_build_id_events __read_mostly; 463 464 static LIST_HEAD(pmus); 465 static DEFINE_MUTEX(pmus_lock); 466 static struct srcu_struct pmus_srcu; 467 static cpumask_var_t perf_online_mask; 468 static cpumask_var_t perf_online_core_mask; 469 static cpumask_var_t perf_online_die_mask; 470 static cpumask_var_t perf_online_cluster_mask; 471 static cpumask_var_t perf_online_pkg_mask; 472 static cpumask_var_t perf_online_sys_mask; 473 static struct kmem_cache *perf_event_cache; 474 475 #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU 476 static DEFINE_PER_CPU(bool, guest_ctx_loaded); 477 478 static __always_inline bool is_guest_mediated_pmu_loaded(void) 479 { 480 return __this_cpu_read(guest_ctx_loaded); 481 } 482 #else 483 static __always_inline bool is_guest_mediated_pmu_loaded(void) 484 { 485 return false; 486 } 487 #endif 488 489 /* 490 * perf event paranoia level: 491 * -1 - not paranoid at all 492 * 0 - disallow raw tracepoint access for unpriv 493 * 1 - disallow cpu events for unpriv 494 * 2 - disallow kernel profiling for unpriv 495 */ 496 int sysctl_perf_event_paranoid __read_mostly = 2; 497 498 /* Minimum for 512 kiB + 1 user control page. 'free' kiB per user. */ 499 static int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); 500 501 /* 502 * max perf event sample rate 503 */ 504 #define DEFAULT_MAX_SAMPLE_RATE 100000 505 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) 506 #define DEFAULT_CPU_TIME_MAX_PERCENT 25 507 508 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; 509 static int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; 510 511 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); 512 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; 513 514 static int perf_sample_allowed_ns __read_mostly = 515 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; 516 517 static void update_perf_cpu_limits(void) 518 { 519 u64 tmp = perf_sample_period_ns; 520 521 tmp *= sysctl_perf_cpu_time_max_percent; 522 tmp = div_u64(tmp, 100); 523 if (!tmp) 524 tmp = 1; 525 526 WRITE_ONCE(perf_sample_allowed_ns, tmp); 527 } 528 529 static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc); 530 531 static int perf_event_max_sample_rate_handler(const struct ctl_table *table, int write, 532 void *buffer, size_t *lenp, loff_t *ppos) 533 { 534 int ret; 535 int perf_cpu = sysctl_perf_cpu_time_max_percent; 536 /* 537 * If throttling is disabled don't allow the write: 538 */ 539 if (write && (perf_cpu == 100 || perf_cpu == 0)) 540 return -EINVAL; 541 542 ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 543 if (ret || !write) 544 return ret; 545 546 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); 547 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 548 update_perf_cpu_limits(); 549 550 return 0; 551 } 552 553 static int perf_cpu_time_max_percent_handler(const struct ctl_table *table, int write, 554 void *buffer, size_t *lenp, loff_t *ppos) 555 { 556 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 557 558 if (ret || !write) 559 return ret; 560 561 if (sysctl_perf_cpu_time_max_percent == 100 || 562 sysctl_perf_cpu_time_max_percent == 0) { 563 printk(KERN_WARNING 564 "perf: Dynamic interrupt throttling disabled, can hang your system!\n"); 565 WRITE_ONCE(perf_sample_allowed_ns, 0); 566 } else { 567 update_perf_cpu_limits(); 568 } 569 570 return 0; 571 } 572 573 static const struct ctl_table events_core_sysctl_table[] = { 574 /* 575 * User-space relies on this file as a feature check for 576 * perf_events being enabled. It's an ABI, do not remove! 577 */ 578 { 579 .procname = "perf_event_paranoid", 580 .data = &sysctl_perf_event_paranoid, 581 .maxlen = sizeof(sysctl_perf_event_paranoid), 582 .mode = 0644, 583 .proc_handler = proc_dointvec, 584 }, 585 { 586 .procname = "perf_event_mlock_kb", 587 .data = &sysctl_perf_event_mlock, 588 .maxlen = sizeof(sysctl_perf_event_mlock), 589 .mode = 0644, 590 .proc_handler = proc_dointvec, 591 }, 592 { 593 .procname = "perf_event_max_sample_rate", 594 .data = &sysctl_perf_event_sample_rate, 595 .maxlen = sizeof(sysctl_perf_event_sample_rate), 596 .mode = 0644, 597 .proc_handler = perf_event_max_sample_rate_handler, 598 .extra1 = SYSCTL_ONE, 599 }, 600 { 601 .procname = "perf_cpu_time_max_percent", 602 .data = &sysctl_perf_cpu_time_max_percent, 603 .maxlen = sizeof(sysctl_perf_cpu_time_max_percent), 604 .mode = 0644, 605 .proc_handler = perf_cpu_time_max_percent_handler, 606 .extra1 = SYSCTL_ZERO, 607 .extra2 = SYSCTL_ONE_HUNDRED, 608 }, 609 }; 610 611 static int __init init_events_core_sysctls(void) 612 { 613 register_sysctl_init("kernel", events_core_sysctl_table); 614 return 0; 615 } 616 core_initcall(init_events_core_sysctls); 617 618 619 /* 620 * perf samples are done in some very critical code paths (NMIs). 621 * If they take too much CPU time, the system can lock up and not 622 * get any real work done. This will drop the sample rate when 623 * we detect that events are taking too long. 624 */ 625 #define NR_ACCUMULATED_SAMPLES 128 626 static DEFINE_PER_CPU(u64, running_sample_length); 627 628 static u64 __report_avg; 629 static u64 __report_allowed; 630 631 static void perf_duration_warn(struct irq_work *w) 632 { 633 printk_ratelimited(KERN_INFO 634 "perf: interrupt took too long (%lld > %lld), lowering " 635 "kernel.perf_event_max_sample_rate to %d\n", 636 __report_avg, __report_allowed, 637 sysctl_perf_event_sample_rate); 638 } 639 640 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn); 641 642 void perf_sample_event_took(u64 sample_len_ns) 643 { 644 u64 max_len = READ_ONCE(perf_sample_allowed_ns); 645 u64 running_len; 646 u64 avg_len; 647 u32 max; 648 649 if (max_len == 0) 650 return; 651 652 /* Decay the counter by 1 average sample. */ 653 running_len = __this_cpu_read(running_sample_length); 654 running_len -= running_len/NR_ACCUMULATED_SAMPLES; 655 running_len += sample_len_ns; 656 __this_cpu_write(running_sample_length, running_len); 657 658 /* 659 * Note: this will be biased artificially low until we have 660 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us 661 * from having to maintain a count. 662 */ 663 avg_len = running_len/NR_ACCUMULATED_SAMPLES; 664 if (avg_len <= max_len) 665 return; 666 667 __report_avg = avg_len; 668 __report_allowed = max_len; 669 670 /* 671 * Compute a throttle threshold 25% below the current duration. 672 */ 673 avg_len += avg_len / 4; 674 max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent; 675 if (avg_len < max) 676 max /= (u32)avg_len; 677 else 678 max = 1; 679 680 WRITE_ONCE(perf_sample_allowed_ns, avg_len); 681 WRITE_ONCE(max_samples_per_tick, max); 682 683 sysctl_perf_event_sample_rate = max * HZ; 684 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 685 686 if (!irq_work_queue(&perf_duration_work)) { 687 early_printk("perf: interrupt took too long (%lld > %lld), lowering " 688 "kernel.perf_event_max_sample_rate to %d\n", 689 __report_avg, __report_allowed, 690 sysctl_perf_event_sample_rate); 691 } 692 } 693 694 static atomic64_t perf_event_id; 695 696 static void update_context_time(struct perf_event_context *ctx); 697 static u64 perf_event_time(struct perf_event *event); 698 699 void __weak perf_event_print_debug(void) { } 700 701 static inline u64 perf_clock(void) 702 { 703 return local_clock(); 704 } 705 706 static inline u64 perf_event_clock(struct perf_event *event) 707 { 708 return event->clock(); 709 } 710 711 /* 712 * State based event timekeeping... 713 * 714 * The basic idea is to use event->state to determine which (if any) time 715 * fields to increment with the current delta. This means we only need to 716 * update timestamps when we change state or when they are explicitly requested 717 * (read). 718 * 719 * Event groups make things a little more complicated, but not terribly so. The 720 * rules for a group are that if the group leader is OFF the entire group is 721 * OFF, irrespective of what the group member states are. This results in 722 * __perf_effective_state(). 723 * 724 * A further ramification is that when a group leader flips between OFF and 725 * !OFF, we need to update all group member times. 726 * 727 * 728 * NOTE: perf_event_time() is based on the (cgroup) context time, and thus we 729 * need to make sure the relevant context time is updated before we try and 730 * update our timestamps. 731 */ 732 733 static __always_inline enum perf_event_state 734 __perf_effective_state(struct perf_event *event) 735 { 736 struct perf_event *leader = event->group_leader; 737 738 if (leader->state <= PERF_EVENT_STATE_OFF) 739 return leader->state; 740 741 return event->state; 742 } 743 744 static __always_inline void 745 __perf_update_times(struct perf_event *event, u64 now, u64 *enabled, u64 *running) 746 { 747 enum perf_event_state state = __perf_effective_state(event); 748 u64 delta = now - event->tstamp; 749 750 *enabled = event->total_time_enabled; 751 if (state >= PERF_EVENT_STATE_INACTIVE) 752 *enabled += delta; 753 754 *running = event->total_time_running; 755 if (state >= PERF_EVENT_STATE_ACTIVE) 756 *running += delta; 757 } 758 759 static void perf_event_update_time(struct perf_event *event) 760 { 761 u64 now = perf_event_time(event); 762 763 __perf_update_times(event, now, &event->total_time_enabled, 764 &event->total_time_running); 765 event->tstamp = now; 766 } 767 768 static void perf_event_update_sibling_time(struct perf_event *leader) 769 { 770 struct perf_event *sibling; 771 772 for_each_sibling_event(sibling, leader) 773 perf_event_update_time(sibling); 774 } 775 776 static void 777 perf_event_set_state(struct perf_event *event, enum perf_event_state state) 778 { 779 if (event->state == state) 780 return; 781 782 perf_event_update_time(event); 783 /* 784 * If a group leader gets enabled/disabled all its siblings 785 * are affected too. 786 */ 787 if ((event->state < 0) ^ (state < 0)) 788 perf_event_update_sibling_time(event); 789 790 WRITE_ONCE(event->state, state); 791 } 792 793 /* 794 * UP store-release, load-acquire 795 */ 796 797 #define __store_release(ptr, val) \ 798 do { \ 799 barrier(); \ 800 WRITE_ONCE(*(ptr), (val)); \ 801 } while (0) 802 803 #define __load_acquire(ptr) \ 804 ({ \ 805 __unqual_scalar_typeof(*(ptr)) ___p = READ_ONCE(*(ptr)); \ 806 barrier(); \ 807 ___p; \ 808 }) 809 810 static bool perf_skip_pmu_ctx(struct perf_event_pmu_context *pmu_ctx, 811 enum event_type_t event_type) 812 { 813 if ((event_type & EVENT_CGROUP) && !pmu_ctx->nr_cgroups) 814 return true; 815 if ((event_type & EVENT_GUEST) && 816 !(pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU)) 817 return true; 818 return false; 819 } 820 821 #define for_each_epc(_epc, _ctx, _pmu, _event_type) \ 822 list_for_each_entry(_epc, &((_ctx)->pmu_ctx_list), pmu_ctx_entry) \ 823 if (perf_skip_pmu_ctx(_epc, _event_type)) \ 824 continue; \ 825 else if (_pmu && _epc->pmu != _pmu) \ 826 continue; \ 827 else 828 829 static void perf_ctx_disable(struct perf_event_context *ctx, 830 enum event_type_t event_type) 831 { 832 struct perf_event_pmu_context *pmu_ctx; 833 834 for_each_epc(pmu_ctx, ctx, NULL, event_type) 835 perf_pmu_disable(pmu_ctx->pmu); 836 } 837 838 static void perf_ctx_enable(struct perf_event_context *ctx, 839 enum event_type_t event_type) 840 { 841 struct perf_event_pmu_context *pmu_ctx; 842 843 for_each_epc(pmu_ctx, ctx, NULL, event_type) 844 perf_pmu_enable(pmu_ctx->pmu); 845 } 846 847 static void ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type); 848 static void ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type); 849 850 static inline void update_perf_time_ctx(struct perf_time_ctx *time, u64 now, bool adv) 851 { 852 if (adv) 853 time->time += now - time->stamp; 854 time->stamp = now; 855 856 /* 857 * The above: time' = time + (now - timestamp), can be re-arranged 858 * into: time` = now + (time - timestamp), which gives a single value 859 * offset to compute future time without locks on. 860 * 861 * See perf_event_time_now(), which can be used from NMI context where 862 * it's (obviously) not possible to acquire ctx->lock in order to read 863 * both the above values in a consistent manner. 864 */ 865 WRITE_ONCE(time->offset, time->time - time->stamp); 866 } 867 868 static_assert(offsetof(struct perf_event_context, timeguest) - 869 offsetof(struct perf_event_context, time) == 870 sizeof(struct perf_time_ctx)); 871 872 #define T_TOTAL 0 873 #define T_GUEST 1 874 875 static inline u64 __perf_event_time_ctx(struct perf_event *event, 876 struct perf_time_ctx *times) 877 { 878 u64 time = times[T_TOTAL].time; 879 880 if (event->attr.exclude_guest) 881 time -= times[T_GUEST].time; 882 883 return time; 884 } 885 886 static inline u64 __perf_event_time_ctx_now(struct perf_event *event, 887 struct perf_time_ctx *times, 888 u64 now) 889 { 890 if (is_guest_mediated_pmu_loaded() && event->attr.exclude_guest) { 891 /* 892 * (now + times[total].offset) - (now + times[guest].offset) := 893 * times[total].offset - times[guest].offset 894 */ 895 return READ_ONCE(times[T_TOTAL].offset) - READ_ONCE(times[T_GUEST].offset); 896 } 897 898 return now + READ_ONCE(times[T_TOTAL].offset); 899 } 900 901 #ifdef CONFIG_CGROUP_PERF 902 903 static inline bool 904 perf_cgroup_match(struct perf_event *event) 905 { 906 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 907 908 /* @event doesn't care about cgroup */ 909 if (!event->cgrp) 910 return true; 911 912 /* wants specific cgroup scope but @cpuctx isn't associated with any */ 913 if (!cpuctx->cgrp) 914 return false; 915 916 /* 917 * Cgroup scoping is recursive. An event enabled for a cgroup is 918 * also enabled for all its descendant cgroups. If @cpuctx's 919 * cgroup is a descendant of @event's (the test covers identity 920 * case), it's a match. 921 */ 922 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 923 event->cgrp->css.cgroup); 924 } 925 926 static inline void perf_detach_cgroup(struct perf_event *event) 927 { 928 css_put(&event->cgrp->css); 929 event->cgrp = NULL; 930 } 931 932 static inline int is_cgroup_event(struct perf_event *event) 933 { 934 return event->cgrp != NULL; 935 } 936 937 static_assert(offsetof(struct perf_cgroup_info, timeguest) - 938 offsetof(struct perf_cgroup_info, time) == 939 sizeof(struct perf_time_ctx)); 940 941 static inline u64 perf_cgroup_event_time(struct perf_event *event) 942 { 943 struct perf_cgroup_info *t; 944 945 t = per_cpu_ptr(event->cgrp->info, event->cpu); 946 return __perf_event_time_ctx(event, &t->time); 947 } 948 949 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now) 950 { 951 struct perf_cgroup_info *t; 952 953 t = per_cpu_ptr(event->cgrp->info, event->cpu); 954 if (!__load_acquire(&t->active)) 955 return __perf_event_time_ctx(event, &t->time); 956 957 return __perf_event_time_ctx_now(event, &t->time, now); 958 } 959 960 static inline void __update_cgrp_guest_time(struct perf_cgroup_info *info, u64 now, bool adv) 961 { 962 update_perf_time_ctx(&info->timeguest, now, adv); 963 } 964 965 static inline void update_cgrp_time(struct perf_cgroup_info *info, u64 now) 966 { 967 update_perf_time_ctx(&info->time, now, true); 968 if (is_guest_mediated_pmu_loaded()) 969 __update_cgrp_guest_time(info, now, true); 970 } 971 972 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, bool final) 973 { 974 struct perf_cgroup *cgrp = cpuctx->cgrp; 975 struct cgroup_subsys_state *css; 976 struct perf_cgroup_info *info; 977 978 if (cgrp) { 979 u64 now = perf_clock(); 980 981 for (css = &cgrp->css; css; css = css->parent) { 982 cgrp = container_of(css, struct perf_cgroup, css); 983 info = this_cpu_ptr(cgrp->info); 984 985 update_cgrp_time(info, now); 986 if (final) 987 __store_release(&info->active, 0); 988 } 989 } 990 } 991 992 static inline void update_cgrp_time_from_event(struct perf_event *event) 993 { 994 struct perf_cgroup_info *info; 995 996 /* 997 * ensure we access cgroup data only when needed and 998 * when we know the cgroup is pinned (css_get) 999 */ 1000 if (!is_cgroup_event(event)) 1001 return; 1002 1003 info = this_cpu_ptr(event->cgrp->info); 1004 /* 1005 * Do not update time when cgroup is not active 1006 */ 1007 if (info->active) 1008 update_cgrp_time(info, perf_clock()); 1009 } 1010 1011 static inline void 1012 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest) 1013 { 1014 struct perf_event_context *ctx = &cpuctx->ctx; 1015 struct perf_cgroup *cgrp = cpuctx->cgrp; 1016 struct perf_cgroup_info *info; 1017 struct cgroup_subsys_state *css; 1018 1019 /* 1020 * ctx->lock held by caller 1021 * ensure we do not access cgroup data 1022 * unless we have the cgroup pinned (css_get) 1023 */ 1024 if (!cgrp) 1025 return; 1026 1027 WARN_ON_ONCE(!ctx->nr_cgroups); 1028 1029 for (css = &cgrp->css; css; css = css->parent) { 1030 cgrp = container_of(css, struct perf_cgroup, css); 1031 info = this_cpu_ptr(cgrp->info); 1032 if (guest) { 1033 __update_cgrp_guest_time(info, ctx->time.stamp, false); 1034 } else { 1035 update_perf_time_ctx(&info->time, ctx->time.stamp, false); 1036 __store_release(&info->active, 1); 1037 } 1038 } 1039 } 1040 1041 /* 1042 * reschedule events based on the cgroup constraint of task. 1043 */ 1044 static void perf_cgroup_switch(struct task_struct *task) 1045 { 1046 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 1047 struct perf_cgroup *cgrp; 1048 1049 /* 1050 * cpuctx->cgrp is set when the first cgroup event enabled, 1051 * and is cleared when the last cgroup event disabled. 1052 */ 1053 if (READ_ONCE(cpuctx->cgrp) == NULL) 1054 return; 1055 1056 cgrp = perf_cgroup_from_task(task, NULL); 1057 if (READ_ONCE(cpuctx->cgrp) == cgrp) 1058 return; 1059 1060 guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx); 1061 /* 1062 * Re-check, could've raced vs perf_remove_from_context(). 1063 */ 1064 if (READ_ONCE(cpuctx->cgrp) == NULL) 1065 return; 1066 1067 WARN_ON_ONCE(cpuctx->ctx.nr_cgroups == 0); 1068 perf_ctx_disable(&cpuctx->ctx, EVENT_CGROUP); 1069 1070 ctx_sched_out(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP); 1071 /* 1072 * must not be done before ctxswout due 1073 * to update_cgrp_time_from_cpuctx() in 1074 * ctx_sched_out() 1075 * 1076 * ctx_sched_out() can consume a pending disable and drop the last cgroup 1077 * event, which clear the cpuctx->cgrp. Cgroup tracking follow active cgroup 1078 * event, so do not resurrect it 1079 */ 1080 if (cpuctx->ctx.nr_cgroups) 1081 cpuctx->cgrp = cgrp; 1082 /* 1083 * set cgrp before ctxsw in to allow 1084 * perf_cgroup_set_timestamp() in ctx_sched_in() 1085 * to not have to pass task around 1086 */ 1087 ctx_sched_in(&cpuctx->ctx, NULL, EVENT_ALL|EVENT_CGROUP); 1088 1089 perf_ctx_enable(&cpuctx->ctx, EVENT_CGROUP); 1090 } 1091 1092 static int perf_cgroup_ensure_storage(struct perf_event *event, 1093 struct cgroup_subsys_state *css) 1094 { 1095 struct perf_cpu_context *cpuctx; 1096 struct perf_event **storage; 1097 int cpu, heap_size, ret = 0; 1098 1099 /* 1100 * Allow storage to have sufficient space for an iterator for each 1101 * possibly nested cgroup plus an iterator for events with no cgroup. 1102 */ 1103 for (heap_size = 1; css; css = css->parent) 1104 heap_size++; 1105 1106 for_each_possible_cpu(cpu) { 1107 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); 1108 if (heap_size <= cpuctx->heap_size) 1109 continue; 1110 1111 storage = kmalloc_node(heap_size * sizeof(struct perf_event *), 1112 GFP_KERNEL, cpu_to_node(cpu)); 1113 if (!storage) { 1114 ret = -ENOMEM; 1115 break; 1116 } 1117 1118 raw_spin_lock_irq(&cpuctx->ctx.lock); 1119 if (cpuctx->heap_size < heap_size) { 1120 swap(cpuctx->heap, storage); 1121 if (storage == cpuctx->heap_default) 1122 storage = NULL; 1123 cpuctx->heap_size = heap_size; 1124 } 1125 raw_spin_unlock_irq(&cpuctx->ctx.lock); 1126 1127 kfree(storage); 1128 } 1129 1130 return ret; 1131 } 1132 1133 static inline int perf_cgroup_connect(int fd, struct perf_event *event, 1134 struct perf_event_attr *attr, 1135 struct perf_event *group_leader) 1136 { 1137 struct perf_cgroup *cgrp; 1138 struct cgroup_subsys_state *css; 1139 CLASS(fd, f)(fd); 1140 int ret = 0; 1141 1142 if (fd_empty(f)) 1143 return -EBADF; 1144 1145 css = css_tryget_online_from_dir(fd_file(f)->f_path.dentry, 1146 &perf_event_cgrp_subsys); 1147 if (IS_ERR(css)) 1148 return PTR_ERR(css); 1149 1150 ret = perf_cgroup_ensure_storage(event, css); 1151 if (ret) 1152 return ret; 1153 1154 cgrp = container_of(css, struct perf_cgroup, css); 1155 event->cgrp = cgrp; 1156 1157 /* 1158 * all events in a group must monitor 1159 * the same cgroup because a task belongs 1160 * to only one perf cgroup at a time 1161 */ 1162 if (group_leader && group_leader->cgrp != cgrp) { 1163 perf_detach_cgroup(event); 1164 ret = -EINVAL; 1165 } 1166 return ret; 1167 } 1168 1169 static inline void 1170 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx) 1171 { 1172 struct perf_cpu_context *cpuctx; 1173 1174 if (!is_cgroup_event(event)) 1175 return; 1176 1177 event->pmu_ctx->nr_cgroups++; 1178 1179 /* 1180 * Because cgroup events are always per-cpu events, 1181 * @ctx == &cpuctx->ctx. 1182 */ 1183 cpuctx = container_of(ctx, struct perf_cpu_context, ctx); 1184 1185 if (ctx->nr_cgroups++) 1186 return; 1187 1188 cpuctx->cgrp = perf_cgroup_from_task(current, ctx); 1189 } 1190 1191 static inline void 1192 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx) 1193 { 1194 struct perf_cpu_context *cpuctx; 1195 1196 if (!is_cgroup_event(event)) 1197 return; 1198 1199 event->pmu_ctx->nr_cgroups--; 1200 1201 /* 1202 * Because cgroup events are always per-cpu events, 1203 * @ctx == &cpuctx->ctx. 1204 */ 1205 cpuctx = container_of(ctx, struct perf_cpu_context, ctx); 1206 1207 if (--ctx->nr_cgroups) 1208 return; 1209 1210 cpuctx->cgrp = NULL; 1211 } 1212 1213 #else /* !CONFIG_CGROUP_PERF */ 1214 1215 static inline bool 1216 perf_cgroup_match(struct perf_event *event) 1217 { 1218 return true; 1219 } 1220 1221 static inline void perf_detach_cgroup(struct perf_event *event) 1222 {} 1223 1224 static inline int is_cgroup_event(struct perf_event *event) 1225 { 1226 return 0; 1227 } 1228 1229 static inline void update_cgrp_time_from_event(struct perf_event *event) 1230 { 1231 } 1232 1233 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx, 1234 bool final) 1235 { 1236 } 1237 1238 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, 1239 struct perf_event_attr *attr, 1240 struct perf_event *group_leader) 1241 { 1242 return -EINVAL; 1243 } 1244 1245 static inline void 1246 perf_cgroup_set_timestamp(struct perf_cpu_context *cpuctx, bool guest) 1247 { 1248 } 1249 1250 static inline u64 perf_cgroup_event_time(struct perf_event *event) 1251 { 1252 return 0; 1253 } 1254 1255 static inline u64 perf_cgroup_event_time_now(struct perf_event *event, u64 now) 1256 { 1257 return 0; 1258 } 1259 1260 static inline void 1261 perf_cgroup_event_enable(struct perf_event *event, struct perf_event_context *ctx) 1262 { 1263 } 1264 1265 static inline void 1266 perf_cgroup_event_disable(struct perf_event *event, struct perf_event_context *ctx) 1267 { 1268 } 1269 1270 static void perf_cgroup_switch(struct task_struct *task) 1271 { 1272 } 1273 #endif 1274 1275 /* 1276 * set default to be dependent on timer tick just 1277 * like original code 1278 */ 1279 #define PERF_CPU_HRTIMER (1000 / HZ) 1280 /* 1281 * function must be called with interrupts disabled 1282 */ 1283 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr) 1284 { 1285 struct perf_cpu_pmu_context *cpc; 1286 bool rotations; 1287 1288 lockdep_assert_irqs_disabled(); 1289 1290 cpc = container_of(hr, struct perf_cpu_pmu_context, hrtimer); 1291 rotations = perf_rotate_context(cpc); 1292 1293 raw_spin_lock(&cpc->hrtimer_lock); 1294 if (rotations) 1295 hrtimer_forward_now(hr, cpc->hrtimer_interval); 1296 else 1297 cpc->hrtimer_active = 0; 1298 raw_spin_unlock(&cpc->hrtimer_lock); 1299 1300 return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART; 1301 } 1302 1303 static void __perf_mux_hrtimer_init(struct perf_cpu_pmu_context *cpc, int cpu) 1304 { 1305 struct hrtimer *timer = &cpc->hrtimer; 1306 struct pmu *pmu = cpc->epc.pmu; 1307 u64 interval; 1308 1309 /* 1310 * check default is sane, if not set then force to 1311 * default interval (1/tick) 1312 */ 1313 interval = pmu->hrtimer_interval_ms; 1314 if (interval < 1) 1315 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; 1316 1317 cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval); 1318 1319 raw_spin_lock_init(&cpc->hrtimer_lock); 1320 hrtimer_setup(timer, perf_mux_hrtimer_handler, CLOCK_MONOTONIC, 1321 HRTIMER_MODE_ABS_PINNED_HARD); 1322 } 1323 1324 static int perf_mux_hrtimer_restart(struct perf_cpu_pmu_context *cpc) 1325 { 1326 struct hrtimer *timer = &cpc->hrtimer; 1327 unsigned long flags; 1328 1329 raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags); 1330 if (!cpc->hrtimer_active) { 1331 cpc->hrtimer_active = 1; 1332 hrtimer_forward_now(timer, cpc->hrtimer_interval); 1333 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED_HARD); 1334 } 1335 raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags); 1336 1337 return 0; 1338 } 1339 1340 static __always_inline struct perf_cpu_pmu_context *this_cpc(struct pmu *pmu) 1341 { 1342 return *this_cpu_ptr(pmu->cpu_pmu_context); 1343 } 1344 1345 static int perf_mux_hrtimer_interval_ipi(void *arg) 1346 { 1347 struct pmu *pmu = arg; 1348 struct perf_cpu_pmu_context *cpc = this_cpc(pmu); 1349 int ms = pmu->hrtimer_interval_ms; 1350 1351 cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * ms); 1352 return perf_mux_hrtimer_restart(cpc); 1353 } 1354 1355 void perf_pmu_disable(struct pmu *pmu) 1356 { 1357 int *count = &this_cpc(pmu)->pmu_disable_count; 1358 if (!(*count)++) 1359 pmu->pmu_disable(pmu); 1360 } 1361 1362 void perf_pmu_enable(struct pmu *pmu) 1363 { 1364 int *count = &this_cpc(pmu)->pmu_disable_count; 1365 if (!--(*count)) 1366 pmu->pmu_enable(pmu); 1367 } 1368 1369 static void perf_assert_pmu_disabled(struct pmu *pmu) 1370 { 1371 int *count = &this_cpc(pmu)->pmu_disable_count; 1372 WARN_ON_ONCE(*count == 0); 1373 } 1374 1375 static inline void perf_pmu_read(struct perf_event *event) 1376 { 1377 if (event->state == PERF_EVENT_STATE_ACTIVE) 1378 event->pmu->read(event); 1379 } 1380 1381 static void get_ctx(struct perf_event_context *ctx) 1382 { 1383 refcount_inc(&ctx->refcount); 1384 } 1385 1386 static void free_ctx(struct rcu_head *head) 1387 { 1388 struct perf_event_context *ctx; 1389 1390 ctx = container_of(head, struct perf_event_context, rcu_head); 1391 kfree(ctx); 1392 } 1393 1394 static void put_ctx(struct perf_event_context *ctx) 1395 { 1396 if (refcount_dec_and_test(&ctx->refcount)) { 1397 if (ctx->parent_ctx) 1398 put_ctx(ctx->parent_ctx); 1399 if (ctx->task && ctx->task != TASK_TOMBSTONE) 1400 put_task_struct(ctx->task); 1401 call_rcu(&ctx->rcu_head, free_ctx); 1402 } else { 1403 smp_mb__after_atomic(); /* pairs with wait_var_event() */ 1404 if (ctx->task == TASK_TOMBSTONE) 1405 wake_up_var(&ctx->refcount); 1406 } 1407 } 1408 1409 /* 1410 * Because of perf_event::ctx migration in sys_perf_event_open::move_group and 1411 * perf_pmu_migrate_context() we need some magic. 1412 * 1413 * Those places that change perf_event::ctx will hold both 1414 * perf_event_ctx::mutex of the 'old' and 'new' ctx value. 1415 * 1416 * Lock ordering is by mutex address. There are two other sites where 1417 * perf_event_context::mutex nests and those are: 1418 * 1419 * - perf_event_exit_task_context() [ child , 0 ] 1420 * perf_event_exit_event() 1421 * put_event() [ parent, 1 ] 1422 * 1423 * - perf_event_init_context() [ parent, 0 ] 1424 * inherit_task_group() 1425 * inherit_group() 1426 * inherit_event() 1427 * perf_event_alloc() 1428 * perf_init_event() 1429 * perf_try_init_event() [ child , 1 ] 1430 * 1431 * While it appears there is an obvious deadlock here -- the parent and child 1432 * nesting levels are inverted between the two. This is in fact safe because 1433 * life-time rules separate them. That is an exiting task cannot fork, and a 1434 * spawning task cannot (yet) exit. 1435 * 1436 * But remember that these are parent<->child context relations, and 1437 * migration does not affect children, therefore these two orderings should not 1438 * interact. 1439 * 1440 * The change in perf_event::ctx does not affect children (as claimed above) 1441 * because the sys_perf_event_open() case will install a new event and break 1442 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only 1443 * concerned with cpuctx and that doesn't have children. 1444 * 1445 * The places that change perf_event::ctx will issue: 1446 * 1447 * perf_remove_from_context(); 1448 * synchronize_rcu(); 1449 * perf_install_in_context(); 1450 * 1451 * to affect the change. The remove_from_context() + synchronize_rcu() should 1452 * quiesce the event, after which we can install it in the new location. This 1453 * means that only external vectors (perf_fops, prctl) can perturb the event 1454 * while in transit. Therefore all such accessors should also acquire 1455 * perf_event_context::mutex to serialize against this. 1456 * 1457 * However; because event->ctx can change while we're waiting to acquire 1458 * ctx->mutex we must be careful and use the below perf_event_ctx_lock() 1459 * function. 1460 * 1461 * Lock order: 1462 * exec_update_lock 1463 * task_struct::perf_event_mutex 1464 * perf_event_context::mutex 1465 * perf_event::child_mutex; 1466 * perf_event_context::lock 1467 * mmap_lock 1468 * perf_event::mmap_mutex 1469 * perf_buffer::aux_mutex 1470 * perf_addr_filters_head::lock 1471 * 1472 * cpu_hotplug_lock 1473 * pmus_lock 1474 * cpuctx->mutex / perf_event_context::mutex 1475 */ 1476 static struct perf_event_context * 1477 perf_event_ctx_lock_nested(struct perf_event *event, int nesting) 1478 { 1479 struct perf_event_context *ctx; 1480 1481 again: 1482 rcu_read_lock(); 1483 ctx = READ_ONCE(event->ctx); 1484 if (!refcount_inc_not_zero(&ctx->refcount)) { 1485 rcu_read_unlock(); 1486 goto again; 1487 } 1488 rcu_read_unlock(); 1489 1490 mutex_lock_nested(&ctx->mutex, nesting); 1491 if (event->ctx != ctx) { 1492 mutex_unlock(&ctx->mutex); 1493 put_ctx(ctx); 1494 goto again; 1495 } 1496 1497 return ctx; 1498 } 1499 1500 static inline struct perf_event_context * 1501 perf_event_ctx_lock(struct perf_event *event) 1502 { 1503 return perf_event_ctx_lock_nested(event, 0); 1504 } 1505 1506 static void perf_event_ctx_unlock(struct perf_event *event, 1507 struct perf_event_context *ctx) 1508 { 1509 mutex_unlock(&ctx->mutex); 1510 put_ctx(ctx); 1511 } 1512 1513 /* 1514 * This must be done under the ctx->lock, such as to serialize against 1515 * context_equiv(), therefore we cannot call put_ctx() since that might end up 1516 * calling scheduler related locks and ctx->lock nests inside those. 1517 */ 1518 static __must_check struct perf_event_context * 1519 unclone_ctx(struct perf_event_context *ctx) 1520 { 1521 struct perf_event_context *parent_ctx = ctx->parent_ctx; 1522 1523 lockdep_assert_held(&ctx->lock); 1524 1525 if (parent_ctx) 1526 ctx->parent_ctx = NULL; 1527 ctx->generation++; 1528 1529 return parent_ctx; 1530 } 1531 1532 static u32 perf_event_pid_type(struct perf_event *event, struct task_struct *p, 1533 enum pid_type type) 1534 { 1535 u32 nr; 1536 /* 1537 * only top level events have the pid namespace they were created in 1538 */ 1539 if (event->parent) 1540 event = event->parent; 1541 1542 nr = __task_pid_nr_ns(p, type, event->ns); 1543 /* avoid -1 if it is idle thread or runs in another ns */ 1544 if (!nr && !pid_alive(p)) 1545 nr = -1; 1546 return nr; 1547 } 1548 1549 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p) 1550 { 1551 return perf_event_pid_type(event, p, PIDTYPE_TGID); 1552 } 1553 1554 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p) 1555 { 1556 return perf_event_pid_type(event, p, PIDTYPE_PID); 1557 } 1558 1559 /* 1560 * If we inherit events we want to return the parent event id 1561 * to userspace. 1562 */ 1563 static u64 primary_event_id(struct perf_event *event) 1564 { 1565 u64 id = event->id; 1566 1567 if (event->parent) 1568 id = event->parent->id; 1569 1570 return id; 1571 } 1572 1573 /* 1574 * Get the perf_event_context for a task and lock it. 1575 * 1576 * This has to cope with the fact that until it is locked, 1577 * the context could get moved to another task. 1578 */ 1579 static struct perf_event_context * 1580 perf_lock_task_context(struct task_struct *task, unsigned long *flags) 1581 { 1582 struct perf_event_context *ctx; 1583 1584 retry: 1585 /* 1586 * One of the few rules of preemptible RCU is that one cannot do 1587 * rcu_read_unlock() while holding a scheduler (or nested) lock when 1588 * part of the read side critical section was irqs-enabled -- see 1589 * rcu_read_unlock_special(). 1590 * 1591 * Since ctx->lock nests under rq->lock we must ensure the entire read 1592 * side critical section has interrupts disabled. 1593 */ 1594 local_irq_save(*flags); 1595 rcu_read_lock(); 1596 ctx = rcu_dereference(task->perf_event_ctxp); 1597 if (ctx) { 1598 /* 1599 * If this context is a clone of another, it might 1600 * get swapped for another underneath us by 1601 * perf_event_task_sched_out, though the 1602 * rcu_read_lock() protects us from any context 1603 * getting freed. Lock the context and check if it 1604 * got swapped before we could get the lock, and retry 1605 * if so. If we locked the right context, then it 1606 * can't get swapped on us any more. 1607 */ 1608 raw_spin_lock(&ctx->lock); 1609 if (ctx != rcu_dereference(task->perf_event_ctxp)) { 1610 raw_spin_unlock(&ctx->lock); 1611 rcu_read_unlock(); 1612 local_irq_restore(*flags); 1613 goto retry; 1614 } 1615 1616 if (ctx->task == TASK_TOMBSTONE || 1617 !refcount_inc_not_zero(&ctx->refcount)) { 1618 raw_spin_unlock(&ctx->lock); 1619 ctx = NULL; 1620 } else { 1621 WARN_ON_ONCE(ctx->task != task); 1622 } 1623 } 1624 rcu_read_unlock(); 1625 if (!ctx) 1626 local_irq_restore(*flags); 1627 return ctx; 1628 } 1629 1630 /* 1631 * Get the context for a task and increment its pin_count so it 1632 * can't get swapped to another task. This also increments its 1633 * reference count so that the context can't get freed. 1634 */ 1635 static struct perf_event_context * 1636 perf_pin_task_context(struct task_struct *task) 1637 { 1638 struct perf_event_context *ctx; 1639 unsigned long flags; 1640 1641 ctx = perf_lock_task_context(task, &flags); 1642 if (ctx) { 1643 ++ctx->pin_count; 1644 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1645 } 1646 return ctx; 1647 } 1648 1649 static void perf_unpin_context(struct perf_event_context *ctx) 1650 { 1651 unsigned long flags; 1652 1653 raw_spin_lock_irqsave(&ctx->lock, flags); 1654 --ctx->pin_count; 1655 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1656 } 1657 1658 /* 1659 * Update the record of the current time in a context. 1660 */ 1661 static void __update_context_time(struct perf_event_context *ctx, bool adv) 1662 { 1663 lockdep_assert_held(&ctx->lock); 1664 1665 update_perf_time_ctx(&ctx->time, perf_clock(), adv); 1666 } 1667 1668 static void __update_context_guest_time(struct perf_event_context *ctx, bool adv) 1669 { 1670 lockdep_assert_held(&ctx->lock); 1671 1672 /* must be called after __update_context_time(); */ 1673 update_perf_time_ctx(&ctx->timeguest, ctx->time.stamp, adv); 1674 } 1675 1676 static void update_context_time(struct perf_event_context *ctx) 1677 { 1678 __update_context_time(ctx, true); 1679 if (is_guest_mediated_pmu_loaded()) 1680 __update_context_guest_time(ctx, true); 1681 } 1682 1683 static u64 perf_event_time(struct perf_event *event) 1684 { 1685 struct perf_event_context *ctx = event->ctx; 1686 1687 if (unlikely(!ctx)) 1688 return 0; 1689 1690 if (is_cgroup_event(event)) 1691 return perf_cgroup_event_time(event); 1692 1693 return __perf_event_time_ctx(event, &ctx->time); 1694 } 1695 1696 static u64 perf_event_time_now(struct perf_event *event, u64 now) 1697 { 1698 struct perf_event_context *ctx = event->ctx; 1699 1700 if (unlikely(!ctx)) 1701 return 0; 1702 1703 if (is_cgroup_event(event)) 1704 return perf_cgroup_event_time_now(event, now); 1705 1706 if (!(__load_acquire(&ctx->is_active) & EVENT_TIME)) 1707 return __perf_event_time_ctx(event, &ctx->time); 1708 1709 return __perf_event_time_ctx_now(event, &ctx->time, now); 1710 } 1711 1712 static enum event_type_t get_event_type(struct perf_event *event) 1713 { 1714 struct perf_event_context *ctx = event->ctx; 1715 enum event_type_t event_type; 1716 1717 lockdep_assert_held(&ctx->lock); 1718 1719 /* 1720 * It's 'group type', really, because if our group leader is 1721 * pinned, so are we. 1722 */ 1723 if (event->group_leader != event) 1724 event = event->group_leader; 1725 1726 event_type = event->attr.pinned ? EVENT_PINNED : EVENT_FLEXIBLE; 1727 if (!ctx->task) 1728 event_type |= EVENT_CPU; 1729 1730 return event_type; 1731 } 1732 1733 /* 1734 * Helper function to initialize event group nodes. 1735 */ 1736 static void init_event_group(struct perf_event *event) 1737 { 1738 RB_CLEAR_NODE(&event->group_node); 1739 event->group_index = 0; 1740 } 1741 1742 /* 1743 * Extract pinned or flexible groups from the context 1744 * based on event attrs bits. 1745 */ 1746 static struct perf_event_groups * 1747 get_event_groups(struct perf_event *event, struct perf_event_context *ctx) 1748 { 1749 if (event->attr.pinned) 1750 return &ctx->pinned_groups; 1751 else 1752 return &ctx->flexible_groups; 1753 } 1754 1755 /* 1756 * Helper function to initializes perf_event_group trees. 1757 */ 1758 static void perf_event_groups_init(struct perf_event_groups *groups) 1759 { 1760 groups->tree = RB_ROOT; 1761 groups->index = 0; 1762 } 1763 1764 static inline struct cgroup *event_cgroup(const struct perf_event *event) 1765 { 1766 struct cgroup *cgroup = NULL; 1767 1768 #ifdef CONFIG_CGROUP_PERF 1769 if (event->cgrp) 1770 cgroup = event->cgrp->css.cgroup; 1771 #endif 1772 1773 return cgroup; 1774 } 1775 1776 /* 1777 * Compare function for event groups; 1778 * 1779 * Implements complex key that first sorts by CPU and then by virtual index 1780 * which provides ordering when rotating groups for the same CPU. 1781 */ 1782 static __always_inline int 1783 perf_event_groups_cmp(const int left_cpu, const struct pmu *left_pmu, 1784 const struct cgroup *left_cgroup, const u64 left_group_index, 1785 const struct perf_event *right) 1786 { 1787 if (left_cpu < right->cpu) 1788 return -1; 1789 if (left_cpu > right->cpu) 1790 return 1; 1791 1792 if (left_pmu) { 1793 if (left_pmu < right->pmu_ctx->pmu) 1794 return -1; 1795 if (left_pmu > right->pmu_ctx->pmu) 1796 return 1; 1797 } 1798 1799 #ifdef CONFIG_CGROUP_PERF 1800 { 1801 const struct cgroup *right_cgroup = event_cgroup(right); 1802 1803 if (left_cgroup != right_cgroup) { 1804 if (!left_cgroup) { 1805 /* 1806 * Left has no cgroup but right does, no 1807 * cgroups come first. 1808 */ 1809 return -1; 1810 } 1811 if (!right_cgroup) { 1812 /* 1813 * Right has no cgroup but left does, no 1814 * cgroups come first. 1815 */ 1816 return 1; 1817 } 1818 /* Two dissimilar cgroups, order by id. */ 1819 if (cgroup_id(left_cgroup) < cgroup_id(right_cgroup)) 1820 return -1; 1821 1822 return 1; 1823 } 1824 } 1825 #endif 1826 1827 if (left_group_index < right->group_index) 1828 return -1; 1829 if (left_group_index > right->group_index) 1830 return 1; 1831 1832 return 0; 1833 } 1834 1835 #define __node_2_pe(node) \ 1836 rb_entry((node), struct perf_event, group_node) 1837 1838 static inline bool __group_less(struct rb_node *a, const struct rb_node *b) 1839 { 1840 struct perf_event *e = __node_2_pe(a); 1841 return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e), 1842 e->group_index, __node_2_pe(b)) < 0; 1843 } 1844 1845 struct __group_key { 1846 int cpu; 1847 struct pmu *pmu; 1848 struct cgroup *cgroup; 1849 }; 1850 1851 static inline int __group_cmp(const void *key, const struct rb_node *node) 1852 { 1853 const struct __group_key *a = key; 1854 const struct perf_event *b = __node_2_pe(node); 1855 1856 /* partial/subtree match: @cpu, @pmu, @cgroup; ignore: @group_index */ 1857 return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b); 1858 } 1859 1860 static inline int 1861 __group_cmp_ignore_cgroup(const void *key, const struct rb_node *node) 1862 { 1863 const struct __group_key *a = key; 1864 const struct perf_event *b = __node_2_pe(node); 1865 1866 /* partial/subtree match: @cpu, @pmu, ignore: @cgroup, @group_index */ 1867 return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b), 1868 b->group_index, b); 1869 } 1870 1871 /* 1872 * Insert @event into @groups' tree; using 1873 * {@event->cpu, @event->pmu_ctx->pmu, event_cgroup(@event), ++@groups->index} 1874 * as key. This places it last inside the {cpu,pmu,cgroup} subtree. 1875 */ 1876 static void 1877 perf_event_groups_insert(struct perf_event_groups *groups, 1878 struct perf_event *event) 1879 { 1880 event->group_index = ++groups->index; 1881 1882 rb_add(&event->group_node, &groups->tree, __group_less); 1883 } 1884 1885 /* 1886 * Helper function to insert event into the pinned or flexible groups. 1887 */ 1888 static void 1889 add_event_to_groups(struct perf_event *event, struct perf_event_context *ctx) 1890 { 1891 struct perf_event_groups *groups; 1892 1893 groups = get_event_groups(event, ctx); 1894 perf_event_groups_insert(groups, event); 1895 } 1896 1897 /* 1898 * Delete a group from a tree. 1899 */ 1900 static void 1901 perf_event_groups_delete(struct perf_event_groups *groups, 1902 struct perf_event *event) 1903 { 1904 WARN_ON_ONCE(RB_EMPTY_NODE(&event->group_node) || 1905 RB_EMPTY_ROOT(&groups->tree)); 1906 1907 rb_erase(&event->group_node, &groups->tree); 1908 init_event_group(event); 1909 } 1910 1911 /* 1912 * Helper function to delete event from its groups. 1913 */ 1914 static void 1915 del_event_from_groups(struct perf_event *event, struct perf_event_context *ctx) 1916 { 1917 struct perf_event_groups *groups; 1918 1919 groups = get_event_groups(event, ctx); 1920 perf_event_groups_delete(groups, event); 1921 } 1922 1923 /* 1924 * Get the leftmost event in the {cpu,pmu,cgroup} subtree. 1925 */ 1926 static struct perf_event * 1927 perf_event_groups_first(struct perf_event_groups *groups, int cpu, 1928 struct pmu *pmu, struct cgroup *cgrp) 1929 { 1930 struct __group_key key = { 1931 .cpu = cpu, 1932 .pmu = pmu, 1933 .cgroup = cgrp, 1934 }; 1935 struct rb_node *node; 1936 1937 node = rb_find_first(&key, &groups->tree, __group_cmp); 1938 if (node) 1939 return __node_2_pe(node); 1940 1941 return NULL; 1942 } 1943 1944 static struct perf_event * 1945 perf_event_groups_next(struct perf_event *event, struct pmu *pmu) 1946 { 1947 struct __group_key key = { 1948 .cpu = event->cpu, 1949 .pmu = pmu, 1950 .cgroup = event_cgroup(event), 1951 }; 1952 struct rb_node *next; 1953 1954 next = rb_next_match(&key, &event->group_node, __group_cmp); 1955 if (next) 1956 return __node_2_pe(next); 1957 1958 return NULL; 1959 } 1960 1961 #define perf_event_groups_for_cpu_pmu(event, groups, cpu, pmu) \ 1962 for (event = perf_event_groups_first(groups, cpu, pmu, NULL); \ 1963 event; event = perf_event_groups_next(event, pmu)) 1964 1965 /* 1966 * Iterate through the whole groups tree. 1967 */ 1968 #define perf_event_groups_for_each(event, groups) \ 1969 for (event = rb_entry_safe(rb_first(&((groups)->tree)), \ 1970 typeof(*event), group_node); event; \ 1971 event = rb_entry_safe(rb_next(&event->group_node), \ 1972 typeof(*event), group_node)) 1973 1974 /* 1975 * Does the event attribute request inherit with PERF_SAMPLE_READ 1976 */ 1977 static inline bool has_inherit_and_sample_read(struct perf_event_attr *attr) 1978 { 1979 return attr->inherit && (attr->sample_type & PERF_SAMPLE_READ); 1980 } 1981 1982 /* 1983 * Add an event from the lists for its context. 1984 * Must be called with ctx->mutex and ctx->lock held. 1985 */ 1986 static void 1987 list_add_event(struct perf_event *event, struct perf_event_context *ctx) 1988 { 1989 lockdep_assert_held(&ctx->lock); 1990 1991 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 1992 event->attach_state |= PERF_ATTACH_CONTEXT; 1993 1994 event->tstamp = perf_event_time(event); 1995 1996 /* 1997 * If we're a stand alone event or group leader, we go to the context 1998 * list, group events are kept attached to the group so that 1999 * perf_group_detach can, at all times, locate all siblings. 2000 */ 2001 if (event->group_leader == event) { 2002 event->group_caps = event->event_caps; 2003 add_event_to_groups(event, ctx); 2004 } 2005 2006 list_add_rcu(&event->event_entry, &ctx->event_list); 2007 ctx->nr_events++; 2008 if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) 2009 ctx->nr_user++; 2010 if (event->attr.inherit_stat) 2011 ctx->nr_stat++; 2012 if (has_inherit_and_sample_read(&event->attr)) 2013 local_inc(&ctx->nr_no_switch_fast); 2014 2015 if (event->state > PERF_EVENT_STATE_OFF) 2016 perf_cgroup_event_enable(event, ctx); 2017 2018 ctx->generation++; 2019 event->pmu_ctx->nr_events++; 2020 } 2021 2022 /* 2023 * Initialize event state based on the perf_event_attr::disabled. 2024 */ 2025 static inline void perf_event__state_init(struct perf_event *event) 2026 { 2027 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : 2028 PERF_EVENT_STATE_INACTIVE; 2029 } 2030 2031 static int __perf_event_read_size(u64 read_format, int nr_siblings) 2032 { 2033 int entry = sizeof(u64); /* value */ 2034 int size = 0; 2035 int nr = 1; 2036 2037 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 2038 size += sizeof(u64); 2039 2040 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 2041 size += sizeof(u64); 2042 2043 if (read_format & PERF_FORMAT_ID) 2044 entry += sizeof(u64); 2045 2046 if (read_format & PERF_FORMAT_LOST) 2047 entry += sizeof(u64); 2048 2049 if (read_format & PERF_FORMAT_GROUP) { 2050 nr += nr_siblings; 2051 size += sizeof(u64); 2052 } 2053 2054 /* 2055 * Since perf_event_validate_size() limits this to 16k and inhibits 2056 * adding more siblings, this will never overflow. 2057 */ 2058 return size + nr * entry; 2059 } 2060 2061 static void __perf_event_header_size(struct perf_event *event, u64 sample_type) 2062 { 2063 struct perf_sample_data *data; 2064 u16 size = 0; 2065 2066 if (sample_type & PERF_SAMPLE_IP) 2067 size += sizeof(data->ip); 2068 2069 if (sample_type & PERF_SAMPLE_ADDR) 2070 size += sizeof(data->addr); 2071 2072 if (sample_type & PERF_SAMPLE_PERIOD) 2073 size += sizeof(data->period); 2074 2075 if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) 2076 size += sizeof(data->weight.full); 2077 2078 if (sample_type & PERF_SAMPLE_READ) 2079 size += event->read_size; 2080 2081 if (sample_type & PERF_SAMPLE_DATA_SRC) 2082 size += sizeof(data->data_src.val); 2083 2084 if (sample_type & PERF_SAMPLE_TRANSACTION) 2085 size += sizeof(data->txn); 2086 2087 if (sample_type & PERF_SAMPLE_PHYS_ADDR) 2088 size += sizeof(data->phys_addr); 2089 2090 if (sample_type & PERF_SAMPLE_CGROUP) 2091 size += sizeof(data->cgroup); 2092 2093 if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) 2094 size += sizeof(data->data_page_size); 2095 2096 if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) 2097 size += sizeof(data->code_page_size); 2098 2099 event->header_size = size; 2100 } 2101 2102 /* 2103 * Called at perf_event creation and when events are attached/detached from a 2104 * group. 2105 */ 2106 static void perf_event__header_size(struct perf_event *event) 2107 { 2108 event->read_size = 2109 __perf_event_read_size(event->attr.read_format, 2110 event->group_leader->nr_siblings); 2111 __perf_event_header_size(event, event->attr.sample_type); 2112 } 2113 2114 static void perf_event__id_header_size(struct perf_event *event) 2115 { 2116 struct perf_sample_data *data; 2117 u64 sample_type = event->attr.sample_type; 2118 u16 size = 0; 2119 2120 if (sample_type & PERF_SAMPLE_TID) 2121 size += sizeof(data->tid_entry); 2122 2123 if (sample_type & PERF_SAMPLE_TIME) 2124 size += sizeof(data->time); 2125 2126 if (sample_type & PERF_SAMPLE_IDENTIFIER) 2127 size += sizeof(data->id); 2128 2129 if (sample_type & PERF_SAMPLE_ID) 2130 size += sizeof(data->id); 2131 2132 if (sample_type & PERF_SAMPLE_STREAM_ID) 2133 size += sizeof(data->stream_id); 2134 2135 if (sample_type & PERF_SAMPLE_CPU) 2136 size += sizeof(data->cpu_entry); 2137 2138 event->id_header_size = size; 2139 } 2140 2141 /* 2142 * Check that adding an event to the group does not result in anybody 2143 * overflowing the 64k event limit imposed by the output buffer. 2144 * 2145 * Specifically, check that the read_size for the event does not exceed 16k, 2146 * read_size being the one term that grows with groups size. Since read_size 2147 * depends on per-event read_format, also (re)check the existing events. 2148 * 2149 * This leaves 48k for the constant size fields and things like callchains, 2150 * branch stacks and register sets. 2151 */ 2152 static bool perf_event_validate_size(struct perf_event *event) 2153 { 2154 struct perf_event *sibling, *group_leader = event->group_leader; 2155 2156 if (__perf_event_read_size(event->attr.read_format, 2157 group_leader->nr_siblings + 1) > 16*1024) 2158 return false; 2159 2160 if (__perf_event_read_size(group_leader->attr.read_format, 2161 group_leader->nr_siblings + 1) > 16*1024) 2162 return false; 2163 2164 /* 2165 * When creating a new group leader, group_leader->ctx is initialized 2166 * after the size has been validated, but we cannot safely use 2167 * for_each_sibling_event() until group_leader->ctx is set. A new group 2168 * leader cannot have any siblings yet, so we can safely skip checking 2169 * the non-existent siblings. 2170 */ 2171 if (event == group_leader) 2172 return true; 2173 2174 for_each_sibling_event(sibling, group_leader) { 2175 if (__perf_event_read_size(sibling->attr.read_format, 2176 group_leader->nr_siblings + 1) > 16*1024) 2177 return false; 2178 } 2179 2180 return true; 2181 } 2182 2183 static void perf_group_attach(struct perf_event *event) 2184 { 2185 struct perf_event *group_leader = event->group_leader, *pos; 2186 2187 lockdep_assert_held(&event->ctx->lock); 2188 2189 /* 2190 * We can have double attach due to group movement (move_group) in 2191 * perf_event_open(). 2192 */ 2193 if (event->attach_state & PERF_ATTACH_GROUP) 2194 return; 2195 2196 event->attach_state |= PERF_ATTACH_GROUP; 2197 2198 if (group_leader == event) 2199 return; 2200 2201 WARN_ON_ONCE(group_leader->ctx != event->ctx); 2202 2203 group_leader->group_caps &= event->event_caps; 2204 2205 list_add_tail(&event->sibling_list, &group_leader->sibling_list); 2206 group_leader->nr_siblings++; 2207 group_leader->group_generation++; 2208 2209 perf_event__header_size(group_leader); 2210 2211 for_each_sibling_event(pos, group_leader) 2212 perf_event__header_size(pos); 2213 } 2214 2215 /* 2216 * Remove an event from the lists for its context. 2217 * Must be called with ctx->mutex and ctx->lock held. 2218 */ 2219 static void 2220 list_del_event(struct perf_event *event, struct perf_event_context *ctx) 2221 { 2222 WARN_ON_ONCE(event->ctx != ctx); 2223 lockdep_assert_held(&ctx->lock); 2224 2225 /* 2226 * We can have double detach due to exit/hot-unplug + close. 2227 */ 2228 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) 2229 return; 2230 2231 event->attach_state &= ~PERF_ATTACH_CONTEXT; 2232 2233 ctx->nr_events--; 2234 if (event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT) 2235 ctx->nr_user--; 2236 if (event->attr.inherit_stat) 2237 ctx->nr_stat--; 2238 if (has_inherit_and_sample_read(&event->attr)) 2239 local_dec(&ctx->nr_no_switch_fast); 2240 2241 list_del_rcu(&event->event_entry); 2242 2243 if (event->group_leader == event) 2244 del_event_from_groups(event, ctx); 2245 2246 ctx->generation++; 2247 event->pmu_ctx->nr_events--; 2248 } 2249 2250 static int 2251 perf_aux_output_match(struct perf_event *event, struct perf_event *aux_event) 2252 { 2253 if (!has_aux(aux_event)) 2254 return 0; 2255 2256 if (!event->pmu->aux_output_match) 2257 return 0; 2258 2259 return event->pmu->aux_output_match(aux_event); 2260 } 2261 2262 static void put_event(struct perf_event *event); 2263 static void __event_disable(struct perf_event *event, 2264 struct perf_event_context *ctx, 2265 enum perf_event_state state); 2266 2267 static void perf_put_aux_event(struct perf_event *event) 2268 { 2269 struct perf_event_context *ctx = event->ctx; 2270 struct perf_event *iter; 2271 2272 /* 2273 * If event uses aux_event tear down the link 2274 */ 2275 if (event->aux_event) { 2276 iter = event->aux_event; 2277 event->aux_event = NULL; 2278 put_event(iter); 2279 return; 2280 } 2281 2282 /* 2283 * If the event is an aux_event, tear down all links to 2284 * it from other events. 2285 */ 2286 for_each_sibling_event(iter, event) { 2287 if (iter->aux_event != event) 2288 continue; 2289 2290 iter->aux_event = NULL; 2291 put_event(event); 2292 2293 /* 2294 * If it's ACTIVE, schedule it out and put it into ERROR 2295 * state so that we don't try to schedule it again. Note 2296 * that perf_event_enable() will clear the ERROR status. 2297 */ 2298 __event_disable(iter, ctx, PERF_EVENT_STATE_ERROR); 2299 } 2300 } 2301 2302 static bool perf_need_aux_event(struct perf_event *event) 2303 { 2304 return event->attr.aux_output || has_aux_action(event); 2305 } 2306 2307 static int perf_get_aux_event(struct perf_event *event, 2308 struct perf_event *group_leader) 2309 { 2310 /* 2311 * Our group leader must be an aux event if we want to be 2312 * an aux_output. This way, the aux event will precede its 2313 * aux_output events in the group, and therefore will always 2314 * schedule first. 2315 */ 2316 if (!group_leader) 2317 return 0; 2318 2319 /* 2320 * aux_output and aux_sample_size are mutually exclusive. 2321 */ 2322 if (event->attr.aux_output && event->attr.aux_sample_size) 2323 return 0; 2324 2325 if (event->attr.aux_output && 2326 !perf_aux_output_match(event, group_leader)) 2327 return 0; 2328 2329 if ((event->attr.aux_pause || event->attr.aux_resume) && 2330 !(group_leader->pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE)) 2331 return 0; 2332 2333 if (event->attr.aux_sample_size && !group_leader->pmu->snapshot_aux) 2334 return 0; 2335 2336 if (!atomic_long_inc_not_zero(&group_leader->refcount)) 2337 return 0; 2338 2339 /* 2340 * Link aux_outputs to their aux event; this is undone in 2341 * perf_group_detach() by perf_put_aux_event(). When the 2342 * group in torn down, the aux_output events loose their 2343 * link to the aux_event and can't schedule any more. 2344 */ 2345 event->aux_event = group_leader; 2346 2347 return 1; 2348 } 2349 2350 static inline struct list_head *get_event_list(struct perf_event *event) 2351 { 2352 return event->attr.pinned ? &event->pmu_ctx->pinned_active : 2353 &event->pmu_ctx->flexible_active; 2354 } 2355 2356 /* @sibling must already be unlinked from its old leader's sibling_list. */ 2357 static void perf_promote_sibling_to_leader(struct perf_event *sibling, 2358 struct perf_event_context *ctx, 2359 int group_caps) 2360 { 2361 /* 2362 * Events that have PERF_EV_CAP_SIBLING require being part of 2363 * a group and cannot exist on their own, schedule them out 2364 * and move them into the ERROR state. Also see 2365 * _perf_event_enable(), it will not be able to recover this 2366 * ERROR state. 2367 */ 2368 if (sibling->event_caps & PERF_EV_CAP_SIBLING) 2369 __event_disable(sibling, ctx, PERF_EVENT_STATE_ERROR); 2370 2371 sibling->group_leader = sibling; 2372 sibling->group_caps = group_caps; 2373 2374 if (sibling->attach_state & PERF_ATTACH_CONTEXT) { 2375 add_event_to_groups(sibling, ctx); 2376 2377 if (sibling->state == PERF_EVENT_STATE_ACTIVE) 2378 list_add_tail(&sibling->active_list, get_event_list(sibling)); 2379 } 2380 2381 perf_event__header_size(sibling); 2382 } 2383 2384 static void perf_group_detach(struct perf_event *event) 2385 { 2386 struct perf_event *leader = event->group_leader; 2387 struct perf_event *sibling, *tmp; 2388 struct perf_event_context *ctx = event->ctx; 2389 2390 lockdep_assert_held(&ctx->lock); 2391 2392 /* 2393 * We can have double detach due to exit/hot-unplug + close. 2394 */ 2395 if (!(event->attach_state & PERF_ATTACH_GROUP)) 2396 return; 2397 2398 event->attach_state &= ~PERF_ATTACH_GROUP; 2399 2400 perf_put_aux_event(event); 2401 2402 /* 2403 * If this is a sibling, remove it from its group. 2404 */ 2405 if (leader != event) { 2406 list_del_init(&event->sibling_list); 2407 leader->nr_siblings--; 2408 leader->group_generation++; 2409 perf_promote_sibling_to_leader(event, ctx, event->event_caps); 2410 goto out; 2411 } 2412 2413 /* 2414 * If this was a group event with sibling events then 2415 * upgrade the siblings to singleton events by adding them 2416 * to whatever list we are on. 2417 */ 2418 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, sibling_list) { 2419 list_del_init(&sibling->sibling_list); 2420 2421 /* Inherit group flags from the previous leader */ 2422 perf_promote_sibling_to_leader(sibling, ctx, event->group_caps); 2423 2424 WARN_ON_ONCE(sibling->ctx != event->ctx); 2425 } 2426 event->nr_siblings = 0; 2427 2428 out: 2429 for_each_sibling_event(tmp, leader) 2430 perf_event__header_size(tmp); 2431 2432 perf_event__header_size(leader); 2433 } 2434 2435 static void perf_child_detach(struct perf_event *event) 2436 { 2437 struct perf_event *parent_event = event->parent; 2438 2439 if (!(event->attach_state & PERF_ATTACH_CHILD)) 2440 return; 2441 2442 event->attach_state &= ~PERF_ATTACH_CHILD; 2443 2444 if (WARN_ON_ONCE(!parent_event)) 2445 return; 2446 2447 /* 2448 * Can't check this from an IPI, the holder is likey another CPU. 2449 * 2450 lockdep_assert_held(&parent_event->child_mutex); 2451 */ 2452 2453 list_del_init(&event->child_list); 2454 } 2455 2456 static bool is_orphaned_event(struct perf_event *event) 2457 { 2458 return event->state == PERF_EVENT_STATE_DEAD; 2459 } 2460 2461 static inline int 2462 event_filter_match(struct perf_event *event) 2463 { 2464 return (event->cpu == -1 || event->cpu == smp_processor_id()) && 2465 perf_cgroup_match(event); 2466 } 2467 2468 static inline bool is_event_in_freq_mode(struct perf_event *event) 2469 { 2470 return event->attr.freq && event->attr.sample_freq; 2471 } 2472 2473 static void 2474 event_sched_out(struct perf_event *event, struct perf_event_context *ctx) 2475 { 2476 struct perf_event_pmu_context *epc = event->pmu_ctx; 2477 struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu); 2478 enum perf_event_state state = PERF_EVENT_STATE_INACTIVE; 2479 2480 // XXX cpc serialization, probably per-cpu IRQ disabled 2481 2482 WARN_ON_ONCE(event->ctx != ctx); 2483 lockdep_assert_held(&ctx->lock); 2484 2485 if (event->state != PERF_EVENT_STATE_ACTIVE) 2486 return; 2487 2488 /* 2489 * Asymmetry; we only schedule events _IN_ through ctx_sched_in(), but 2490 * we can schedule events _OUT_ individually through things like 2491 * __perf_remove_from_context(). 2492 */ 2493 list_del_init(&event->active_list); 2494 2495 perf_pmu_disable(event->pmu); 2496 2497 event->pmu->del(event, 0); 2498 event->oncpu = -1; 2499 2500 if (event->pending_disable) { 2501 event->pending_disable = 0; 2502 perf_cgroup_event_disable(event, ctx); 2503 state = PERF_EVENT_STATE_OFF; 2504 } 2505 2506 perf_event_set_state(event, state); 2507 2508 if (!is_software_event(event)) 2509 cpc->active_oncpu--; 2510 if (is_event_in_freq_mode(event)) { 2511 ctx->nr_freq--; 2512 epc->nr_freq--; 2513 } 2514 if (event->attr.exclusive || !cpc->active_oncpu) 2515 cpc->exclusive = 0; 2516 2517 perf_pmu_enable(event->pmu); 2518 } 2519 2520 static void 2521 group_sched_out(struct perf_event *group_event, struct perf_event_context *ctx) 2522 { 2523 struct perf_event *event; 2524 2525 if (group_event->state != PERF_EVENT_STATE_ACTIVE) 2526 return; 2527 2528 perf_assert_pmu_disabled(group_event->pmu_ctx->pmu); 2529 2530 event_sched_out(group_event, ctx); 2531 2532 /* 2533 * Schedule out siblings (if any): 2534 */ 2535 for_each_sibling_event(event, group_event) 2536 event_sched_out(event, ctx); 2537 } 2538 2539 static inline void 2540 __ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx, 2541 bool final, enum event_type_t event_type) 2542 { 2543 if (ctx->is_active & EVENT_TIME) { 2544 if (ctx->is_active & EVENT_FROZEN) 2545 return; 2546 2547 update_context_time(ctx); 2548 /* vPMU should not stop time */ 2549 update_cgrp_time_from_cpuctx(cpuctx, !(event_type & EVENT_GUEST) && final); 2550 } 2551 } 2552 2553 static inline void 2554 ctx_time_update(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx) 2555 { 2556 __ctx_time_update(cpuctx, ctx, false, 0); 2557 } 2558 2559 /* 2560 * To be used inside perf_ctx_lock() / perf_ctx_unlock(). Lasts until perf_ctx_unlock(). 2561 */ 2562 static inline void 2563 ctx_time_freeze(struct perf_cpu_context *cpuctx, struct perf_event_context *ctx) 2564 { 2565 ctx_time_update(cpuctx, ctx); 2566 if (ctx->is_active & EVENT_TIME) 2567 ctx->is_active |= EVENT_FROZEN; 2568 } 2569 2570 static inline void 2571 ctx_time_update_event(struct perf_event_context *ctx, struct perf_event *event) 2572 { 2573 if (ctx->is_active & EVENT_TIME) { 2574 if (ctx->is_active & EVENT_FROZEN) 2575 return; 2576 update_context_time(ctx); 2577 update_cgrp_time_from_event(event); 2578 } 2579 } 2580 2581 #define DETACH_GROUP 0x01UL 2582 #define DETACH_CHILD 0x02UL 2583 #define DETACH_EXIT 0x04UL 2584 #define DETACH_REVOKE 0x08UL 2585 #define DETACH_DEAD 0x10UL 2586 2587 /* 2588 * Cross CPU call to remove a performance event 2589 * 2590 * We disable the event on the hardware level first. After that we 2591 * remove it from the context list. 2592 */ 2593 static void 2594 __perf_remove_from_context(struct perf_event *event, 2595 struct perf_cpu_context *cpuctx, 2596 struct perf_event_context *ctx, 2597 void *info) 2598 { 2599 struct perf_event_pmu_context *pmu_ctx = event->pmu_ctx; 2600 enum perf_event_state state = PERF_EVENT_STATE_OFF; 2601 unsigned long flags = (unsigned long)info; 2602 2603 perf_pmu_disable(pmu_ctx->pmu); 2604 ctx_time_update(cpuctx, ctx); 2605 2606 /* 2607 * Ensure event_sched_out() switches to OFF, at the very least 2608 * this avoids raising perf_pending_task() at this time. 2609 */ 2610 if (flags & DETACH_EXIT) 2611 state = PERF_EVENT_STATE_EXIT; 2612 if (flags & DETACH_REVOKE) 2613 state = PERF_EVENT_STATE_REVOKED; 2614 if (flags & DETACH_DEAD) 2615 state = PERF_EVENT_STATE_DEAD; 2616 2617 /* 2618 * When disabling a group leader, the whole group becomes ineligible 2619 * to run, so schedule out the full group. 2620 */ 2621 if (!(flags & DETACH_GROUP) && event == event->group_leader) 2622 group_sched_out(event, ctx); 2623 2624 __event_disable(event, ctx, state); 2625 2626 if (flags & DETACH_GROUP) 2627 perf_group_detach(event); 2628 if (flags & DETACH_CHILD) 2629 perf_child_detach(event); 2630 list_del_event(event, ctx); 2631 2632 if (!pmu_ctx->nr_events) { 2633 pmu_ctx->rotate_necessary = 0; 2634 2635 if (ctx->task && ctx->is_active) { 2636 struct perf_cpu_pmu_context *cpc = this_cpc(pmu_ctx->pmu); 2637 2638 WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx); 2639 cpc->task_epc = NULL; 2640 } 2641 } 2642 2643 if (!ctx->nr_events && ctx->is_active) { 2644 if (ctx == &cpuctx->ctx) 2645 update_cgrp_time_from_cpuctx(cpuctx, true); 2646 2647 ctx->is_active = 0; 2648 if (ctx->task) { 2649 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 2650 cpuctx->task_ctx = NULL; 2651 } 2652 } 2653 perf_pmu_enable(pmu_ctx->pmu); 2654 } 2655 2656 /* 2657 * Remove the event from a task's (or a CPU's) list of events. 2658 * 2659 * If event->ctx is a cloned context, callers must make sure that 2660 * every task struct that event->ctx->task could possibly point to 2661 * remains valid. This is OK when called from perf_release since 2662 * that only calls us on the top-level context, which can't be a clone. 2663 * When called from perf_event_exit_task, it's OK because the 2664 * context has been detached from its task. 2665 */ 2666 static void perf_remove_from_context(struct perf_event *event, unsigned long flags) 2667 { 2668 struct perf_event_context *ctx = event->ctx; 2669 2670 lockdep_assert_held(&ctx->mutex); 2671 2672 /* 2673 * Because of perf_event_exit_task(), perf_remove_from_context() ought 2674 * to work in the face of TASK_TOMBSTONE, unlike every other 2675 * event_function_call() user. 2676 */ 2677 raw_spin_lock_irq(&ctx->lock); 2678 if (!ctx->is_active) { 2679 __perf_remove_from_context(event, this_cpu_ptr(&perf_cpu_context), 2680 ctx, (void *)flags); 2681 raw_spin_unlock_irq(&ctx->lock); 2682 return; 2683 } 2684 raw_spin_unlock_irq(&ctx->lock); 2685 2686 event_function_call(event, __perf_remove_from_context, (void *)flags); 2687 } 2688 2689 static void __event_disable(struct perf_event *event, 2690 struct perf_event_context *ctx, 2691 enum perf_event_state state) 2692 { 2693 event_sched_out(event, ctx); 2694 if (event->state > PERF_EVENT_STATE_OFF) 2695 perf_cgroup_event_disable(event, ctx); 2696 perf_event_set_state(event, min(event->state, state)); 2697 } 2698 2699 /* 2700 * Cross CPU call to disable a performance event 2701 */ 2702 static void __perf_event_disable(struct perf_event *event, 2703 struct perf_cpu_context *cpuctx, 2704 struct perf_event_context *ctx, 2705 void *info) 2706 { 2707 if (event->state < PERF_EVENT_STATE_INACTIVE) 2708 return; 2709 2710 perf_pmu_disable(event->pmu_ctx->pmu); 2711 ctx_time_update_event(ctx, event); 2712 2713 /* 2714 * When disabling a group leader, the whole group becomes ineligible 2715 * to run, so schedule out the full group. 2716 */ 2717 if (event == event->group_leader) 2718 group_sched_out(event, ctx); 2719 2720 /* 2721 * But only mark the leader OFF; the siblings will remain 2722 * INACTIVE. 2723 */ 2724 __event_disable(event, ctx, PERF_EVENT_STATE_OFF); 2725 2726 perf_pmu_enable(event->pmu_ctx->pmu); 2727 } 2728 2729 /* 2730 * Disable an event. 2731 * 2732 * If event->ctx is a cloned context, callers must make sure that 2733 * every task struct that event->ctx->task could possibly point to 2734 * remains valid. This condition is satisfied when called through 2735 * perf_event_for_each_child or perf_event_for_each because they 2736 * hold the top-level event's child_mutex, so any descendant that 2737 * goes to exit will block in perf_event_exit_event(). 2738 * 2739 * When called from perf_pending_disable it's OK because event->ctx 2740 * is the current context on this CPU and preemption is disabled, 2741 * hence we can't get into perf_event_task_sched_out for this context. 2742 */ 2743 static void _perf_event_disable(struct perf_event *event) 2744 { 2745 struct perf_event_context *ctx = event->ctx; 2746 2747 raw_spin_lock_irq(&ctx->lock); 2748 if (event->state <= PERF_EVENT_STATE_OFF) { 2749 raw_spin_unlock_irq(&ctx->lock); 2750 return; 2751 } 2752 raw_spin_unlock_irq(&ctx->lock); 2753 2754 event_function_call(event, __perf_event_disable, NULL); 2755 } 2756 2757 void perf_event_disable_local(struct perf_event *event) 2758 { 2759 event_function_local(event, __perf_event_disable, NULL); 2760 } 2761 2762 /* 2763 * Strictly speaking kernel users cannot create groups and therefore this 2764 * interface does not need the perf_event_ctx_lock() magic. 2765 */ 2766 void perf_event_disable(struct perf_event *event) 2767 { 2768 struct perf_event_context *ctx; 2769 2770 ctx = perf_event_ctx_lock(event); 2771 _perf_event_disable(event); 2772 perf_event_ctx_unlock(event, ctx); 2773 } 2774 EXPORT_SYMBOL_GPL(perf_event_disable); 2775 2776 void perf_event_disable_inatomic(struct perf_event *event) 2777 { 2778 event->pending_disable = 1; 2779 irq_work_queue(&event->pending_disable_irq); 2780 } 2781 2782 #define MAX_INTERRUPTS (~0ULL) 2783 2784 static void perf_log_throttle(struct perf_event *event, int enable); 2785 static void perf_log_itrace_start(struct perf_event *event); 2786 2787 static void perf_event_unthrottle(struct perf_event *event, bool start) 2788 { 2789 if (event->state != PERF_EVENT_STATE_ACTIVE) 2790 return; 2791 2792 event->hw.interrupts = 0; 2793 if (start) 2794 event->pmu->start(event, 0); 2795 if (event == event->group_leader) 2796 perf_log_throttle(event, 1); 2797 } 2798 2799 static void perf_event_throttle(struct perf_event *event) 2800 { 2801 if (event->state != PERF_EVENT_STATE_ACTIVE) 2802 return; 2803 2804 event->hw.interrupts = MAX_INTERRUPTS; 2805 event->pmu->stop(event, 0); 2806 if (event == event->group_leader) 2807 perf_log_throttle(event, 0); 2808 } 2809 2810 static void perf_event_unthrottle_group(struct perf_event *event, bool skip_start_event) 2811 { 2812 struct perf_event *sibling, *leader = event->group_leader; 2813 2814 perf_event_unthrottle(leader, skip_start_event ? leader != event : true); 2815 for_each_sibling_event(sibling, leader) 2816 perf_event_unthrottle(sibling, skip_start_event ? sibling != event : true); 2817 } 2818 2819 static void perf_event_throttle_group(struct perf_event *event) 2820 { 2821 struct perf_event *sibling, *leader = event->group_leader; 2822 2823 perf_event_throttle(leader); 2824 for_each_sibling_event(sibling, leader) 2825 perf_event_throttle(sibling); 2826 } 2827 2828 static int 2829 event_sched_in(struct perf_event *event, struct perf_event_context *ctx) 2830 { 2831 struct perf_event_pmu_context *epc = event->pmu_ctx; 2832 struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu); 2833 int ret = 0; 2834 2835 WARN_ON_ONCE(event->ctx != ctx); 2836 2837 lockdep_assert_held(&ctx->lock); 2838 2839 if (event->state <= PERF_EVENT_STATE_OFF) 2840 return 0; 2841 2842 WRITE_ONCE(event->oncpu, smp_processor_id()); 2843 /* 2844 * Order event::oncpu write to happen before the ACTIVE state is 2845 * visible. This allows perf_event_{stop,read}() to observe the correct 2846 * ->oncpu if it sees ACTIVE. 2847 */ 2848 smp_wmb(); 2849 perf_event_set_state(event, PERF_EVENT_STATE_ACTIVE); 2850 2851 /* 2852 * Unthrottle events, since we scheduled we might have missed several 2853 * ticks already, also for a heavily scheduling task there is little 2854 * guarantee it'll get a tick in a timely manner. 2855 */ 2856 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) 2857 perf_event_unthrottle(event, false); 2858 2859 perf_pmu_disable(event->pmu); 2860 2861 perf_log_itrace_start(event); 2862 2863 if (event->pmu->add(event, PERF_EF_START)) { 2864 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); 2865 event->oncpu = -1; 2866 ret = -EAGAIN; 2867 goto out; 2868 } 2869 2870 if (!is_software_event(event)) 2871 cpc->active_oncpu++; 2872 if (is_event_in_freq_mode(event)) { 2873 ctx->nr_freq++; 2874 epc->nr_freq++; 2875 } 2876 if (event->attr.exclusive) 2877 cpc->exclusive = 1; 2878 2879 out: 2880 perf_pmu_enable(event->pmu); 2881 2882 return ret; 2883 } 2884 2885 static int 2886 group_sched_in(struct perf_event *group_event, struct perf_event_context *ctx) 2887 { 2888 struct perf_event *event, *partial_group = NULL; 2889 struct pmu *pmu = group_event->pmu_ctx->pmu; 2890 2891 if (group_event->state == PERF_EVENT_STATE_OFF) 2892 return 0; 2893 2894 pmu->start_txn(pmu, PERF_PMU_TXN_ADD); 2895 2896 if (event_sched_in(group_event, ctx)) 2897 goto error; 2898 2899 /* 2900 * Schedule in siblings as one group (if any): 2901 */ 2902 for_each_sibling_event(event, group_event) { 2903 if (event_sched_in(event, ctx)) { 2904 partial_group = event; 2905 goto group_error; 2906 } 2907 } 2908 2909 if (!pmu->commit_txn(pmu)) 2910 return 0; 2911 2912 group_error: 2913 /* 2914 * Groups can be scheduled in as one unit only, so undo any 2915 * partial group before returning: 2916 * The events up to the failed event are scheduled out normally. 2917 */ 2918 for_each_sibling_event(event, group_event) { 2919 if (event == partial_group) 2920 break; 2921 2922 event_sched_out(event, ctx); 2923 } 2924 event_sched_out(group_event, ctx); 2925 2926 error: 2927 pmu->cancel_txn(pmu); 2928 return -EAGAIN; 2929 } 2930 2931 /* 2932 * Work out whether we can put this event group on the CPU now. 2933 */ 2934 static int group_can_go_on(struct perf_event *event, int can_add_hw) 2935 { 2936 struct perf_event_pmu_context *epc = event->pmu_ctx; 2937 struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu); 2938 2939 /* 2940 * Groups consisting entirely of software events can always go on. 2941 */ 2942 if (event->group_caps & PERF_EV_CAP_SOFTWARE) 2943 return 1; 2944 /* 2945 * If an exclusive group is already on, no other hardware 2946 * events can go on. 2947 */ 2948 if (cpc->exclusive) 2949 return 0; 2950 /* 2951 * If this group is exclusive and there are already 2952 * events on the CPU, it can't go on. 2953 */ 2954 if (event->attr.exclusive && !list_empty(get_event_list(event))) 2955 return 0; 2956 /* 2957 * Otherwise, try to add it if all previous groups were able 2958 * to go on. 2959 */ 2960 return can_add_hw; 2961 } 2962 2963 static void add_event_to_ctx(struct perf_event *event, 2964 struct perf_event_context *ctx) 2965 { 2966 list_add_event(event, ctx); 2967 perf_group_attach(event); 2968 } 2969 2970 static void task_ctx_sched_out(struct perf_event_context *ctx, 2971 struct pmu *pmu, 2972 enum event_type_t event_type) 2973 { 2974 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 2975 2976 if (!cpuctx->task_ctx) 2977 return; 2978 2979 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) 2980 return; 2981 2982 ctx_sched_out(ctx, pmu, event_type); 2983 } 2984 2985 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, 2986 struct perf_event_context *ctx, 2987 struct pmu *pmu, 2988 enum event_type_t event_type) 2989 { 2990 ctx_sched_in(&cpuctx->ctx, pmu, EVENT_PINNED | event_type); 2991 if (ctx) 2992 ctx_sched_in(ctx, pmu, EVENT_PINNED | event_type); 2993 ctx_sched_in(&cpuctx->ctx, pmu, EVENT_FLEXIBLE | event_type); 2994 if (ctx) 2995 ctx_sched_in(ctx, pmu, EVENT_FLEXIBLE | event_type); 2996 } 2997 2998 /* 2999 * We want to maintain the following priority of scheduling: 3000 * - CPU pinned (EVENT_CPU | EVENT_PINNED) 3001 * - task pinned (EVENT_PINNED) 3002 * - CPU flexible (EVENT_CPU | EVENT_FLEXIBLE) 3003 * - task flexible (EVENT_FLEXIBLE). 3004 * 3005 * In order to avoid unscheduling and scheduling back in everything every 3006 * time an event is added, only do it for the groups of equal priority and 3007 * below. 3008 * 3009 * This can be called after a batch operation on task events, in which case 3010 * event_type is a bit mask of the types of events involved. For CPU events, 3011 * event_type is only either EVENT_PINNED or EVENT_FLEXIBLE. 3012 */ 3013 static void ctx_resched(struct perf_cpu_context *cpuctx, 3014 struct perf_event_context *task_ctx, 3015 struct pmu *pmu, enum event_type_t event_type) 3016 { 3017 bool cpu_event = !!(event_type & EVENT_CPU); 3018 struct perf_event_pmu_context *epc; 3019 3020 /* 3021 * If pinned groups are involved, flexible groups also need to be 3022 * scheduled out. 3023 */ 3024 if (event_type & EVENT_PINNED) 3025 event_type |= EVENT_FLEXIBLE; 3026 3027 event_type &= EVENT_ALL; 3028 3029 for_each_epc(epc, &cpuctx->ctx, pmu, 0) 3030 perf_pmu_disable(epc->pmu); 3031 3032 if (task_ctx) { 3033 for_each_epc(epc, task_ctx, pmu, 0) 3034 perf_pmu_disable(epc->pmu); 3035 3036 task_ctx_sched_out(task_ctx, pmu, event_type); 3037 } 3038 3039 /* 3040 * Decide which cpu ctx groups to schedule out based on the types 3041 * of events that caused rescheduling: 3042 * - EVENT_CPU: schedule out corresponding groups; 3043 * - EVENT_PINNED task events: schedule out EVENT_FLEXIBLE groups; 3044 * - otherwise, do nothing more. 3045 */ 3046 if (cpu_event) 3047 ctx_sched_out(&cpuctx->ctx, pmu, event_type); 3048 else if (event_type & EVENT_PINNED) 3049 ctx_sched_out(&cpuctx->ctx, pmu, EVENT_FLEXIBLE); 3050 3051 perf_event_sched_in(cpuctx, task_ctx, pmu, 0); 3052 3053 for_each_epc(epc, &cpuctx->ctx, pmu, 0) 3054 perf_pmu_enable(epc->pmu); 3055 3056 if (task_ctx) { 3057 for_each_epc(epc, task_ctx, pmu, 0) 3058 perf_pmu_enable(epc->pmu); 3059 } 3060 } 3061 3062 void perf_pmu_resched(struct pmu *pmu) 3063 { 3064 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 3065 struct perf_event_context *task_ctx = cpuctx->task_ctx; 3066 3067 perf_ctx_lock(cpuctx, task_ctx); 3068 ctx_resched(cpuctx, task_ctx, pmu, EVENT_ALL|EVENT_CPU); 3069 perf_ctx_unlock(cpuctx, task_ctx); 3070 } 3071 3072 /* 3073 * Cross CPU call to install and enable a performance event 3074 * 3075 * Very similar to remote_function() + event_function() but cannot assume that 3076 * things like ctx->is_active and cpuctx->task_ctx are set. 3077 */ 3078 static int __perf_install_in_context(void *info) 3079 { 3080 struct perf_event *event = info; 3081 struct perf_event_context *ctx = event->ctx; 3082 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 3083 struct perf_event_context *task_ctx = cpuctx->task_ctx; 3084 bool reprogram = true; 3085 int ret = 0; 3086 3087 raw_spin_lock(&cpuctx->ctx.lock); 3088 if (ctx->task) { 3089 raw_spin_lock(&ctx->lock); 3090 task_ctx = ctx; 3091 3092 reprogram = (ctx->task == current); 3093 3094 /* 3095 * If the task is running, it must be running on this CPU, 3096 * otherwise we cannot reprogram things. 3097 * 3098 * If its not running, we don't care, ctx->lock will 3099 * serialize against it becoming runnable. 3100 */ 3101 if (task_curr(ctx->task) && !reprogram) { 3102 ret = -ESRCH; 3103 goto unlock; 3104 } 3105 3106 WARN_ON_ONCE(reprogram && cpuctx->task_ctx && cpuctx->task_ctx != ctx); 3107 } else if (task_ctx) { 3108 raw_spin_lock(&task_ctx->lock); 3109 } 3110 3111 #ifdef CONFIG_CGROUP_PERF 3112 if (event->state > PERF_EVENT_STATE_OFF && is_cgroup_event(event)) { 3113 /* 3114 * If the current cgroup doesn't match the event's 3115 * cgroup, we should not try to schedule it. 3116 */ 3117 struct perf_cgroup *cgrp = perf_cgroup_from_task(current, ctx); 3118 reprogram = cgroup_is_descendant(cgrp->css.cgroup, 3119 event->cgrp->css.cgroup); 3120 } 3121 #endif 3122 3123 if (reprogram) { 3124 ctx_time_freeze(cpuctx, ctx); 3125 add_event_to_ctx(event, ctx); 3126 ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, 3127 get_event_type(event)); 3128 } else { 3129 add_event_to_ctx(event, ctx); 3130 } 3131 3132 unlock: 3133 perf_ctx_unlock(cpuctx, task_ctx); 3134 3135 return ret; 3136 } 3137 3138 static bool exclusive_event_installable(struct perf_event *event, 3139 struct perf_event_context *ctx); 3140 3141 /* 3142 * Attach a performance event to a context. 3143 * 3144 * Very similar to event_function_call, see comment there. 3145 */ 3146 static void 3147 perf_install_in_context(struct perf_event_context *ctx, 3148 struct perf_event *event, 3149 int cpu) 3150 { 3151 struct task_struct *task = READ_ONCE(ctx->task); 3152 3153 lockdep_assert_held(&ctx->mutex); 3154 3155 WARN_ON_ONCE(!exclusive_event_installable(event, ctx)); 3156 3157 if (event->cpu != -1) 3158 WARN_ON_ONCE(event->cpu != cpu); 3159 3160 /* 3161 * Ensures that if we can observe event->ctx, both the event and ctx 3162 * will be 'complete'. See perf_iterate_sb_cpu(). 3163 */ 3164 smp_store_release(&event->ctx, ctx); 3165 3166 /* 3167 * perf_event_attr::disabled events will not run and can be initialized 3168 * without IPI. Except when this is the first event for the context, in 3169 * that case we need the magic of the IPI to set ctx->is_active. 3170 * 3171 * The IOC_ENABLE that is sure to follow the creation of a disabled 3172 * event will issue the IPI and reprogram the hardware. 3173 */ 3174 if (__perf_effective_state(event) == PERF_EVENT_STATE_OFF && 3175 ctx->nr_events && !is_cgroup_event(event)) { 3176 raw_spin_lock_irq(&ctx->lock); 3177 if (ctx->task == TASK_TOMBSTONE) { 3178 raw_spin_unlock_irq(&ctx->lock); 3179 return; 3180 } 3181 add_event_to_ctx(event, ctx); 3182 raw_spin_unlock_irq(&ctx->lock); 3183 return; 3184 } 3185 3186 if (!task) { 3187 cpu_function_call(cpu, __perf_install_in_context, event); 3188 return; 3189 } 3190 3191 /* 3192 * Should not happen, we validate the ctx is still alive before calling. 3193 */ 3194 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) 3195 return; 3196 3197 /* 3198 * Installing events is tricky because we cannot rely on ctx->is_active 3199 * to be set in case this is the nr_events 0 -> 1 transition. 3200 * 3201 * Instead we use task_curr(), which tells us if the task is running. 3202 * However, since we use task_curr() outside of rq::lock, we can race 3203 * against the actual state. This means the result can be wrong. 3204 * 3205 * If we get a false positive, we retry, this is harmless. 3206 * 3207 * If we get a false negative, things are complicated. If we are after 3208 * perf_event_context_sched_in() ctx::lock will serialize us, and the 3209 * value must be correct. If we're before, it doesn't matter since 3210 * perf_event_context_sched_in() will program the counter. 3211 * 3212 * However, this hinges on the remote context switch having observed 3213 * our task->perf_event_ctxp[] store, such that it will in fact take 3214 * ctx::lock in perf_event_context_sched_in(). 3215 * 3216 * We do this by task_function_call(), if the IPI fails to hit the task 3217 * we know any future context switch of task must see the 3218 * perf_event_ctpx[] store. 3219 */ 3220 3221 /* 3222 * This smp_mb() orders the task->perf_event_ctxp[] store with the 3223 * task_cpu() load, such that if the IPI then does not find the task 3224 * running, a future context switch of that task must observe the 3225 * store. 3226 */ 3227 smp_mb(); 3228 again: 3229 if (!task_function_call(task, __perf_install_in_context, event)) 3230 return; 3231 3232 raw_spin_lock_irq(&ctx->lock); 3233 task = ctx->task; 3234 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) { 3235 /* 3236 * Cannot happen because we already checked above (which also 3237 * cannot happen), and we hold ctx->mutex, which serializes us 3238 * against perf_event_exit_task_context(). 3239 */ 3240 raw_spin_unlock_irq(&ctx->lock); 3241 return; 3242 } 3243 /* 3244 * If the task is not running, ctx->lock will avoid it becoming so, 3245 * thus we can safely install the event. 3246 */ 3247 if (task_curr(task)) { 3248 raw_spin_unlock_irq(&ctx->lock); 3249 goto again; 3250 } 3251 add_event_to_ctx(event, ctx); 3252 raw_spin_unlock_irq(&ctx->lock); 3253 } 3254 3255 /* 3256 * Cross CPU call to enable a performance event 3257 */ 3258 static void __perf_event_enable(struct perf_event *event, 3259 struct perf_cpu_context *cpuctx, 3260 struct perf_event_context *ctx, 3261 void *info) 3262 { 3263 struct perf_event *leader = event->group_leader; 3264 struct perf_event_context *task_ctx; 3265 3266 if (event->state >= PERF_EVENT_STATE_INACTIVE || 3267 event->state <= PERF_EVENT_STATE_ERROR) 3268 return; 3269 3270 ctx_time_freeze(cpuctx, ctx); 3271 3272 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); 3273 perf_cgroup_event_enable(event, ctx); 3274 3275 if (!ctx->is_active) 3276 return; 3277 3278 if (!event_filter_match(event)) 3279 return; 3280 3281 /* 3282 * If the event is in a group and isn't the group leader, 3283 * then don't put it on unless the group is on. 3284 */ 3285 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) 3286 return; 3287 3288 task_ctx = cpuctx->task_ctx; 3289 if (ctx->task) 3290 WARN_ON_ONCE(task_ctx != ctx); 3291 3292 ctx_resched(cpuctx, task_ctx, event->pmu_ctx->pmu, get_event_type(event)); 3293 } 3294 3295 /* 3296 * Enable an event. 3297 * 3298 * If event->ctx is a cloned context, callers must make sure that 3299 * every task struct that event->ctx->task could possibly point to 3300 * remains valid. This condition is satisfied when called through 3301 * perf_event_for_each_child or perf_event_for_each as described 3302 * for perf_event_disable. 3303 */ 3304 static void _perf_event_enable(struct perf_event *event) 3305 { 3306 struct perf_event_context *ctx = event->ctx; 3307 3308 raw_spin_lock_irq(&ctx->lock); 3309 if (event->state >= PERF_EVENT_STATE_INACTIVE || 3310 event->state < PERF_EVENT_STATE_ERROR) { 3311 out: 3312 raw_spin_unlock_irq(&ctx->lock); 3313 return; 3314 } 3315 3316 /* 3317 * If the event is in error state, clear that first. 3318 * 3319 * That way, if we see the event in error state below, we know that it 3320 * has gone back into error state, as distinct from the task having 3321 * been scheduled away before the cross-call arrived. 3322 */ 3323 if (event->state == PERF_EVENT_STATE_ERROR) { 3324 /* 3325 * Detached SIBLING events cannot leave ERROR state. 3326 */ 3327 if (event->event_caps & PERF_EV_CAP_SIBLING && 3328 event->group_leader == event) 3329 goto out; 3330 3331 event->state = PERF_EVENT_STATE_OFF; 3332 } 3333 raw_spin_unlock_irq(&ctx->lock); 3334 3335 event_function_call(event, __perf_event_enable, NULL); 3336 } 3337 3338 /* 3339 * See perf_event_disable(); 3340 */ 3341 void perf_event_enable(struct perf_event *event) 3342 { 3343 struct perf_event_context *ctx; 3344 3345 ctx = perf_event_ctx_lock(event); 3346 _perf_event_enable(event); 3347 perf_event_ctx_unlock(event, ctx); 3348 } 3349 EXPORT_SYMBOL_GPL(perf_event_enable); 3350 3351 struct stop_event_data { 3352 struct perf_event *event; 3353 unsigned int restart; 3354 }; 3355 3356 static int __perf_event_stop(void *info) 3357 { 3358 struct stop_event_data *sd = info; 3359 struct perf_event *event = sd->event; 3360 3361 /* if it's already INACTIVE, do nothing */ 3362 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 3363 return 0; 3364 3365 /* matches smp_wmb() in event_sched_in() */ 3366 smp_rmb(); 3367 3368 /* 3369 * There is a window with interrupts enabled before we get here, 3370 * so we need to check again lest we try to stop another CPU's event. 3371 */ 3372 if (READ_ONCE(event->oncpu) != smp_processor_id()) 3373 return -EAGAIN; 3374 3375 event->pmu->stop(event, PERF_EF_UPDATE); 3376 3377 /* 3378 * May race with the actual stop (through perf_pmu_output_stop()), 3379 * but it is only used for events with AUX ring buffer, and such 3380 * events will refuse to restart because of rb::aux_mmap_count==0, 3381 * see comments in perf_aux_output_begin(). 3382 * 3383 * Since this is happening on an event-local CPU, no trace is lost 3384 * while restarting. 3385 */ 3386 if (sd->restart) 3387 event->pmu->start(event, 0); 3388 3389 return 0; 3390 } 3391 3392 static int perf_event_stop(struct perf_event *event, int restart) 3393 { 3394 struct stop_event_data sd = { 3395 .event = event, 3396 .restart = restart, 3397 }; 3398 int ret = 0; 3399 3400 do { 3401 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 3402 return 0; 3403 3404 /* matches smp_wmb() in event_sched_in() */ 3405 smp_rmb(); 3406 3407 /* 3408 * We only want to restart ACTIVE events, so if the event goes 3409 * inactive here (event->oncpu==-1), there's nothing more to do; 3410 * fall through with ret==-ENXIO. 3411 */ 3412 ret = cpu_function_call(READ_ONCE(event->oncpu), 3413 __perf_event_stop, &sd); 3414 } while (ret == -EAGAIN); 3415 3416 return ret; 3417 } 3418 3419 /* 3420 * In order to contain the amount of racy and tricky in the address filter 3421 * configuration management, it is a two part process: 3422 * 3423 * (p1) when userspace mappings change as a result of (1) or (2) or (3) below, 3424 * we update the addresses of corresponding vmas in 3425 * event::addr_filter_ranges array and bump the event::addr_filters_gen; 3426 * (p2) when an event is scheduled in (pmu::add), it calls 3427 * perf_event_addr_filters_sync() which calls pmu::addr_filters_sync() 3428 * if the generation has changed since the previous call. 3429 * 3430 * If (p1) happens while the event is active, we restart it to force (p2). 3431 * 3432 * (1) perf_addr_filters_apply(): adjusting filters' offsets based on 3433 * pre-existing mappings, called once when new filters arrive via SET_FILTER 3434 * ioctl; 3435 * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly 3436 * registered mapping, called for every new mmap(), with mm::mmap_lock down 3437 * for reading; 3438 * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process 3439 * of exec. 3440 */ 3441 void perf_event_addr_filters_sync(struct perf_event *event) 3442 { 3443 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 3444 3445 if (!has_addr_filter(event)) 3446 return; 3447 3448 raw_spin_lock(&ifh->lock); 3449 if (event->addr_filters_gen != event->hw.addr_filters_gen) { 3450 event->pmu->addr_filters_sync(event); 3451 event->hw.addr_filters_gen = event->addr_filters_gen; 3452 } 3453 raw_spin_unlock(&ifh->lock); 3454 } 3455 EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync); 3456 3457 static int _perf_event_refresh(struct perf_event *event, int refresh) 3458 { 3459 /* 3460 * not supported on inherited events 3461 */ 3462 if (event->attr.inherit || !is_sampling_event(event)) 3463 return -EINVAL; 3464 3465 atomic_add(refresh, &event->event_limit); 3466 _perf_event_enable(event); 3467 3468 return 0; 3469 } 3470 3471 /* 3472 * See perf_event_disable() 3473 */ 3474 int perf_event_refresh(struct perf_event *event, int refresh) 3475 { 3476 struct perf_event_context *ctx; 3477 int ret; 3478 3479 ctx = perf_event_ctx_lock(event); 3480 ret = _perf_event_refresh(event, refresh); 3481 perf_event_ctx_unlock(event, ctx); 3482 3483 return ret; 3484 } 3485 EXPORT_SYMBOL_GPL(perf_event_refresh); 3486 3487 static int perf_event_modify_breakpoint(struct perf_event *bp, 3488 struct perf_event_attr *attr) 3489 { 3490 int err; 3491 3492 _perf_event_disable(bp); 3493 3494 err = modify_user_hw_breakpoint_check(bp, attr, true); 3495 3496 if (!bp->attr.disabled) 3497 _perf_event_enable(bp); 3498 3499 return err; 3500 } 3501 3502 /* 3503 * Copy event-type-independent attributes that may be modified. 3504 */ 3505 static void perf_event_modify_copy_attr(struct perf_event_attr *to, 3506 const struct perf_event_attr *from) 3507 { 3508 to->sig_data = from->sig_data; 3509 } 3510 3511 static int perf_event_modify_attr(struct perf_event *event, 3512 struct perf_event_attr *attr) 3513 { 3514 int (*func)(struct perf_event *, struct perf_event_attr *); 3515 struct perf_event *child; 3516 int err; 3517 3518 if (event->attr.type != attr->type) 3519 return -EINVAL; 3520 3521 switch (event->attr.type) { 3522 case PERF_TYPE_BREAKPOINT: 3523 func = perf_event_modify_breakpoint; 3524 break; 3525 default: 3526 /* Place holder for future additions. */ 3527 return -EOPNOTSUPP; 3528 } 3529 3530 WARN_ON_ONCE(event->ctx->parent_ctx); 3531 3532 mutex_lock(&event->child_mutex); 3533 /* 3534 * Event-type-independent attributes must be copied before event-type 3535 * modification, which will validate that final attributes match the 3536 * source attributes after all relevant attributes have been copied. 3537 */ 3538 perf_event_modify_copy_attr(&event->attr, attr); 3539 err = func(event, attr); 3540 if (err) 3541 goto out; 3542 list_for_each_entry(child, &event->child_list, child_list) { 3543 perf_event_modify_copy_attr(&child->attr, attr); 3544 err = func(child, attr); 3545 if (err) 3546 goto out; 3547 } 3548 out: 3549 mutex_unlock(&event->child_mutex); 3550 return err; 3551 } 3552 3553 static void __pmu_ctx_sched_out(struct perf_event_pmu_context *pmu_ctx, 3554 enum event_type_t event_type) 3555 { 3556 struct perf_event_context *ctx = pmu_ctx->ctx; 3557 struct perf_event *event, *tmp; 3558 struct pmu *pmu = pmu_ctx->pmu; 3559 3560 if (ctx->task && !(ctx->is_active & EVENT_ALL)) { 3561 struct perf_cpu_pmu_context *cpc = this_cpc(pmu); 3562 3563 WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx); 3564 cpc->task_epc = NULL; 3565 } 3566 3567 if (!(event_type & EVENT_ALL)) 3568 return; 3569 3570 perf_pmu_disable(pmu); 3571 if (event_type & EVENT_PINNED) { 3572 list_for_each_entry_safe(event, tmp, 3573 &pmu_ctx->pinned_active, 3574 active_list) 3575 group_sched_out(event, ctx); 3576 } 3577 3578 if (event_type & EVENT_FLEXIBLE) { 3579 list_for_each_entry_safe(event, tmp, 3580 &pmu_ctx->flexible_active, 3581 active_list) 3582 group_sched_out(event, ctx); 3583 /* 3584 * Since we cleared EVENT_FLEXIBLE, also clear 3585 * rotate_necessary, is will be reset by 3586 * ctx_flexible_sched_in() when needed. 3587 */ 3588 pmu_ctx->rotate_necessary = 0; 3589 } 3590 perf_pmu_enable(pmu); 3591 } 3592 3593 /* 3594 * Be very careful with the @pmu argument since this will change ctx state. 3595 * The @pmu argument works for ctx_resched(), because that is symmetric in 3596 * ctx_sched_out() / ctx_sched_in() usage and the ctx state ends up invariant. 3597 * 3598 * However, if you were to be asymmetrical, you could end up with messed up 3599 * state, eg. ctx->is_active cleared even though most EPCs would still actually 3600 * be active. 3601 */ 3602 static void 3603 ctx_sched_out(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type) 3604 { 3605 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 3606 enum event_type_t active_type = event_type & ~EVENT_FLAGS; 3607 struct perf_event_pmu_context *pmu_ctx; 3608 int is_active = ctx->is_active; 3609 3610 3611 lockdep_assert_held(&ctx->lock); 3612 3613 if (likely(!ctx->nr_events)) { 3614 /* 3615 * See __perf_remove_from_context(). 3616 */ 3617 WARN_ON_ONCE(ctx->is_active); 3618 if (ctx->task) 3619 WARN_ON_ONCE(cpuctx->task_ctx); 3620 return; 3621 } 3622 3623 /* 3624 * Always update time if it was set; not only when it changes. 3625 * Otherwise we can 'forget' to update time for any but the last 3626 * context we sched out. For example: 3627 * 3628 * ctx_sched_out(.event_type = EVENT_FLEXIBLE) 3629 * ctx_sched_out(.event_type = EVENT_PINNED) 3630 * 3631 * would only update time for the pinned events. 3632 */ 3633 __ctx_time_update(cpuctx, ctx, ctx == &cpuctx->ctx, event_type); 3634 3635 /* 3636 * CPU-release for the below ->is_active store, 3637 * see __load_acquire() in perf_event_time_now() 3638 */ 3639 barrier(); 3640 ctx->is_active &= ~active_type; 3641 3642 if (!(ctx->is_active & EVENT_ALL)) { 3643 /* 3644 * For FROZEN, preserve TIME|FROZEN such that perf_event_time_now() 3645 * does not observe a hole. perf_ctx_unlock() will clean up. 3646 */ 3647 if (ctx->is_active & EVENT_FROZEN) 3648 ctx->is_active &= EVENT_TIME_FROZEN; 3649 else 3650 ctx->is_active = 0; 3651 } 3652 3653 if (ctx->task) { 3654 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 3655 if (!(ctx->is_active & EVENT_ALL)) 3656 cpuctx->task_ctx = NULL; 3657 } 3658 3659 if (event_type & EVENT_GUEST) { 3660 /* 3661 * Schedule out all exclude_guest events of PMU 3662 * with PERF_PMU_CAP_MEDIATED_VPMU. 3663 */ 3664 is_active = EVENT_ALL; 3665 __update_context_guest_time(ctx, false); 3666 perf_cgroup_set_timestamp(cpuctx, true); 3667 barrier(); 3668 } else { 3669 is_active ^= ctx->is_active; /* changed bits */ 3670 } 3671 3672 for_each_epc(pmu_ctx, ctx, pmu, event_type) 3673 __pmu_ctx_sched_out(pmu_ctx, is_active); 3674 } 3675 3676 /* 3677 * Test whether two contexts are equivalent, i.e. whether they have both been 3678 * cloned from the same version of the same context. 3679 * 3680 * Equivalence is measured using a generation number in the context that is 3681 * incremented on each modification to it; see unclone_ctx(), list_add_event() 3682 * and list_del_event(). 3683 */ 3684 static int context_equiv(struct perf_event_context *ctx1, 3685 struct perf_event_context *ctx2) 3686 { 3687 lockdep_assert_held(&ctx1->lock); 3688 lockdep_assert_held(&ctx2->lock); 3689 3690 /* Pinning disables the swap optimization */ 3691 if (ctx1->pin_count || ctx2->pin_count) 3692 return 0; 3693 3694 /* If ctx1 is the parent of ctx2 */ 3695 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) 3696 return 1; 3697 3698 /* If ctx2 is the parent of ctx1 */ 3699 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) 3700 return 1; 3701 3702 /* 3703 * If ctx1 and ctx2 have the same parent; we flatten the parent 3704 * hierarchy, see perf_event_init_context(). 3705 */ 3706 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && 3707 ctx1->parent_gen == ctx2->parent_gen) 3708 return 1; 3709 3710 /* Unmatched */ 3711 return 0; 3712 } 3713 3714 static void __perf_event_sync_stat(struct perf_event *event, 3715 struct perf_event *next_event) 3716 { 3717 u64 value; 3718 3719 if (!event->attr.inherit_stat) 3720 return; 3721 3722 /* 3723 * Update the event value, we cannot use perf_event_read() 3724 * because we're in the middle of a context switch and have IRQs 3725 * disabled, which upsets smp_call_function_single(), however 3726 * we know the event must be on the current CPU, therefore we 3727 * don't need to use it. 3728 */ 3729 perf_pmu_read(event); 3730 3731 perf_event_update_time(event); 3732 3733 /* 3734 * In order to keep per-task stats reliable we need to flip the event 3735 * values when we flip the contexts. 3736 */ 3737 value = local64_read(&next_event->count); 3738 value = local64_xchg(&event->count, value); 3739 local64_set(&next_event->count, value); 3740 3741 swap(event->total_time_enabled, next_event->total_time_enabled); 3742 swap(event->total_time_running, next_event->total_time_running); 3743 3744 /* 3745 * Since we swizzled the values, update the user visible data too. 3746 */ 3747 perf_event_update_userpage(event); 3748 perf_event_update_userpage(next_event); 3749 } 3750 3751 static void perf_event_sync_stat(struct perf_event_context *ctx, 3752 struct perf_event_context *next_ctx) 3753 { 3754 struct perf_event *event, *next_event; 3755 3756 if (!ctx->nr_stat) 3757 return; 3758 3759 update_context_time(ctx); 3760 3761 event = list_first_entry(&ctx->event_list, 3762 struct perf_event, event_entry); 3763 3764 next_event = list_first_entry(&next_ctx->event_list, 3765 struct perf_event, event_entry); 3766 3767 while (&event->event_entry != &ctx->event_list && 3768 &next_event->event_entry != &next_ctx->event_list) { 3769 3770 __perf_event_sync_stat(event, next_event); 3771 3772 event = list_next_entry(event, event_entry); 3773 next_event = list_next_entry(next_event, event_entry); 3774 } 3775 } 3776 3777 static void perf_ctx_sched_task_cb(struct perf_event_context *ctx, 3778 struct task_struct *task, bool sched_in) 3779 { 3780 struct perf_event_pmu_context *pmu_ctx; 3781 struct perf_cpu_pmu_context *cpc; 3782 3783 list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { 3784 cpc = this_cpc(pmu_ctx->pmu); 3785 3786 if (cpc->task_epc != pmu_ctx) 3787 continue; 3788 3789 if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task) 3790 pmu_ctx->pmu->sched_task(pmu_ctx, task, sched_in); 3791 } 3792 } 3793 3794 static void 3795 perf_event_context_sched_out(struct task_struct *task, struct task_struct *next) 3796 { 3797 struct perf_event_context *ctx = task->perf_event_ctxp; 3798 struct perf_event_context *next_ctx; 3799 struct perf_event_context *parent, *next_parent; 3800 int do_switch = 1; 3801 3802 if (likely(!ctx)) 3803 return; 3804 3805 rcu_read_lock(); 3806 next_ctx = rcu_dereference(next->perf_event_ctxp); 3807 if (!next_ctx) 3808 goto unlock; 3809 3810 parent = rcu_dereference(ctx->parent_ctx); 3811 next_parent = rcu_dereference(next_ctx->parent_ctx); 3812 3813 /* If neither context have a parent context; they cannot be clones. */ 3814 if (!parent && !next_parent) 3815 goto unlock; 3816 3817 if (next_parent == ctx || next_ctx == parent || next_parent == parent) { 3818 /* 3819 * Looks like the two contexts are clones, so we might be 3820 * able to optimize the context switch. We lock both 3821 * contexts and check that they are clones under the 3822 * lock (including re-checking that neither has been 3823 * uncloned in the meantime). It doesn't matter which 3824 * order we take the locks because no other cpu could 3825 * be trying to lock both of these tasks. 3826 */ 3827 raw_spin_lock(&ctx->lock); 3828 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 3829 if (context_equiv(ctx, next_ctx)) { 3830 3831 perf_ctx_disable(ctx, 0); 3832 3833 /* PMIs are disabled; ctx->nr_no_switch_fast is stable. */ 3834 if (local_read(&ctx->nr_no_switch_fast) || 3835 local_read(&next_ctx->nr_no_switch_fast)) { 3836 /* 3837 * Must not swap out ctx when there's pending 3838 * events that rely on the ctx->task relation. 3839 * 3840 * Likewise, when a context contains inherit + 3841 * SAMPLE_READ events they should be switched 3842 * out using the slow path so that they are 3843 * treated as if they were distinct contexts. 3844 */ 3845 raw_spin_unlock(&next_ctx->lock); 3846 rcu_read_unlock(); 3847 goto inside_switch; 3848 } 3849 3850 WRITE_ONCE(ctx->task, next); 3851 WRITE_ONCE(next_ctx->task, task); 3852 3853 perf_ctx_sched_task_cb(ctx, task, false); 3854 3855 perf_ctx_enable(ctx, 0); 3856 3857 /* 3858 * RCU_INIT_POINTER here is safe because we've not 3859 * modified the ctx and the above modification of 3860 * ctx->task is immaterial since this value is 3861 * always verified under ctx->lock which we're now 3862 * holding. 3863 */ 3864 RCU_INIT_POINTER(task->perf_event_ctxp, next_ctx); 3865 RCU_INIT_POINTER(next->perf_event_ctxp, ctx); 3866 3867 do_switch = 0; 3868 3869 perf_event_sync_stat(ctx, next_ctx); 3870 } 3871 raw_spin_unlock(&next_ctx->lock); 3872 raw_spin_unlock(&ctx->lock); 3873 } 3874 unlock: 3875 rcu_read_unlock(); 3876 3877 if (do_switch) { 3878 raw_spin_lock(&ctx->lock); 3879 perf_ctx_disable(ctx, 0); 3880 3881 inside_switch: 3882 perf_ctx_sched_task_cb(ctx, task, false); 3883 task_ctx_sched_out(ctx, NULL, EVENT_ALL); 3884 3885 perf_ctx_enable(ctx, 0); 3886 raw_spin_unlock(&ctx->lock); 3887 } 3888 } 3889 3890 static DEFINE_PER_CPU(struct list_head, sched_cb_list); 3891 static DEFINE_PER_CPU(int, perf_sched_cb_usages); 3892 3893 void perf_sched_cb_dec(struct pmu *pmu) 3894 { 3895 struct perf_cpu_pmu_context *cpc = this_cpc(pmu); 3896 3897 this_cpu_dec(perf_sched_cb_usages); 3898 barrier(); 3899 3900 if (!--cpc->sched_cb_usage) 3901 list_del(&cpc->sched_cb_entry); 3902 } 3903 3904 3905 void perf_sched_cb_inc(struct pmu *pmu) 3906 { 3907 struct perf_cpu_pmu_context *cpc = this_cpc(pmu); 3908 3909 if (!cpc->sched_cb_usage++) 3910 list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list)); 3911 3912 barrier(); 3913 this_cpu_inc(perf_sched_cb_usages); 3914 } 3915 3916 /* 3917 * This function provides the context switch callback to the lower code 3918 * layer. It is invoked ONLY when the context switch callback is enabled. 3919 * 3920 * This callback is relevant even to per-cpu events; for example multi event 3921 * PEBS requires this to provide PID/TID information. This requires we flush 3922 * all queued PEBS records before we context switch to a new task. 3923 */ 3924 static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc, 3925 struct task_struct *task, bool sched_in) 3926 { 3927 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 3928 struct pmu *pmu; 3929 3930 pmu = cpc->epc.pmu; 3931 3932 /* software PMUs will not have sched_task */ 3933 if (WARN_ON_ONCE(!pmu->sched_task)) 3934 return; 3935 3936 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 3937 perf_pmu_disable(pmu); 3938 3939 pmu->sched_task(&cpc->epc, task, sched_in); 3940 3941 perf_pmu_enable(pmu); 3942 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 3943 } 3944 3945 static void perf_pmu_sched_task(struct task_struct *prev, 3946 struct task_struct *next, 3947 bool sched_in) 3948 { 3949 struct perf_cpu_pmu_context *cpc, *cpc2; 3950 3951 if (prev == next) 3952 return; 3953 3954 list_for_each_entry_safe(cpc, cpc2, this_cpu_ptr(&sched_cb_list), sched_cb_entry) { 3955 if (cpc->task_epc) 3956 continue; 3957 3958 __perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in); 3959 } 3960 } 3961 3962 static void perf_event_switch(struct task_struct *task, 3963 struct task_struct *next_prev, bool sched_in); 3964 3965 /* 3966 * Called from scheduler to remove the events of the current task, 3967 * with interrupts disabled. 3968 * 3969 * We stop each event and update the event value in event->count. 3970 * 3971 * This does not protect us against NMI, but disable() 3972 * sets the disabled bit in the control field of event _before_ 3973 * accessing the event control register. If a NMI hits, then it will 3974 * not restart the event. 3975 */ 3976 void __perf_event_task_sched_out(struct task_struct *task, 3977 struct task_struct *next) 3978 { 3979 if (__this_cpu_read(perf_sched_cb_usages)) 3980 perf_pmu_sched_task(task, next, false); 3981 3982 if (atomic_read(&nr_switch_events)) 3983 perf_event_switch(task, next, false); 3984 3985 perf_event_context_sched_out(task, next); 3986 3987 /* 3988 * if cgroup events exist on this CPU, then we need 3989 * to check if we have to switch out PMU state. 3990 * cgroup event are system-wide mode only 3991 */ 3992 perf_cgroup_switch(next); 3993 } 3994 3995 static bool perf_less_group_idx(const void *l, const void *r, void __always_unused *args) 3996 { 3997 const struct perf_event *le = *(const struct perf_event **)l; 3998 const struct perf_event *re = *(const struct perf_event **)r; 3999 4000 return le->group_index < re->group_index; 4001 } 4002 4003 DEFINE_MIN_HEAP(struct perf_event *, perf_event_min_heap); 4004 4005 static const struct min_heap_callbacks perf_min_heap = { 4006 .less = perf_less_group_idx, 4007 .swp = NULL, 4008 }; 4009 4010 static void __heap_add(struct perf_event_min_heap *heap, struct perf_event *event) 4011 { 4012 struct perf_event **itrs = heap->data; 4013 4014 if (event) { 4015 itrs[heap->nr] = event; 4016 heap->nr++; 4017 } 4018 } 4019 4020 static void __link_epc(struct perf_event_pmu_context *pmu_ctx) 4021 { 4022 struct perf_cpu_pmu_context *cpc; 4023 4024 if (!pmu_ctx->ctx->task) 4025 return; 4026 4027 cpc = this_cpc(pmu_ctx->pmu); 4028 WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx); 4029 cpc->task_epc = pmu_ctx; 4030 } 4031 4032 static noinline int visit_groups_merge(struct perf_event_context *ctx, 4033 struct perf_event_groups *groups, int cpu, 4034 struct pmu *pmu, 4035 int (*func)(struct perf_event *, void *), 4036 void *data) 4037 { 4038 #ifdef CONFIG_CGROUP_PERF 4039 struct cgroup_subsys_state *css = NULL; 4040 #endif 4041 struct perf_cpu_context *cpuctx = NULL; 4042 /* Space for per CPU and/or any CPU event iterators. */ 4043 struct perf_event *itrs[2]; 4044 struct perf_event_min_heap event_heap; 4045 struct perf_event **evt; 4046 int ret; 4047 4048 if (pmu->filter && pmu->filter(pmu, cpu)) 4049 return 0; 4050 4051 if (!ctx->task) { 4052 cpuctx = this_cpu_ptr(&perf_cpu_context); 4053 event_heap = (struct perf_event_min_heap){ 4054 .data = cpuctx->heap, 4055 .nr = 0, 4056 .size = cpuctx->heap_size, 4057 }; 4058 4059 lockdep_assert_held(&cpuctx->ctx.lock); 4060 4061 #ifdef CONFIG_CGROUP_PERF 4062 if (cpuctx->cgrp) 4063 css = &cpuctx->cgrp->css; 4064 #endif 4065 } else { 4066 event_heap = (struct perf_event_min_heap){ 4067 .data = itrs, 4068 .nr = 0, 4069 .size = ARRAY_SIZE(itrs), 4070 }; 4071 /* Events not within a CPU context may be on any CPU. */ 4072 __heap_add(&event_heap, perf_event_groups_first(groups, -1, pmu, NULL)); 4073 } 4074 evt = event_heap.data; 4075 4076 __heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, NULL)); 4077 4078 #ifdef CONFIG_CGROUP_PERF 4079 for (; css; css = css->parent) 4080 __heap_add(&event_heap, perf_event_groups_first(groups, cpu, pmu, css->cgroup)); 4081 #endif 4082 4083 if (event_heap.nr) { 4084 __link_epc((*evt)->pmu_ctx); 4085 perf_assert_pmu_disabled((*evt)->pmu_ctx->pmu); 4086 } 4087 4088 min_heapify_all_inline(&event_heap, &perf_min_heap, NULL); 4089 4090 while (event_heap.nr) { 4091 ret = func(*evt, data); 4092 if (ret) 4093 return ret; 4094 4095 *evt = perf_event_groups_next(*evt, pmu); 4096 if (*evt) 4097 min_heap_sift_down_inline(&event_heap, 0, &perf_min_heap, NULL); 4098 else 4099 min_heap_pop_inline(&event_heap, &perf_min_heap, NULL); 4100 } 4101 4102 return 0; 4103 } 4104 4105 /* 4106 * Because the userpage is strictly per-event (there is no concept of context, 4107 * so there cannot be a context indirection), every userpage must be updated 4108 * when context time starts :-( 4109 * 4110 * IOW, we must not miss EVENT_TIME edges. 4111 */ 4112 static inline bool event_update_userpage(struct perf_event *event) 4113 { 4114 if (likely(!refcount_read(&event->mmap_count))) 4115 return false; 4116 4117 perf_event_update_time(event); 4118 perf_event_update_userpage(event); 4119 4120 return true; 4121 } 4122 4123 static inline void group_update_userpage(struct perf_event *group_event) 4124 { 4125 struct perf_event *event; 4126 4127 if (!event_update_userpage(group_event)) 4128 return; 4129 4130 for_each_sibling_event(event, group_event) 4131 event_update_userpage(event); 4132 } 4133 4134 struct merge_sched_data { 4135 int can_add_hw; 4136 enum event_type_t event_type; 4137 }; 4138 4139 static int merge_sched_in(struct perf_event *event, void *data) 4140 { 4141 struct perf_event_context *ctx = event->ctx; 4142 struct merge_sched_data *msd = data; 4143 4144 if (event->state <= PERF_EVENT_STATE_OFF) 4145 return 0; 4146 4147 if (!event_filter_match(event)) 4148 return 0; 4149 4150 /* 4151 * Don't schedule in any host events from PMU with 4152 * PERF_PMU_CAP_MEDIATED_VPMU, while a guest is running. 4153 */ 4154 if (is_guest_mediated_pmu_loaded() && 4155 event->pmu_ctx->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU && 4156 !(msd->event_type & EVENT_GUEST)) 4157 return 0; 4158 4159 if (group_can_go_on(event, msd->can_add_hw)) { 4160 if (!group_sched_in(event, ctx)) 4161 list_add_tail(&event->active_list, get_event_list(event)); 4162 } 4163 4164 if (event->state == PERF_EVENT_STATE_INACTIVE) { 4165 msd->can_add_hw = 0; 4166 if (event->attr.pinned) { 4167 perf_cgroup_event_disable(event, ctx); 4168 perf_event_set_state(event, PERF_EVENT_STATE_ERROR); 4169 4170 if (*perf_event_fasync(event)) 4171 event->pending_kill = POLL_ERR; 4172 4173 event->pending_wakeup = 1; 4174 irq_work_queue(&event->pending_irq); 4175 } else { 4176 struct perf_cpu_pmu_context *cpc = this_cpc(event->pmu_ctx->pmu); 4177 4178 event->pmu_ctx->rotate_necessary = 1; 4179 perf_mux_hrtimer_restart(cpc); 4180 group_update_userpage(event); 4181 } 4182 } 4183 4184 return 0; 4185 } 4186 4187 static void pmu_groups_sched_in(struct perf_event_context *ctx, 4188 struct perf_event_groups *groups, 4189 struct pmu *pmu, 4190 enum event_type_t event_type) 4191 { 4192 struct merge_sched_data msd = { 4193 .can_add_hw = 1, 4194 .event_type = event_type, 4195 }; 4196 visit_groups_merge(ctx, groups, smp_processor_id(), pmu, 4197 merge_sched_in, &msd); 4198 } 4199 4200 static void __pmu_ctx_sched_in(struct perf_event_pmu_context *pmu_ctx, 4201 enum event_type_t event_type) 4202 { 4203 struct perf_event_context *ctx = pmu_ctx->ctx; 4204 4205 if (event_type & EVENT_PINNED) 4206 pmu_groups_sched_in(ctx, &ctx->pinned_groups, pmu_ctx->pmu, event_type); 4207 if (event_type & EVENT_FLEXIBLE) 4208 pmu_groups_sched_in(ctx, &ctx->flexible_groups, pmu_ctx->pmu, event_type); 4209 } 4210 4211 static void 4212 ctx_sched_in(struct perf_event_context *ctx, struct pmu *pmu, enum event_type_t event_type) 4213 { 4214 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 4215 enum event_type_t active_type = event_type & ~EVENT_FLAGS; 4216 struct perf_event_pmu_context *pmu_ctx; 4217 int is_active = ctx->is_active; 4218 4219 lockdep_assert_held(&ctx->lock); 4220 4221 if (likely(!ctx->nr_events)) 4222 return; 4223 4224 if (!(is_active & EVENT_TIME)) { 4225 /* EVENT_TIME should be active while the guest runs */ 4226 WARN_ON_ONCE(event_type & EVENT_GUEST); 4227 /* start ctx time */ 4228 __update_context_time(ctx, false); 4229 perf_cgroup_set_timestamp(cpuctx, false); 4230 /* 4231 * CPU-release for the below ->is_active store, 4232 * see __load_acquire() in perf_event_time_now() 4233 */ 4234 barrier(); 4235 } 4236 4237 ctx->is_active |= active_type | EVENT_TIME; 4238 if (ctx->task) { 4239 if (!(is_active & EVENT_ALL)) 4240 cpuctx->task_ctx = ctx; 4241 else 4242 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 4243 } 4244 4245 if (event_type & EVENT_GUEST) { 4246 /* 4247 * Schedule in the required exclude_guest events of PMU 4248 * with PERF_PMU_CAP_MEDIATED_VPMU. 4249 */ 4250 is_active = event_type & EVENT_ALL; 4251 4252 /* 4253 * Update ctx time to set the new start time for 4254 * the exclude_guest events. 4255 */ 4256 update_context_time(ctx); 4257 update_cgrp_time_from_cpuctx(cpuctx, false); 4258 barrier(); 4259 } else { 4260 is_active ^= ctx->is_active; /* changed bits */ 4261 } 4262 4263 /* 4264 * First go through the list and put on any pinned groups 4265 * in order to give them the best chance of going on. 4266 */ 4267 if (is_active & EVENT_PINNED) { 4268 for_each_epc(pmu_ctx, ctx, pmu, event_type) 4269 __pmu_ctx_sched_in(pmu_ctx, EVENT_PINNED | (event_type & EVENT_GUEST)); 4270 } 4271 4272 /* Then walk through the lower prio flexible groups */ 4273 if (is_active & EVENT_FLEXIBLE) { 4274 for_each_epc(pmu_ctx, ctx, pmu, event_type) 4275 __pmu_ctx_sched_in(pmu_ctx, EVENT_FLEXIBLE | (event_type & EVENT_GUEST)); 4276 } 4277 } 4278 4279 static void perf_event_context_sched_in(struct task_struct *task) 4280 { 4281 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 4282 struct perf_event_context *ctx; 4283 4284 rcu_read_lock(); 4285 ctx = rcu_dereference(task->perf_event_ctxp); 4286 if (!ctx) 4287 goto rcu_unlock; 4288 4289 if (cpuctx->task_ctx == ctx) { 4290 perf_ctx_lock(cpuctx, ctx); 4291 perf_ctx_disable(ctx, 0); 4292 4293 perf_ctx_sched_task_cb(ctx, task, true); 4294 4295 perf_ctx_enable(ctx, 0); 4296 perf_ctx_unlock(cpuctx, ctx); 4297 goto rcu_unlock; 4298 } 4299 4300 perf_ctx_lock(cpuctx, ctx); 4301 /* 4302 * We must check ctx->nr_events while holding ctx->lock, such 4303 * that we serialize against perf_install_in_context(). 4304 */ 4305 if (!ctx->nr_events) 4306 goto unlock; 4307 4308 perf_ctx_disable(ctx, 0); 4309 /* 4310 * We want to keep the following priority order: 4311 * cpu pinned (that don't need to move), task pinned, 4312 * cpu flexible, task flexible. 4313 * 4314 * However, if task's ctx is not carrying any pinned 4315 * events, no need to flip the cpuctx's events around. 4316 */ 4317 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) { 4318 perf_ctx_disable(&cpuctx->ctx, 0); 4319 ctx_sched_out(&cpuctx->ctx, NULL, EVENT_FLEXIBLE); 4320 } 4321 4322 perf_event_sched_in(cpuctx, ctx, NULL, 0); 4323 4324 perf_ctx_sched_task_cb(cpuctx->task_ctx, task, true); 4325 4326 if (!RB_EMPTY_ROOT(&ctx->pinned_groups.tree)) 4327 perf_ctx_enable(&cpuctx->ctx, 0); 4328 4329 perf_ctx_enable(ctx, 0); 4330 4331 unlock: 4332 perf_ctx_unlock(cpuctx, ctx); 4333 rcu_unlock: 4334 rcu_read_unlock(); 4335 } 4336 4337 /* 4338 * Called from scheduler to add the events of the current task 4339 * with interrupts disabled. 4340 * 4341 * We restore the event value and then enable it. 4342 * 4343 * This does not protect us against NMI, but enable() 4344 * sets the enabled bit in the control field of event _before_ 4345 * accessing the event control register. If a NMI hits, then it will 4346 * keep the event running. 4347 */ 4348 void __perf_event_task_sched_in(struct task_struct *prev, 4349 struct task_struct *task) 4350 { 4351 perf_event_context_sched_in(task); 4352 4353 if (atomic_read(&nr_switch_events)) 4354 perf_event_switch(task, prev, true); 4355 4356 if (__this_cpu_read(perf_sched_cb_usages)) 4357 perf_pmu_sched_task(prev, task, true); 4358 } 4359 4360 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 4361 { 4362 u64 frequency = event->attr.sample_freq; 4363 u64 sec = NSEC_PER_SEC; 4364 u64 divisor, dividend; 4365 4366 int count_fls, nsec_fls, frequency_fls, sec_fls; 4367 4368 count_fls = fls64(count); 4369 nsec_fls = fls64(nsec); 4370 frequency_fls = fls64(frequency); 4371 sec_fls = 30; 4372 4373 /* 4374 * We got @count in @nsec, with a target of sample_freq HZ 4375 * the target period becomes: 4376 * 4377 * @count * 10^9 4378 * period = ------------------- 4379 * @nsec * sample_freq 4380 * 4381 */ 4382 4383 /* 4384 * Reduce accuracy by one bit such that @a and @b converge 4385 * to a similar magnitude. 4386 */ 4387 #define REDUCE_FLS(a, b) \ 4388 do { \ 4389 if (a##_fls > b##_fls) { \ 4390 a >>= 1; \ 4391 a##_fls--; \ 4392 } else { \ 4393 b >>= 1; \ 4394 b##_fls--; \ 4395 } \ 4396 } while (0) 4397 4398 /* 4399 * Reduce accuracy until either term fits in a u64, then proceed with 4400 * the other, so that finally we can do a u64/u64 division. 4401 */ 4402 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) { 4403 REDUCE_FLS(nsec, frequency); 4404 REDUCE_FLS(sec, count); 4405 } 4406 4407 if (count_fls + sec_fls > 64) { 4408 divisor = nsec * frequency; 4409 4410 while (count_fls + sec_fls > 64) { 4411 REDUCE_FLS(count, sec); 4412 divisor >>= 1; 4413 } 4414 4415 dividend = count * sec; 4416 } else { 4417 dividend = count * sec; 4418 4419 while (nsec_fls + frequency_fls > 64) { 4420 REDUCE_FLS(nsec, frequency); 4421 dividend >>= 1; 4422 } 4423 4424 divisor = nsec * frequency; 4425 } 4426 4427 if (!divisor) 4428 return dividend; 4429 4430 return div64_u64(dividend, divisor); 4431 } 4432 4433 static DEFINE_PER_CPU(int, perf_throttled_count); 4434 static DEFINE_PER_CPU(u64, perf_throttled_seq); 4435 4436 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable) 4437 { 4438 struct hw_perf_event *hwc = &event->hw; 4439 s64 period, sample_period; 4440 s64 delta; 4441 4442 period = perf_calculate_period(event, nsec, count); 4443 4444 delta = (s64)(period - hwc->sample_period); 4445 if (delta >= 0) 4446 delta += 7; 4447 else 4448 delta -= 7; 4449 delta /= 8; /* low pass filter */ 4450 4451 sample_period = hwc->sample_period + delta; 4452 4453 if (!sample_period) 4454 sample_period = 1; 4455 4456 hwc->sample_period = sample_period; 4457 4458 if (local64_read(&hwc->period_left) > 8*sample_period) { 4459 if (disable) 4460 event->pmu->stop(event, PERF_EF_UPDATE); 4461 4462 local64_set(&hwc->period_left, 0); 4463 4464 if (disable) 4465 event->pmu->start(event, PERF_EF_RELOAD); 4466 } 4467 } 4468 4469 static void perf_adjust_freq_unthr_events(struct list_head *event_list) 4470 { 4471 struct perf_event *event; 4472 struct hw_perf_event *hwc; 4473 u64 now, period = TICK_NSEC; 4474 s64 delta; 4475 4476 list_for_each_entry(event, event_list, active_list) { 4477 if (event->state != PERF_EVENT_STATE_ACTIVE) 4478 continue; 4479 4480 // XXX use visit thingy to avoid the -1,cpu match 4481 if (!event_filter_match(event)) 4482 continue; 4483 4484 hwc = &event->hw; 4485 4486 if (hwc->interrupts == MAX_INTERRUPTS) 4487 perf_event_unthrottle_group(event, is_event_in_freq_mode(event)); 4488 4489 if (!is_event_in_freq_mode(event)) 4490 continue; 4491 4492 /* 4493 * stop the event and update event->count 4494 */ 4495 event->pmu->stop(event, PERF_EF_UPDATE); 4496 4497 now = local64_read(&event->count); 4498 delta = now - hwc->freq_count_stamp; 4499 hwc->freq_count_stamp = now; 4500 4501 /* 4502 * restart the event 4503 * reload only if value has changed 4504 * we have stopped the event so tell that 4505 * to perf_adjust_period() to avoid stopping it 4506 * twice. 4507 */ 4508 if (delta > 0) 4509 perf_adjust_period(event, period, delta, false); 4510 4511 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 4512 } 4513 } 4514 4515 /* 4516 * combine freq adjustment with unthrottling to avoid two passes over the 4517 * events. At the same time, make sure, having freq events does not change 4518 * the rate of unthrottling as that would introduce bias. 4519 */ 4520 static void 4521 perf_adjust_freq_unthr_context(struct perf_event_context *ctx, bool unthrottle) 4522 { 4523 struct perf_event_pmu_context *pmu_ctx; 4524 4525 /* 4526 * only need to iterate over all events iff: 4527 * - context have events in frequency mode (needs freq adjust) 4528 * - there are events to unthrottle on this cpu 4529 */ 4530 if (!(ctx->nr_freq || unthrottle)) 4531 return; 4532 4533 raw_spin_lock(&ctx->lock); 4534 4535 list_for_each_entry(pmu_ctx, &ctx->pmu_ctx_list, pmu_ctx_entry) { 4536 if (!(pmu_ctx->nr_freq || unthrottle)) 4537 continue; 4538 if (!perf_pmu_ctx_is_active(pmu_ctx)) 4539 continue; 4540 if (pmu_ctx->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) 4541 continue; 4542 4543 perf_pmu_disable(pmu_ctx->pmu); 4544 perf_adjust_freq_unthr_events(&pmu_ctx->pinned_active); 4545 perf_adjust_freq_unthr_events(&pmu_ctx->flexible_active); 4546 perf_pmu_enable(pmu_ctx->pmu); 4547 } 4548 4549 raw_spin_unlock(&ctx->lock); 4550 } 4551 4552 /* 4553 * Move @event to the tail of the @ctx's elegible events. 4554 */ 4555 static void rotate_ctx(struct perf_event_context *ctx, struct perf_event *event) 4556 { 4557 /* 4558 * Rotate the first entry last of non-pinned groups. Rotation might be 4559 * disabled by the inheritance code. 4560 */ 4561 if (ctx->rotate_disable) 4562 return; 4563 4564 perf_event_groups_delete(&ctx->flexible_groups, event); 4565 perf_event_groups_insert(&ctx->flexible_groups, event); 4566 } 4567 4568 /* pick an event from the flexible_groups to rotate */ 4569 static inline struct perf_event * 4570 ctx_event_to_rotate(struct perf_event_pmu_context *pmu_ctx) 4571 { 4572 struct perf_event *event; 4573 struct rb_node *node; 4574 struct rb_root *tree; 4575 struct __group_key key = { 4576 .pmu = pmu_ctx->pmu, 4577 }; 4578 4579 /* pick the first active flexible event */ 4580 event = list_first_entry_or_null(&pmu_ctx->flexible_active, 4581 struct perf_event, active_list); 4582 if (event) 4583 goto out; 4584 4585 /* if no active flexible event, pick the first event */ 4586 tree = &pmu_ctx->ctx->flexible_groups.tree; 4587 4588 if (!pmu_ctx->ctx->task) { 4589 key.cpu = smp_processor_id(); 4590 4591 node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup); 4592 if (node) 4593 event = __node_2_pe(node); 4594 goto out; 4595 } 4596 4597 key.cpu = -1; 4598 node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup); 4599 if (node) { 4600 event = __node_2_pe(node); 4601 goto out; 4602 } 4603 4604 key.cpu = smp_processor_id(); 4605 node = rb_find_first(&key, tree, __group_cmp_ignore_cgroup); 4606 if (node) 4607 event = __node_2_pe(node); 4608 4609 out: 4610 /* 4611 * Unconditionally clear rotate_necessary; if ctx_flexible_sched_in() 4612 * finds there are unschedulable events, it will set it again. 4613 */ 4614 pmu_ctx->rotate_necessary = 0; 4615 4616 return event; 4617 } 4618 4619 static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc) 4620 { 4621 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 4622 struct perf_event_pmu_context *cpu_epc, *task_epc = NULL; 4623 struct perf_event *cpu_event = NULL, *task_event = NULL; 4624 int cpu_rotate, task_rotate; 4625 struct pmu *pmu; 4626 4627 /* 4628 * Since we run this from IRQ context, nobody can install new 4629 * events, thus the event count values are stable. 4630 */ 4631 4632 cpu_epc = &cpc->epc; 4633 pmu = cpu_epc->pmu; 4634 task_epc = cpc->task_epc; 4635 4636 cpu_rotate = cpu_epc->rotate_necessary; 4637 task_rotate = task_epc ? task_epc->rotate_necessary : 0; 4638 4639 if (!(cpu_rotate || task_rotate)) 4640 return false; 4641 4642 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 4643 perf_pmu_disable(pmu); 4644 4645 if (task_rotate) 4646 task_event = ctx_event_to_rotate(task_epc); 4647 if (cpu_rotate) 4648 cpu_event = ctx_event_to_rotate(cpu_epc); 4649 4650 /* 4651 * As per the order given at ctx_resched() first 'pop' task flexible 4652 * and then, if needed CPU flexible. 4653 */ 4654 if (task_event || (task_epc && cpu_event)) { 4655 update_context_time(task_epc->ctx); 4656 __pmu_ctx_sched_out(task_epc, EVENT_FLEXIBLE); 4657 } 4658 4659 if (cpu_event) { 4660 update_context_time(&cpuctx->ctx); 4661 update_cgrp_time_from_cpuctx(cpuctx, false); 4662 __pmu_ctx_sched_out(cpu_epc, EVENT_FLEXIBLE); 4663 rotate_ctx(&cpuctx->ctx, cpu_event); 4664 __pmu_ctx_sched_in(cpu_epc, EVENT_FLEXIBLE); 4665 } 4666 4667 if (task_event) 4668 rotate_ctx(task_epc->ctx, task_event); 4669 4670 if (task_event || (task_epc && cpu_event)) 4671 __pmu_ctx_sched_in(task_epc, EVENT_FLEXIBLE); 4672 4673 perf_pmu_enable(pmu); 4674 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 4675 4676 return true; 4677 } 4678 4679 void perf_event_task_tick(void) 4680 { 4681 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 4682 struct perf_event_context *ctx; 4683 int throttled; 4684 4685 lockdep_assert_irqs_disabled(); 4686 4687 __this_cpu_inc(perf_throttled_seq); 4688 throttled = __this_cpu_xchg(perf_throttled_count, 0); 4689 tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); 4690 4691 perf_adjust_freq_unthr_context(&cpuctx->ctx, !!throttled); 4692 4693 rcu_read_lock(); 4694 ctx = rcu_dereference(current->perf_event_ctxp); 4695 if (ctx) 4696 perf_adjust_freq_unthr_context(ctx, !!throttled); 4697 rcu_read_unlock(); 4698 } 4699 4700 static int event_enable_on_exec(struct perf_event *event, 4701 struct perf_event_context *ctx) 4702 { 4703 if (!event->attr.enable_on_exec) 4704 return 0; 4705 4706 event->attr.enable_on_exec = 0; 4707 if (event->state >= PERF_EVENT_STATE_INACTIVE) 4708 return 0; 4709 4710 perf_event_set_state(event, PERF_EVENT_STATE_INACTIVE); 4711 4712 return 1; 4713 } 4714 4715 /* 4716 * Enable all of a task's events that have been marked enable-on-exec. 4717 * This expects task == current. 4718 */ 4719 static void perf_event_enable_on_exec(struct perf_event_context *ctx) 4720 { 4721 struct perf_event_context *clone_ctx = NULL; 4722 enum event_type_t event_type = 0; 4723 struct perf_cpu_context *cpuctx; 4724 struct perf_event *event; 4725 unsigned long flags; 4726 int enabled = 0; 4727 4728 local_irq_save(flags); 4729 if (WARN_ON_ONCE(current->perf_event_ctxp != ctx)) 4730 goto out; 4731 4732 if (!ctx->nr_events) 4733 goto out; 4734 4735 cpuctx = this_cpu_ptr(&perf_cpu_context); 4736 perf_ctx_lock(cpuctx, ctx); 4737 ctx_time_freeze(cpuctx, ctx); 4738 4739 list_for_each_entry(event, &ctx->event_list, event_entry) { 4740 enabled |= event_enable_on_exec(event, ctx); 4741 event_type |= get_event_type(event); 4742 } 4743 4744 /* 4745 * Unclone and reschedule this context if we enabled any event. 4746 */ 4747 if (enabled) { 4748 clone_ctx = unclone_ctx(ctx); 4749 ctx_resched(cpuctx, ctx, NULL, event_type); 4750 } 4751 perf_ctx_unlock(cpuctx, ctx); 4752 4753 out: 4754 local_irq_restore(flags); 4755 4756 if (clone_ctx) 4757 put_ctx(clone_ctx); 4758 } 4759 4760 static void perf_remove_from_owner(struct perf_event *event); 4761 static void perf_event_exit_event(struct perf_event *event, 4762 struct perf_event_context *ctx, 4763 struct task_struct *task, 4764 unsigned long detach_flags); 4765 4766 /* 4767 * Removes all events from the current task that have been marked 4768 * remove-on-exec, and feeds their values back to parent events. 4769 */ 4770 static void perf_event_remove_on_exec(struct perf_event_context *ctx) 4771 { 4772 struct perf_event_context *clone_ctx = NULL; 4773 struct perf_event *event, *next; 4774 unsigned long flags; 4775 bool modified = false; 4776 4777 mutex_lock(&ctx->mutex); 4778 4779 if (WARN_ON_ONCE(ctx->task != current)) 4780 goto unlock; 4781 4782 list_for_each_entry_safe(event, next, &ctx->event_list, event_entry) { 4783 if (!event->attr.remove_on_exec) 4784 continue; 4785 4786 if (!is_kernel_event(event)) 4787 perf_remove_from_owner(event); 4788 4789 modified = true; 4790 4791 perf_event_exit_event(event, ctx, ctx->task, DETACH_GROUP); 4792 } 4793 4794 raw_spin_lock_irqsave(&ctx->lock, flags); 4795 if (modified) 4796 clone_ctx = unclone_ctx(ctx); 4797 raw_spin_unlock_irqrestore(&ctx->lock, flags); 4798 4799 unlock: 4800 mutex_unlock(&ctx->mutex); 4801 4802 if (clone_ctx) 4803 put_ctx(clone_ctx); 4804 } 4805 4806 struct perf_read_data { 4807 struct perf_event *event; 4808 bool group; 4809 int ret; 4810 }; 4811 4812 static inline const struct cpumask *perf_scope_cpu_topology_cpumask(unsigned int scope, int cpu); 4813 4814 static int __perf_event_read_cpu(struct perf_event *event, int event_cpu) 4815 { 4816 int local_cpu = smp_processor_id(); 4817 u16 local_pkg, event_pkg; 4818 4819 if ((unsigned)event_cpu >= nr_cpu_ids) 4820 return event_cpu; 4821 4822 if (event->group_caps & PERF_EV_CAP_READ_SCOPE) { 4823 const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(event->pmu->scope, event_cpu); 4824 4825 if (cpumask && cpumask_test_cpu(local_cpu, cpumask)) 4826 return local_cpu; 4827 } 4828 4829 if (event->group_caps & PERF_EV_CAP_READ_ACTIVE_PKG) { 4830 event_pkg = topology_physical_package_id(event_cpu); 4831 local_pkg = topology_physical_package_id(local_cpu); 4832 4833 if (event_pkg == local_pkg) 4834 return local_cpu; 4835 } 4836 4837 return event_cpu; 4838 } 4839 4840 /* 4841 * Cross CPU call to read the hardware event 4842 */ 4843 static void __perf_event_read(void *info) 4844 { 4845 struct perf_read_data *data = info; 4846 struct perf_event *sub, *event = data->event; 4847 struct perf_event_context *ctx = event->ctx; 4848 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 4849 struct pmu *pmu; 4850 4851 /* 4852 * If this is a task context, we need to check whether it is 4853 * the current task context of this cpu. If not it has been 4854 * scheduled out before the smp call arrived. In that case 4855 * event->count would have been updated to a recent sample 4856 * when the event was scheduled out. 4857 */ 4858 if (ctx->task && cpuctx->task_ctx != ctx) 4859 return; 4860 4861 guard(raw_spinlock)(&ctx->lock); 4862 ctx_time_update_event(ctx, event); 4863 4864 perf_event_update_time(event); 4865 if (data->group) 4866 perf_event_update_sibling_time(event); 4867 4868 if (event->state != PERF_EVENT_STATE_ACTIVE) 4869 return; 4870 4871 if (!data->group) { 4872 perf_pmu_read(event); 4873 data->ret = 0; 4874 return; 4875 } 4876 4877 pmu = event->pmu_ctx->pmu; 4878 pmu->start_txn(pmu, PERF_PMU_TXN_READ); 4879 4880 perf_pmu_read(event); 4881 for_each_sibling_event(sub, event) 4882 perf_pmu_read(sub); 4883 4884 data->ret = pmu->commit_txn(pmu); 4885 } 4886 4887 static inline u64 perf_event_count(struct perf_event *event, bool self) 4888 { 4889 if (self) 4890 return local64_read(&event->count); 4891 4892 return local64_read(&event->count) + atomic64_read(&event->child_count); 4893 } 4894 4895 static void calc_timer_values(struct perf_event *event, 4896 u64 *now, 4897 u64 *enabled, 4898 u64 *running) 4899 { 4900 u64 ctx_time; 4901 4902 *now = perf_clock(); 4903 ctx_time = perf_event_time_now(event, *now); 4904 __perf_update_times(event, ctx_time, enabled, running); 4905 } 4906 4907 /* 4908 * NMI-safe method to read a local event, that is an event that 4909 * is: 4910 * - either for the current task, or for this CPU 4911 * - does not have inherit set, for inherited task events 4912 * will not be local and we cannot read them atomically 4913 * - must not have a pmu::count method 4914 */ 4915 int perf_event_read_local(struct perf_event *event, u64 *value, 4916 u64 *enabled, u64 *running) 4917 { 4918 unsigned long flags; 4919 int event_oncpu; 4920 int event_cpu; 4921 int ret = 0; 4922 4923 /* 4924 * Disabling interrupts avoids all counter scheduling (context 4925 * switches, timer based rotation and IPIs). 4926 */ 4927 local_irq_save(flags); 4928 4929 /* 4930 * It must not be an event with inherit set, we cannot read 4931 * all child counters from atomic context. 4932 */ 4933 if (event->attr.inherit) { 4934 ret = -EOPNOTSUPP; 4935 goto out; 4936 } 4937 4938 /* If this is a per-task event, it must be for current */ 4939 if ((event->attach_state & PERF_ATTACH_TASK) && 4940 event->hw.target != current) { 4941 ret = -EINVAL; 4942 goto out; 4943 } 4944 4945 /* 4946 * Get the event CPU numbers, and adjust them to local if the event is 4947 * a per-package event that can be read locally 4948 */ 4949 event_oncpu = __perf_event_read_cpu(event, event->oncpu); 4950 event_cpu = __perf_event_read_cpu(event, event->cpu); 4951 4952 /* If this is a per-CPU event, it must be for this CPU */ 4953 if (!(event->attach_state & PERF_ATTACH_TASK) && 4954 event_cpu != smp_processor_id()) { 4955 ret = -EINVAL; 4956 goto out; 4957 } 4958 4959 /* If this is a pinned event it must be running on this CPU */ 4960 if (event->attr.pinned && event_oncpu != smp_processor_id()) { 4961 ret = -EBUSY; 4962 goto out; 4963 } 4964 4965 /* 4966 * If the event is currently on this CPU, its either a per-task event, 4967 * or local to this CPU. Furthermore it means its ACTIVE (otherwise 4968 * oncpu == -1). 4969 */ 4970 if (event_oncpu == smp_processor_id()) 4971 event->pmu->read(event); 4972 4973 *value = local64_read(&event->count); 4974 if (enabled || running) { 4975 u64 __enabled, __running, __now; 4976 4977 calc_timer_values(event, &__now, &__enabled, &__running); 4978 if (enabled) 4979 *enabled = __enabled; 4980 if (running) 4981 *running = __running; 4982 } 4983 out: 4984 local_irq_restore(flags); 4985 4986 return ret; 4987 } 4988 4989 static int perf_event_read(struct perf_event *event, bool group) 4990 { 4991 enum perf_event_state state = READ_ONCE(event->state); 4992 int event_cpu, ret = 0; 4993 4994 /* 4995 * If event is enabled and currently active on a CPU, update the 4996 * value in the event structure: 4997 */ 4998 again: 4999 if (state == PERF_EVENT_STATE_ACTIVE) { 5000 struct perf_read_data data; 5001 5002 /* 5003 * Orders the ->state and ->oncpu loads such that if we see 5004 * ACTIVE we must also see the right ->oncpu. 5005 * 5006 * Matches the smp_wmb() from event_sched_in(). 5007 */ 5008 smp_rmb(); 5009 5010 event_cpu = READ_ONCE(event->oncpu); 5011 if ((unsigned)event_cpu >= nr_cpu_ids) 5012 return 0; 5013 5014 data = (struct perf_read_data){ 5015 .event = event, 5016 .group = group, 5017 .ret = 0, 5018 }; 5019 5020 preempt_disable(); 5021 event_cpu = __perf_event_read_cpu(event, event_cpu); 5022 5023 /* 5024 * Purposely ignore the smp_call_function_single() return 5025 * value. 5026 * 5027 * If event_cpu isn't a valid CPU it means the event got 5028 * scheduled out and that will have updated the event count. 5029 * 5030 * Therefore, either way, we'll have an up-to-date event count 5031 * after this. 5032 */ 5033 (void)smp_call_function_single(event_cpu, __perf_event_read, &data, 1); 5034 preempt_enable(); 5035 ret = data.ret; 5036 5037 } else if (state == PERF_EVENT_STATE_INACTIVE) { 5038 struct perf_event_context *ctx = event->ctx; 5039 unsigned long flags; 5040 5041 raw_spin_lock_irqsave(&ctx->lock, flags); 5042 state = event->state; 5043 if (state != PERF_EVENT_STATE_INACTIVE) { 5044 raw_spin_unlock_irqrestore(&ctx->lock, flags); 5045 goto again; 5046 } 5047 5048 /* 5049 * May read while context is not active (e.g., thread is 5050 * blocked), in that case we cannot update context time 5051 */ 5052 ctx_time_update_event(ctx, event); 5053 5054 perf_event_update_time(event); 5055 if (group) 5056 perf_event_update_sibling_time(event); 5057 raw_spin_unlock_irqrestore(&ctx->lock, flags); 5058 } 5059 5060 return ret; 5061 } 5062 5063 /* 5064 * Initialize the perf_event context in a task_struct: 5065 */ 5066 static void __perf_event_init_context(struct perf_event_context *ctx) 5067 { 5068 raw_spin_lock_init(&ctx->lock); 5069 mutex_init(&ctx->mutex); 5070 INIT_LIST_HEAD(&ctx->pmu_ctx_list); 5071 perf_event_groups_init(&ctx->pinned_groups); 5072 perf_event_groups_init(&ctx->flexible_groups); 5073 INIT_LIST_HEAD(&ctx->event_list); 5074 refcount_set(&ctx->refcount, 1); 5075 } 5076 5077 static void 5078 __perf_init_event_pmu_context(struct perf_event_pmu_context *epc, struct pmu *pmu) 5079 { 5080 epc->pmu = pmu; 5081 INIT_LIST_HEAD(&epc->pmu_ctx_entry); 5082 INIT_LIST_HEAD(&epc->pinned_active); 5083 INIT_LIST_HEAD(&epc->flexible_active); 5084 atomic_set(&epc->refcount, 1); 5085 } 5086 5087 static struct perf_event_context * 5088 alloc_perf_context(struct task_struct *task) 5089 { 5090 struct perf_event_context *ctx; 5091 5092 ctx = kzalloc_obj(struct perf_event_context); 5093 if (!ctx) 5094 return NULL; 5095 5096 __perf_event_init_context(ctx); 5097 if (task) 5098 ctx->task = get_task_struct(task); 5099 5100 return ctx; 5101 } 5102 5103 static struct task_struct * 5104 find_lively_task_by_vpid(pid_t vpid) 5105 { 5106 struct task_struct *task; 5107 5108 rcu_read_lock(); 5109 if (!vpid) 5110 task = current; 5111 else 5112 task = find_task_by_vpid(vpid); 5113 if (task) 5114 get_task_struct(task); 5115 rcu_read_unlock(); 5116 5117 if (!task) 5118 return ERR_PTR(-ESRCH); 5119 5120 return task; 5121 } 5122 5123 /* 5124 * Returns a matching context with refcount and pincount. 5125 */ 5126 static struct perf_event_context * 5127 find_get_context(struct task_struct *task, struct perf_event *event) 5128 { 5129 struct perf_event_context *ctx, *clone_ctx = NULL; 5130 struct perf_cpu_context *cpuctx; 5131 unsigned long flags; 5132 int err; 5133 5134 if (!task) { 5135 /* Must be root to operate on a CPU event: */ 5136 err = perf_allow_cpu(); 5137 if (err) 5138 return ERR_PTR(err); 5139 5140 cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); 5141 ctx = &cpuctx->ctx; 5142 get_ctx(ctx); 5143 raw_spin_lock_irqsave(&ctx->lock, flags); 5144 ++ctx->pin_count; 5145 raw_spin_unlock_irqrestore(&ctx->lock, flags); 5146 5147 return ctx; 5148 } 5149 5150 err = -EINVAL; 5151 retry: 5152 ctx = perf_lock_task_context(task, &flags); 5153 if (ctx) { 5154 clone_ctx = unclone_ctx(ctx); 5155 ++ctx->pin_count; 5156 5157 raw_spin_unlock_irqrestore(&ctx->lock, flags); 5158 5159 if (clone_ctx) 5160 put_ctx(clone_ctx); 5161 } else { 5162 ctx = alloc_perf_context(task); 5163 err = -ENOMEM; 5164 if (!ctx) 5165 goto errout; 5166 5167 err = 0; 5168 mutex_lock(&task->perf_event_mutex); 5169 /* 5170 * If it has already passed perf_event_exit_task(). 5171 * we must see PF_EXITING, it takes this mutex too. 5172 */ 5173 if (task->flags & PF_EXITING) 5174 err = -ESRCH; 5175 else if (task->perf_event_ctxp) 5176 err = -EAGAIN; 5177 else { 5178 get_ctx(ctx); 5179 ++ctx->pin_count; 5180 rcu_assign_pointer(task->perf_event_ctxp, ctx); 5181 } 5182 mutex_unlock(&task->perf_event_mutex); 5183 5184 if (unlikely(err)) { 5185 put_ctx(ctx); 5186 5187 if (err == -EAGAIN) 5188 goto retry; 5189 goto errout; 5190 } 5191 } 5192 5193 return ctx; 5194 5195 errout: 5196 return ERR_PTR(err); 5197 } 5198 5199 static struct perf_event_pmu_context * 5200 find_get_pmu_context(struct pmu *pmu, struct perf_event_context *ctx, 5201 struct perf_event *event) 5202 { 5203 struct perf_event_pmu_context *new = NULL, *pos = NULL, *epc; 5204 5205 if (!ctx->task) { 5206 /* 5207 * perf_pmu_migrate_context() / __perf_pmu_install_event() 5208 * relies on the fact that find_get_pmu_context() cannot fail 5209 * for CPU contexts. 5210 */ 5211 struct perf_cpu_pmu_context *cpc; 5212 5213 cpc = *per_cpu_ptr(pmu->cpu_pmu_context, event->cpu); 5214 epc = &cpc->epc; 5215 raw_spin_lock_irq(&ctx->lock); 5216 if (!epc->ctx) { 5217 /* 5218 * One extra reference for the pmu; see perf_pmu_free(). 5219 */ 5220 atomic_set(&epc->refcount, 2); 5221 epc->embedded = 1; 5222 list_add(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list); 5223 epc->ctx = ctx; 5224 } else { 5225 WARN_ON_ONCE(epc->ctx != ctx); 5226 atomic_inc(&epc->refcount); 5227 } 5228 raw_spin_unlock_irq(&ctx->lock); 5229 return epc; 5230 } 5231 5232 new = kzalloc_obj(*epc); 5233 if (!new) 5234 return ERR_PTR(-ENOMEM); 5235 5236 __perf_init_event_pmu_context(new, pmu); 5237 5238 /* 5239 * XXX 5240 * 5241 * lockdep_assert_held(&ctx->mutex); 5242 * 5243 * can't because perf_event_init_task() doesn't actually hold the 5244 * child_ctx->mutex. 5245 */ 5246 5247 raw_spin_lock_irq(&ctx->lock); 5248 list_for_each_entry(epc, &ctx->pmu_ctx_list, pmu_ctx_entry) { 5249 if (epc->pmu == pmu) { 5250 WARN_ON_ONCE(epc->ctx != ctx); 5251 atomic_inc(&epc->refcount); 5252 goto found_epc; 5253 } 5254 /* Make sure the pmu_ctx_list is sorted by PMU type: */ 5255 if (!pos && epc->pmu->type > pmu->type) 5256 pos = epc; 5257 } 5258 5259 epc = new; 5260 new = NULL; 5261 5262 if (!pos) 5263 list_add_tail(&epc->pmu_ctx_entry, &ctx->pmu_ctx_list); 5264 else 5265 list_add(&epc->pmu_ctx_entry, pos->pmu_ctx_entry.prev); 5266 5267 epc->ctx = ctx; 5268 5269 found_epc: 5270 raw_spin_unlock_irq(&ctx->lock); 5271 kfree(new); 5272 5273 return epc; 5274 } 5275 5276 static void get_pmu_ctx(struct perf_event_pmu_context *epc) 5277 { 5278 WARN_ON_ONCE(!atomic_inc_not_zero(&epc->refcount)); 5279 } 5280 5281 static void free_cpc_rcu(struct rcu_head *head) 5282 { 5283 struct perf_cpu_pmu_context *cpc = 5284 container_of(head, typeof(*cpc), epc.rcu_head); 5285 5286 kfree(cpc); 5287 } 5288 5289 static void free_epc_rcu(struct rcu_head *head) 5290 { 5291 struct perf_event_pmu_context *epc = container_of(head, typeof(*epc), rcu_head); 5292 5293 kfree(epc); 5294 } 5295 5296 static void put_pmu_ctx(struct perf_event_pmu_context *epc) 5297 { 5298 struct perf_event_context *ctx = epc->ctx; 5299 unsigned long flags; 5300 5301 /* 5302 * XXX 5303 * 5304 * lockdep_assert_held(&ctx->mutex); 5305 * 5306 * can't because of the call-site in _free_event()/put_event() 5307 * which isn't always called under ctx->mutex. 5308 */ 5309 if (!atomic_dec_and_raw_lock_irqsave(&epc->refcount, &ctx->lock, flags)) 5310 return; 5311 5312 WARN_ON_ONCE(list_empty(&epc->pmu_ctx_entry)); 5313 5314 list_del_init(&epc->pmu_ctx_entry); 5315 epc->ctx = NULL; 5316 5317 WARN_ON_ONCE(!list_empty(&epc->pinned_active)); 5318 WARN_ON_ONCE(!list_empty(&epc->flexible_active)); 5319 5320 raw_spin_unlock_irqrestore(&ctx->lock, flags); 5321 5322 if (epc->embedded) { 5323 call_rcu(&epc->rcu_head, free_cpc_rcu); 5324 return; 5325 } 5326 5327 call_rcu(&epc->rcu_head, free_epc_rcu); 5328 } 5329 5330 static void perf_event_free_filter(struct perf_event *event); 5331 5332 static void free_event_rcu(struct rcu_head *head) 5333 { 5334 struct perf_event *event = container_of(head, typeof(*event), rcu_head); 5335 5336 if (event->ns) 5337 put_pid_ns(event->ns); 5338 perf_event_free_filter(event); 5339 kfree(event->addr_filter_ranges); 5340 kmem_cache_free(perf_event_cache, event); 5341 } 5342 5343 static void ring_buffer_attach(struct perf_event *event, 5344 struct perf_buffer *rb); 5345 5346 static void detach_sb_event(struct perf_event *event) 5347 { 5348 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); 5349 5350 raw_spin_lock(&pel->lock); 5351 list_del_rcu(&event->sb_list); 5352 raw_spin_unlock(&pel->lock); 5353 } 5354 5355 static bool is_sb_event(struct perf_event *event) 5356 { 5357 struct perf_event_attr *attr = &event->attr; 5358 5359 if (event->parent) 5360 return false; 5361 5362 if (event->attach_state & PERF_ATTACH_TASK) 5363 return false; 5364 5365 if (attr->mmap || attr->mmap_data || attr->mmap2 || 5366 attr->comm || attr->comm_exec || 5367 attr->task || attr->ksymbol || 5368 attr->context_switch || attr->text_poke || 5369 attr->bpf_event || attr->defer_output) 5370 return true; 5371 5372 return false; 5373 } 5374 5375 static void unaccount_pmu_sb_event(struct perf_event *event) 5376 { 5377 if (is_sb_event(event)) 5378 detach_sb_event(event); 5379 } 5380 5381 #ifdef CONFIG_NO_HZ_FULL 5382 static DEFINE_SPINLOCK(nr_freq_lock); 5383 #endif 5384 5385 static void unaccount_freq_event_nohz(void) 5386 { 5387 #ifdef CONFIG_NO_HZ_FULL 5388 spin_lock(&nr_freq_lock); 5389 if (atomic_dec_and_test(&nr_freq_events)) 5390 tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS); 5391 spin_unlock(&nr_freq_lock); 5392 #endif 5393 } 5394 5395 static void unaccount_freq_event(void) 5396 { 5397 if (tick_nohz_full_enabled()) 5398 unaccount_freq_event_nohz(); 5399 else 5400 atomic_dec(&nr_freq_events); 5401 } 5402 5403 5404 static struct perf_ctx_data * 5405 alloc_perf_ctx_data(struct kmem_cache *ctx_cache, bool global, gfp_t gfp_flags) 5406 { 5407 struct perf_ctx_data *cd; 5408 5409 cd = kzalloc_obj(*cd, gfp_flags); 5410 if (!cd) 5411 return NULL; 5412 5413 cd->data = kmem_cache_zalloc(ctx_cache, gfp_flags); 5414 if (!cd->data) { 5415 kfree(cd); 5416 return NULL; 5417 } 5418 5419 cd->global = global; 5420 cd->ctx_cache = ctx_cache; 5421 refcount_set(&cd->refcount, 1); 5422 5423 return cd; 5424 } 5425 5426 static void free_perf_ctx_data(struct perf_ctx_data *cd) 5427 { 5428 kmem_cache_free(cd->ctx_cache, cd->data); 5429 kfree(cd); 5430 } 5431 5432 static void __free_perf_ctx_data_rcu(struct rcu_head *rcu_head) 5433 { 5434 struct perf_ctx_data *cd; 5435 5436 cd = container_of(rcu_head, struct perf_ctx_data, rcu_head); 5437 free_perf_ctx_data(cd); 5438 } 5439 5440 static inline void perf_free_ctx_data_rcu(struct perf_ctx_data *cd) 5441 { 5442 call_rcu(&cd->rcu_head, __free_perf_ctx_data_rcu); 5443 } 5444 5445 static int 5446 attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache, 5447 bool global, gfp_t gfp_flags) 5448 { 5449 struct perf_ctx_data *cd, *old = NULL; 5450 5451 cd = alloc_perf_ctx_data(ctx_cache, global, gfp_flags); 5452 if (!cd) 5453 return -ENOMEM; 5454 5455 for (;;) { 5456 /* Protect @old until inc_not_zero(&old->refcount) */ 5457 guard(rcu)(); 5458 5459 if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) { 5460 if (old) 5461 perf_free_ctx_data_rcu(old); 5462 /* 5463 * Above try_cmpxchg() pairs with try_cmpxchg() from 5464 * detach_task_ctx_data() such that 5465 * if we race with perf_event_exit_task(), we must 5466 * observe PF_EXITING. 5467 */ 5468 if (task->flags & PF_EXITING) { 5469 /* detach_task_ctx_data() may free it already */ 5470 if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL)) 5471 perf_free_ctx_data_rcu(cd); 5472 } 5473 return 0; 5474 } 5475 5476 if (!old) { 5477 /* 5478 * After seeing a dead @old, we raced with 5479 * removal and lost, try again to install @cd. 5480 */ 5481 continue; 5482 } 5483 5484 if (refcount_inc_not_zero(&old->refcount)) { 5485 if (global) 5486 old->global = true; 5487 free_perf_ctx_data(cd); /* unused */ 5488 return 0; 5489 } 5490 5491 /* 5492 * @old is a dead object, refcount==0 is stable, try and 5493 * replace it with @cd. 5494 */ 5495 } 5496 return 0; 5497 } 5498 5499 static void __detach_global_ctx_data(void); 5500 DEFINE_STATIC_PERCPU_RWSEM(global_ctx_data_rwsem); 5501 static refcount_t global_ctx_data_ref; 5502 5503 static int 5504 attach_global_ctx_data(struct kmem_cache *ctx_cache) 5505 { 5506 struct task_struct *g, *p; 5507 struct perf_ctx_data *cd; 5508 int ret; 5509 5510 if (refcount_inc_not_zero(&global_ctx_data_ref)) 5511 return 0; 5512 5513 guard(percpu_write)(&global_ctx_data_rwsem); 5514 if (refcount_inc_not_zero(&global_ctx_data_ref)) 5515 return 0; 5516 again: 5517 /* Allocate everything */ 5518 scoped_guard (rcu) { 5519 for_each_process_thread(g, p) { 5520 if (p->flags & PF_EXITING) 5521 continue; 5522 cd = rcu_dereference(p->perf_ctx_data); 5523 if (cd && !cd->global) { 5524 cd->global = 1; 5525 if (!refcount_inc_not_zero(&cd->refcount)) 5526 cd = NULL; 5527 } 5528 if (!cd) { 5529 /* 5530 * Try to allocate context quickly before 5531 * traversing the whole thread list again. 5532 */ 5533 if (!attach_task_ctx_data(p, ctx_cache, true, GFP_NOWAIT)) 5534 continue; 5535 get_task_struct(p); 5536 goto alloc; 5537 } 5538 } 5539 } 5540 5541 refcount_set(&global_ctx_data_ref, 1); 5542 5543 return 0; 5544 alloc: 5545 ret = attach_task_ctx_data(p, ctx_cache, true, GFP_KERNEL); 5546 put_task_struct(p); 5547 if (ret) { 5548 __detach_global_ctx_data(); 5549 return ret; 5550 } 5551 goto again; 5552 } 5553 5554 static int 5555 attach_perf_ctx_data(struct perf_event *event) 5556 { 5557 struct task_struct *task = event->hw.target; 5558 struct kmem_cache *ctx_cache = event->pmu->task_ctx_cache; 5559 int ret; 5560 5561 if (!ctx_cache) 5562 return -ENOMEM; 5563 5564 if (task) 5565 return attach_task_ctx_data(task, ctx_cache, false, GFP_KERNEL); 5566 5567 ret = attach_global_ctx_data(ctx_cache); 5568 if (ret) 5569 return ret; 5570 5571 event->attach_state |= PERF_ATTACH_GLOBAL_DATA; 5572 return 0; 5573 } 5574 5575 static void 5576 detach_task_ctx_data(struct task_struct *p) 5577 { 5578 struct perf_ctx_data *cd; 5579 5580 scoped_guard (rcu) { 5581 cd = rcu_dereference(p->perf_ctx_data); 5582 if (!cd || !refcount_dec_and_test(&cd->refcount)) 5583 return; 5584 } 5585 5586 /* 5587 * The old ctx_data may be lost because of the race. 5588 * Nothing is required to do for the case. 5589 * See attach_task_ctx_data(). 5590 */ 5591 if (try_cmpxchg((struct perf_ctx_data **)&p->perf_ctx_data, &cd, NULL)) 5592 perf_free_ctx_data_rcu(cd); 5593 } 5594 5595 static void __detach_global_ctx_data(void) 5596 { 5597 struct task_struct *g, *p; 5598 struct perf_ctx_data *cd; 5599 5600 scoped_guard (rcu) { 5601 for_each_process_thread(g, p) { 5602 cd = rcu_dereference(p->perf_ctx_data); 5603 if (cd && cd->global) { 5604 cd->global = 0; 5605 detach_task_ctx_data(p); 5606 } 5607 } 5608 } 5609 } 5610 5611 static void detach_global_ctx_data(void) 5612 { 5613 if (refcount_dec_not_one(&global_ctx_data_ref)) 5614 return; 5615 5616 guard(percpu_write)(&global_ctx_data_rwsem); 5617 if (!refcount_dec_and_test(&global_ctx_data_ref)) 5618 return; 5619 5620 /* remove everything */ 5621 __detach_global_ctx_data(); 5622 } 5623 5624 static void detach_perf_ctx_data(struct perf_event *event) 5625 { 5626 struct task_struct *task = event->hw.target; 5627 5628 event->attach_state &= ~PERF_ATTACH_TASK_DATA; 5629 5630 if (task) 5631 return detach_task_ctx_data(task); 5632 5633 if (event->attach_state & PERF_ATTACH_GLOBAL_DATA) { 5634 detach_global_ctx_data(); 5635 event->attach_state &= ~PERF_ATTACH_GLOBAL_DATA; 5636 } 5637 } 5638 5639 static void unaccount_event(struct perf_event *event) 5640 { 5641 bool dec = false; 5642 5643 if (event->parent) 5644 return; 5645 5646 if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB)) 5647 dec = true; 5648 if (event->attr.mmap || event->attr.mmap_data) 5649 atomic_dec(&nr_mmap_events); 5650 if (event->attr.build_id) 5651 atomic_dec(&nr_build_id_events); 5652 if (event->attr.comm) 5653 atomic_dec(&nr_comm_events); 5654 if (event->attr.namespaces) 5655 atomic_dec(&nr_namespaces_events); 5656 if (event->attr.cgroup) 5657 atomic_dec(&nr_cgroup_events); 5658 if (event->attr.task) 5659 atomic_dec(&nr_task_events); 5660 if (event->attr.freq) 5661 unaccount_freq_event(); 5662 if (event->attr.context_switch) { 5663 dec = true; 5664 atomic_dec(&nr_switch_events); 5665 } 5666 if (is_cgroup_event(event)) 5667 dec = true; 5668 if (has_branch_stack(event)) 5669 dec = true; 5670 if (event->attr.ksymbol) 5671 atomic_dec(&nr_ksymbol_events); 5672 if (event->attr.bpf_event) 5673 atomic_dec(&nr_bpf_events); 5674 if (event->attr.text_poke) 5675 atomic_dec(&nr_text_poke_events); 5676 5677 if (dec) { 5678 if (!atomic_add_unless(&perf_sched_count, -1, 1)) 5679 schedule_delayed_work(&perf_sched_work, HZ); 5680 } 5681 5682 unaccount_pmu_sb_event(event); 5683 } 5684 5685 static void perf_sched_delayed(struct work_struct *work) 5686 { 5687 mutex_lock(&perf_sched_mutex); 5688 if (atomic_dec_and_test(&perf_sched_count)) 5689 static_branch_disable(&perf_sched_events); 5690 mutex_unlock(&perf_sched_mutex); 5691 } 5692 5693 /* 5694 * The following implement mutual exclusion of events on "exclusive" pmus 5695 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled 5696 * at a time, so we disallow creating events that might conflict, namely: 5697 * 5698 * 1) cpu-wide events in the presence of per-task events, 5699 * 2) per-task events in the presence of cpu-wide events, 5700 * 3) two matching events on the same perf_event_context. 5701 * 5702 * The former two cases are handled in the allocation path (perf_event_alloc(), 5703 * _free_event()), the latter -- before the first perf_install_in_context(). 5704 */ 5705 static int exclusive_event_init(struct perf_event *event) 5706 { 5707 struct pmu *pmu = event->pmu; 5708 5709 if (!is_exclusive_pmu(pmu)) 5710 return 0; 5711 5712 /* 5713 * Prevent co-existence of per-task and cpu-wide events on the 5714 * same exclusive pmu. 5715 * 5716 * Negative pmu::exclusive_cnt means there are cpu-wide 5717 * events on this "exclusive" pmu, positive means there are 5718 * per-task events. 5719 * 5720 * Since this is called in perf_event_alloc() path, event::ctx 5721 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK 5722 * to mean "per-task event", because unlike other attach states it 5723 * never gets cleared. 5724 */ 5725 if (event->attach_state & PERF_ATTACH_TASK) { 5726 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt)) 5727 return -EBUSY; 5728 } else { 5729 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt)) 5730 return -EBUSY; 5731 } 5732 5733 event->attach_state |= PERF_ATTACH_EXCLUSIVE; 5734 5735 return 0; 5736 } 5737 5738 static void exclusive_event_destroy(struct perf_event *event) 5739 { 5740 struct pmu *pmu = event->pmu; 5741 5742 /* see comment in exclusive_event_init() */ 5743 if (event->attach_state & PERF_ATTACH_TASK) 5744 atomic_dec(&pmu->exclusive_cnt); 5745 else 5746 atomic_inc(&pmu->exclusive_cnt); 5747 5748 event->attach_state &= ~PERF_ATTACH_EXCLUSIVE; 5749 } 5750 5751 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2) 5752 { 5753 if ((e1->pmu == e2->pmu) && 5754 (e1->cpu == e2->cpu || 5755 e1->cpu == -1 || 5756 e2->cpu == -1)) 5757 return true; 5758 return false; 5759 } 5760 5761 static bool exclusive_event_installable(struct perf_event *event, 5762 struct perf_event_context *ctx) 5763 { 5764 struct perf_event *iter_event; 5765 struct pmu *pmu = event->pmu; 5766 5767 lockdep_assert_held(&ctx->mutex); 5768 5769 if (!is_exclusive_pmu(pmu)) 5770 return true; 5771 5772 list_for_each_entry(iter_event, &ctx->event_list, event_entry) { 5773 if (exclusive_event_match(iter_event, event)) 5774 return false; 5775 } 5776 5777 return true; 5778 } 5779 5780 static void perf_free_addr_filters(struct perf_event *event); 5781 5782 /* vs perf_event_alloc() error */ 5783 static void __free_event(struct perf_event *event) 5784 { 5785 struct pmu *pmu = event->pmu; 5786 5787 security_perf_event_free(event); 5788 5789 if (event->attach_state & PERF_ATTACH_CALLCHAIN) 5790 put_callchain_buffers(); 5791 5792 if (event->attach_state & PERF_ATTACH_EXCLUSIVE) 5793 exclusive_event_destroy(event); 5794 5795 if (is_cgroup_event(event)) 5796 perf_detach_cgroup(event); 5797 5798 if (event->attach_state & PERF_ATTACH_TASK_DATA) 5799 detach_perf_ctx_data(event); 5800 5801 if (event->destroy) 5802 event->destroy(event); 5803 5804 /* 5805 * Must be after ->destroy(), due to uprobe_perf_close() using 5806 * hw.target. 5807 */ 5808 if (event->hw.target) 5809 put_task_struct(event->hw.target); 5810 5811 if (event->pmu_ctx) { 5812 /* 5813 * put_pmu_ctx() needs an event->ctx reference, because of 5814 * epc->ctx. 5815 */ 5816 WARN_ON_ONCE(!pmu); 5817 WARN_ON_ONCE(!event->ctx); 5818 WARN_ON_ONCE(event->pmu_ctx->ctx != event->ctx); 5819 put_pmu_ctx(event->pmu_ctx); 5820 } 5821 5822 /* 5823 * perf_event_free_task() relies on put_ctx() being 'last', in 5824 * particular all task references must be cleaned up. 5825 */ 5826 if (event->ctx) 5827 put_ctx(event->ctx); 5828 5829 if (pmu) { 5830 module_put(pmu->module); 5831 scoped_guard (spinlock, &pmu->events_lock) { 5832 list_del(&event->pmu_list); 5833 wake_up_var(pmu); 5834 } 5835 } 5836 5837 call_rcu(&event->rcu_head, free_event_rcu); 5838 } 5839 5840 static void mediated_pmu_unaccount_event(struct perf_event *event); 5841 5842 DEFINE_FREE(__free_event, struct perf_event *, if (_T) __free_event(_T)) 5843 5844 /* vs perf_event_alloc() success */ 5845 static void _free_event(struct perf_event *event) 5846 { 5847 irq_work_sync(&event->pending_irq); 5848 irq_work_sync(&event->pending_disable_irq); 5849 5850 unaccount_event(event); 5851 mediated_pmu_unaccount_event(event); 5852 5853 if (event->rb) { 5854 /* 5855 * Can happen when we close an event with re-directed output. 5856 * 5857 * Since we have a 0 refcount, perf_mmap_close() will skip 5858 * over us; possibly making our ring_buffer_put() the last. 5859 */ 5860 mutex_lock(&event->mmap_mutex); 5861 ring_buffer_attach(event, NULL); 5862 mutex_unlock(&event->mmap_mutex); 5863 } 5864 5865 perf_event_free_bpf_prog(event); 5866 perf_free_addr_filters(event); 5867 5868 __free_event(event); 5869 } 5870 5871 /* 5872 * Used to free events which have a known refcount of 1, such as in error paths 5873 * of inherited events. 5874 */ 5875 static void free_event(struct perf_event *event) 5876 { 5877 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, 5878 "unexpected event refcount: %ld; ptr=%p\n", 5879 atomic_long_read(&event->refcount), event)) { 5880 /* leak to avoid use-after-free */ 5881 return; 5882 } 5883 5884 _free_event(event); 5885 } 5886 5887 /* 5888 * Remove user event from the owner task. 5889 */ 5890 static void perf_remove_from_owner(struct perf_event *event) 5891 { 5892 struct task_struct *owner; 5893 5894 rcu_read_lock(); 5895 /* 5896 * Matches the smp_store_release() in perf_event_exit_task(). If we 5897 * observe !owner it means the list deletion is complete and we can 5898 * indeed free this event, otherwise we need to serialize on 5899 * owner->perf_event_mutex. 5900 */ 5901 owner = READ_ONCE(event->owner); 5902 if (owner) { 5903 /* 5904 * Since delayed_put_task_struct() also drops the last 5905 * task reference we can safely take a new reference 5906 * while holding the rcu_read_lock(). 5907 */ 5908 get_task_struct(owner); 5909 } 5910 rcu_read_unlock(); 5911 5912 if (owner) { 5913 /* 5914 * If we're here through perf_event_exit_task() we're already 5915 * holding ctx->mutex which would be an inversion wrt. the 5916 * normal lock order. 5917 * 5918 * However we can safely take this lock because its the child 5919 * ctx->mutex. 5920 */ 5921 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); 5922 5923 /* 5924 * We have to re-check the event->owner field, if it is cleared 5925 * we raced with perf_event_exit_task(), acquiring the mutex 5926 * ensured they're done, and we can proceed with freeing the 5927 * event. 5928 */ 5929 if (event->owner) { 5930 list_del_init(&event->owner_entry); 5931 smp_store_release(&event->owner, NULL); 5932 } 5933 mutex_unlock(&owner->perf_event_mutex); 5934 put_task_struct(owner); 5935 } 5936 } 5937 5938 static void put_event(struct perf_event *event) 5939 { 5940 struct perf_event *parent; 5941 5942 if (!atomic_long_dec_and_test(&event->refcount)) 5943 return; 5944 5945 parent = event->parent; 5946 _free_event(event); 5947 5948 /* Matches the refcount bump in inherit_event() */ 5949 if (parent) 5950 put_event(parent); 5951 } 5952 5953 /* 5954 * Kill an event dead; while event:refcount will preserve the event 5955 * object, it will not preserve its functionality. Once the last 'user' 5956 * gives up the object, we'll destroy the thing. 5957 */ 5958 int perf_event_release_kernel(struct perf_event *event) 5959 { 5960 struct perf_event_context *ctx = event->ctx; 5961 struct perf_event *child, *tmp; 5962 5963 /* 5964 * If we got here through err_alloc: free_event(event); we will not 5965 * have attached to a context yet. 5966 */ 5967 if (!ctx) { 5968 WARN_ON_ONCE(event->attach_state & 5969 (PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP)); 5970 goto no_ctx; 5971 } 5972 5973 if (!is_kernel_event(event)) 5974 perf_remove_from_owner(event); 5975 5976 ctx = perf_event_ctx_lock(event); 5977 WARN_ON_ONCE(ctx->parent_ctx); 5978 5979 /* 5980 * Mark this event as STATE_DEAD, there is no external reference to it 5981 * anymore. 5982 * 5983 * Anybody acquiring event->child_mutex after the below loop _must_ 5984 * also see this, most importantly inherit_event() which will avoid 5985 * placing more children on the list. 5986 * 5987 * Thus this guarantees that we will in fact observe and kill _ALL_ 5988 * child events. 5989 */ 5990 if (event->state > PERF_EVENT_STATE_REVOKED) { 5991 perf_remove_from_context(event, DETACH_GROUP|DETACH_DEAD); 5992 } else { 5993 event->state = PERF_EVENT_STATE_DEAD; 5994 } 5995 5996 perf_event_ctx_unlock(event, ctx); 5997 5998 again: 5999 mutex_lock(&event->child_mutex); 6000 list_for_each_entry(child, &event->child_list, child_list) { 6001 /* 6002 * Cannot change, child events are not migrated, see the 6003 * comment with perf_event_ctx_lock_nested(). 6004 */ 6005 ctx = READ_ONCE(child->ctx); 6006 /* 6007 * Since child_mutex nests inside ctx::mutex, we must jump 6008 * through hoops. We start by grabbing a reference on the ctx. 6009 * 6010 * Since the event cannot get freed while we hold the 6011 * child_mutex, the context must also exist and have a !0 6012 * reference count. 6013 */ 6014 get_ctx(ctx); 6015 6016 /* 6017 * Now that we have a ctx ref, we can drop child_mutex, and 6018 * acquire ctx::mutex without fear of it going away. Then we 6019 * can re-acquire child_mutex. 6020 */ 6021 mutex_unlock(&event->child_mutex); 6022 mutex_lock(&ctx->mutex); 6023 mutex_lock(&event->child_mutex); 6024 6025 /* 6026 * Now that we hold ctx::mutex and child_mutex, revalidate our 6027 * state, if child is still the first entry, it didn't get freed 6028 * and we can continue doing so. 6029 */ 6030 tmp = list_first_entry_or_null(&event->child_list, 6031 struct perf_event, child_list); 6032 if (tmp == child) { 6033 perf_remove_from_context(child, DETACH_GROUP | DETACH_CHILD); 6034 } else { 6035 child = NULL; 6036 } 6037 6038 mutex_unlock(&event->child_mutex); 6039 mutex_unlock(&ctx->mutex); 6040 6041 if (child) { 6042 /* Last reference unless ->pending_task work is pending */ 6043 put_event(child); 6044 } 6045 put_ctx(ctx); 6046 6047 goto again; 6048 } 6049 mutex_unlock(&event->child_mutex); 6050 6051 no_ctx: 6052 /* 6053 * Last reference unless ->pending_task work is pending on this event 6054 * or any of its children. 6055 */ 6056 put_event(event); 6057 return 0; 6058 } 6059 EXPORT_SYMBOL_GPL(perf_event_release_kernel); 6060 6061 /* 6062 * Called when the last reference to the file is gone. 6063 */ 6064 static int perf_release(struct inode *inode, struct file *file) 6065 { 6066 perf_event_release_kernel(file->private_data); 6067 return 0; 6068 } 6069 6070 static u64 __perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 6071 { 6072 struct perf_event *child; 6073 u64 total = 0; 6074 6075 *enabled = 0; 6076 *running = 0; 6077 6078 mutex_lock(&event->child_mutex); 6079 6080 (void)perf_event_read(event, false); 6081 total += perf_event_count(event, false); 6082 6083 *enabled += event->total_time_enabled + 6084 atomic64_read(&event->child_total_time_enabled); 6085 *running += event->total_time_running + 6086 atomic64_read(&event->child_total_time_running); 6087 6088 list_for_each_entry(child, &event->child_list, child_list) { 6089 (void)perf_event_read(child, false); 6090 total += perf_event_count(child, false); 6091 *enabled += child->total_time_enabled; 6092 *running += child->total_time_running; 6093 } 6094 mutex_unlock(&event->child_mutex); 6095 6096 return total; 6097 } 6098 6099 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 6100 { 6101 struct perf_event_context *ctx; 6102 u64 count; 6103 6104 ctx = perf_event_ctx_lock(event); 6105 count = __perf_event_read_value(event, enabled, running); 6106 perf_event_ctx_unlock(event, ctx); 6107 6108 return count; 6109 } 6110 EXPORT_SYMBOL_GPL(perf_event_read_value); 6111 6112 static int __perf_read_group_add(struct perf_event *leader, 6113 u64 read_format, u64 *values) 6114 { 6115 struct perf_event_context *ctx = leader->ctx; 6116 struct perf_event *sub, *parent; 6117 unsigned long flags; 6118 int n = 1; /* skip @nr */ 6119 int ret; 6120 6121 ret = perf_event_read(leader, true); 6122 if (ret) 6123 return ret; 6124 6125 raw_spin_lock_irqsave(&ctx->lock, flags); 6126 /* 6127 * Verify the grouping between the parent and child (inherited) 6128 * events is still in tact. 6129 * 6130 * Specifically: 6131 * - leader->ctx->lock pins leader->sibling_list 6132 * - parent->child_mutex pins parent->child_list 6133 * - parent->ctx->mutex pins parent->sibling_list 6134 * 6135 * Because parent->ctx != leader->ctx (and child_list nests inside 6136 * ctx->mutex), group destruction is not atomic between children, also 6137 * see perf_event_release_kernel(). Additionally, parent can grow the 6138 * group. 6139 * 6140 * Therefore it is possible to have parent and child groups in a 6141 * different configuration and summing over such a beast makes no sense 6142 * what so ever. 6143 * 6144 * Reject this. 6145 */ 6146 parent = leader->parent; 6147 if (parent && 6148 (parent->group_generation != leader->group_generation || 6149 parent->nr_siblings != leader->nr_siblings)) { 6150 ret = -ECHILD; 6151 goto unlock; 6152 } 6153 6154 /* 6155 * Since we co-schedule groups, {enabled,running} times of siblings 6156 * will be identical to those of the leader, so we only publish one 6157 * set. 6158 */ 6159 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 6160 values[n++] += leader->total_time_enabled + 6161 atomic64_read(&leader->child_total_time_enabled); 6162 } 6163 6164 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 6165 values[n++] += leader->total_time_running + 6166 atomic64_read(&leader->child_total_time_running); 6167 } 6168 6169 /* 6170 * Write {count,id} tuples for every sibling. 6171 */ 6172 values[n++] += perf_event_count(leader, false); 6173 if (read_format & PERF_FORMAT_ID) 6174 values[n++] = primary_event_id(leader); 6175 if (read_format & PERF_FORMAT_LOST) 6176 values[n++] = atomic64_read(&leader->lost_samples); 6177 6178 for_each_sibling_event(sub, leader) { 6179 values[n++] += perf_event_count(sub, false); 6180 if (read_format & PERF_FORMAT_ID) 6181 values[n++] = primary_event_id(sub); 6182 if (read_format & PERF_FORMAT_LOST) 6183 values[n++] = atomic64_read(&sub->lost_samples); 6184 } 6185 6186 unlock: 6187 raw_spin_unlock_irqrestore(&ctx->lock, flags); 6188 return ret; 6189 } 6190 6191 static int perf_read_group(struct perf_event *event, 6192 u64 read_format, char __user *buf) 6193 { 6194 struct perf_event *leader = event->group_leader, *child; 6195 struct perf_event_context *ctx = leader->ctx; 6196 int ret; 6197 u64 *values; 6198 6199 lockdep_assert_held(&ctx->mutex); 6200 6201 values = kzalloc(event->read_size, GFP_KERNEL); 6202 if (!values) 6203 return -ENOMEM; 6204 6205 values[0] = 1 + leader->nr_siblings; 6206 6207 mutex_lock(&leader->child_mutex); 6208 6209 ret = __perf_read_group_add(leader, read_format, values); 6210 if (ret) 6211 goto unlock; 6212 6213 list_for_each_entry(child, &leader->child_list, child_list) { 6214 ret = __perf_read_group_add(child, read_format, values); 6215 if (ret) 6216 goto unlock; 6217 } 6218 6219 mutex_unlock(&leader->child_mutex); 6220 6221 ret = event->read_size; 6222 if (copy_to_user(buf, values, event->read_size)) 6223 ret = -EFAULT; 6224 goto out; 6225 6226 unlock: 6227 mutex_unlock(&leader->child_mutex); 6228 out: 6229 kfree(values); 6230 return ret; 6231 } 6232 6233 static int perf_read_one(struct perf_event *event, 6234 u64 read_format, char __user *buf) 6235 { 6236 u64 enabled, running; 6237 u64 values[5]; 6238 int n = 0; 6239 6240 values[n++] = __perf_event_read_value(event, &enabled, &running); 6241 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 6242 values[n++] = enabled; 6243 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 6244 values[n++] = running; 6245 if (read_format & PERF_FORMAT_ID) 6246 values[n++] = primary_event_id(event); 6247 if (read_format & PERF_FORMAT_LOST) 6248 values[n++] = atomic64_read(&event->lost_samples); 6249 6250 if (copy_to_user(buf, values, n * sizeof(u64))) 6251 return -EFAULT; 6252 6253 return n * sizeof(u64); 6254 } 6255 6256 static bool is_event_hup(struct perf_event *event) 6257 { 6258 bool no_children; 6259 6260 if (event->state > PERF_EVENT_STATE_EXIT) 6261 return false; 6262 6263 mutex_lock(&event->child_mutex); 6264 no_children = list_empty(&event->child_list); 6265 mutex_unlock(&event->child_mutex); 6266 return no_children; 6267 } 6268 6269 /* 6270 * Read the performance event - simple non blocking version for now 6271 */ 6272 static ssize_t 6273 __perf_read(struct perf_event *event, char __user *buf, size_t count) 6274 { 6275 u64 read_format = event->attr.read_format; 6276 int ret; 6277 6278 /* 6279 * Return end-of-file for a read on an event that is in 6280 * error state (i.e. because it was pinned but it couldn't be 6281 * scheduled on to the CPU at some point). 6282 */ 6283 if (event->state == PERF_EVENT_STATE_ERROR) 6284 return 0; 6285 6286 if (count < event->read_size) 6287 return -ENOSPC; 6288 6289 WARN_ON_ONCE(event->ctx->parent_ctx); 6290 if (read_format & PERF_FORMAT_GROUP) 6291 ret = perf_read_group(event, read_format, buf); 6292 else 6293 ret = perf_read_one(event, read_format, buf); 6294 6295 return ret; 6296 } 6297 6298 static ssize_t 6299 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 6300 { 6301 struct perf_event *event = file->private_data; 6302 struct perf_event_context *ctx; 6303 int ret; 6304 6305 ret = security_perf_event_read(event); 6306 if (ret) 6307 return ret; 6308 6309 ctx = perf_event_ctx_lock(event); 6310 ret = __perf_read(event, buf, count); 6311 perf_event_ctx_unlock(event, ctx); 6312 6313 return ret; 6314 } 6315 6316 static __poll_t perf_poll(struct file *file, poll_table *wait) 6317 { 6318 struct perf_event *event = file->private_data; 6319 struct perf_buffer *rb; 6320 __poll_t events = EPOLLHUP; 6321 6322 if (event->state <= PERF_EVENT_STATE_REVOKED) 6323 return EPOLLERR; 6324 6325 poll_wait(file, &event->waitq, wait); 6326 6327 if (event->state <= PERF_EVENT_STATE_REVOKED) 6328 return EPOLLERR; 6329 6330 if (is_event_hup(event)) 6331 return events; 6332 6333 if (unlikely(READ_ONCE(event->state) == PERF_EVENT_STATE_ERROR && 6334 event->attr.pinned)) 6335 return EPOLLERR; 6336 6337 /* 6338 * Pin the event->rb by taking event->mmap_mutex; otherwise 6339 * perf_event_set_output() can swizzle our rb and make us miss wakeups. 6340 */ 6341 mutex_lock(&event->mmap_mutex); 6342 rb = event->rb; 6343 if (rb) 6344 events = atomic_xchg(&rb->poll, 0); 6345 mutex_unlock(&event->mmap_mutex); 6346 return events; 6347 } 6348 6349 static void _perf_event_reset(struct perf_event *event) 6350 { 6351 (void)perf_event_read(event, false); 6352 local64_set(&event->count, 0); 6353 perf_event_update_userpage(event); 6354 } 6355 6356 /* Assume it's not an event with inherit set. */ 6357 u64 perf_event_pause(struct perf_event *event, bool reset) 6358 { 6359 struct perf_event_context *ctx; 6360 u64 count; 6361 6362 ctx = perf_event_ctx_lock(event); 6363 WARN_ON_ONCE(event->attr.inherit); 6364 _perf_event_disable(event); 6365 count = local64_read(&event->count); 6366 if (reset) 6367 local64_set(&event->count, 0); 6368 perf_event_ctx_unlock(event, ctx); 6369 6370 return count; 6371 } 6372 EXPORT_SYMBOL_GPL(perf_event_pause); 6373 6374 #ifdef CONFIG_PERF_GUEST_MEDIATED_PMU 6375 static atomic_t nr_include_guest_events __read_mostly; 6376 6377 static atomic_t nr_mediated_pmu_vms __read_mostly; 6378 static DEFINE_MUTEX(perf_mediated_pmu_mutex); 6379 6380 /* !exclude_guest event of PMU with PERF_PMU_CAP_MEDIATED_VPMU */ 6381 static inline bool is_include_guest_event(struct perf_event *event) 6382 { 6383 if (!event->pmu) 6384 return false; 6385 6386 if ((event->pmu->capabilities & PERF_PMU_CAP_MEDIATED_VPMU) && 6387 !event->attr.exclude_guest) 6388 return true; 6389 6390 return false; 6391 } 6392 6393 static int mediated_pmu_account_event(struct perf_event *event) 6394 { 6395 if (!is_include_guest_event(event)) 6396 return 0; 6397 6398 if (atomic_inc_not_zero(&nr_include_guest_events)) 6399 return 0; 6400 6401 guard(mutex)(&perf_mediated_pmu_mutex); 6402 if (atomic_read(&nr_mediated_pmu_vms)) 6403 return -EOPNOTSUPP; 6404 6405 atomic_inc(&nr_include_guest_events); 6406 return 0; 6407 } 6408 6409 static void mediated_pmu_unaccount_event(struct perf_event *event) 6410 { 6411 if (!is_include_guest_event(event)) 6412 return; 6413 6414 if (WARN_ON_ONCE(!atomic_read(&nr_include_guest_events))) 6415 return; 6416 6417 atomic_dec(&nr_include_guest_events); 6418 } 6419 6420 /* 6421 * Currently invoked at VM creation to 6422 * - Check whether there are existing !exclude_guest events of PMU with 6423 * PERF_PMU_CAP_MEDIATED_VPMU 6424 * - Set nr_mediated_pmu_vms to prevent !exclude_guest event creation on 6425 * PMUs with PERF_PMU_CAP_MEDIATED_VPMU 6426 * 6427 * No impact for the PMU without PERF_PMU_CAP_MEDIATED_VPMU. The perf 6428 * still owns all the PMU resources. 6429 */ 6430 int perf_create_mediated_pmu(void) 6431 { 6432 if (atomic_inc_not_zero(&nr_mediated_pmu_vms)) 6433 return 0; 6434 6435 guard(mutex)(&perf_mediated_pmu_mutex); 6436 if (atomic_read(&nr_include_guest_events)) 6437 return -EBUSY; 6438 6439 atomic_inc(&nr_mediated_pmu_vms); 6440 return 0; 6441 } 6442 EXPORT_SYMBOL_FOR_KVM(perf_create_mediated_pmu); 6443 6444 void perf_release_mediated_pmu(void) 6445 { 6446 if (WARN_ON_ONCE(!atomic_read(&nr_mediated_pmu_vms))) 6447 return; 6448 6449 atomic_dec(&nr_mediated_pmu_vms); 6450 } 6451 EXPORT_SYMBOL_FOR_KVM(perf_release_mediated_pmu); 6452 6453 /* When loading a guest's mediated PMU, schedule out all exclude_guest events. */ 6454 void perf_load_guest_context(void) 6455 { 6456 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 6457 6458 lockdep_assert_irqs_disabled(); 6459 6460 guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx); 6461 6462 if (WARN_ON_ONCE(__this_cpu_read(guest_ctx_loaded))) 6463 return; 6464 6465 perf_ctx_disable(&cpuctx->ctx, EVENT_GUEST); 6466 ctx_sched_out(&cpuctx->ctx, NULL, EVENT_GUEST); 6467 if (cpuctx->task_ctx) { 6468 perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST); 6469 task_ctx_sched_out(cpuctx->task_ctx, NULL, EVENT_GUEST); 6470 } 6471 6472 perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST); 6473 if (cpuctx->task_ctx) 6474 perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST); 6475 6476 __this_cpu_write(guest_ctx_loaded, true); 6477 } 6478 EXPORT_SYMBOL_GPL(perf_load_guest_context); 6479 6480 void perf_put_guest_context(void) 6481 { 6482 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 6483 6484 lockdep_assert_irqs_disabled(); 6485 6486 guard(perf_ctx_lock)(cpuctx, cpuctx->task_ctx); 6487 6488 if (WARN_ON_ONCE(!__this_cpu_read(guest_ctx_loaded))) 6489 return; 6490 6491 perf_ctx_disable(&cpuctx->ctx, EVENT_GUEST); 6492 if (cpuctx->task_ctx) 6493 perf_ctx_disable(cpuctx->task_ctx, EVENT_GUEST); 6494 6495 perf_event_sched_in(cpuctx, cpuctx->task_ctx, NULL, EVENT_GUEST); 6496 6497 if (cpuctx->task_ctx) 6498 perf_ctx_enable(cpuctx->task_ctx, EVENT_GUEST); 6499 perf_ctx_enable(&cpuctx->ctx, EVENT_GUEST); 6500 6501 __this_cpu_write(guest_ctx_loaded, false); 6502 } 6503 EXPORT_SYMBOL_GPL(perf_put_guest_context); 6504 #else 6505 static int mediated_pmu_account_event(struct perf_event *event) { return 0; } 6506 static void mediated_pmu_unaccount_event(struct perf_event *event) {} 6507 #endif 6508 6509 /* 6510 * Holding the top-level event's child_mutex means that any 6511 * descendant process that has inherited this event will block 6512 * in perf_event_exit_event() if it goes to exit, thus satisfying the 6513 * task existence requirements of perf_event_enable/disable. 6514 */ 6515 static void perf_event_for_each_child(struct perf_event *event, 6516 void (*func)(struct perf_event *)) 6517 { 6518 struct perf_event *child; 6519 6520 WARN_ON_ONCE(event->ctx->parent_ctx); 6521 6522 mutex_lock(&event->child_mutex); 6523 func(event); 6524 list_for_each_entry(child, &event->child_list, child_list) 6525 func(child); 6526 mutex_unlock(&event->child_mutex); 6527 } 6528 6529 static void perf_event_for_each(struct perf_event *event, 6530 void (*func)(struct perf_event *)) 6531 { 6532 struct perf_event_context *ctx = event->ctx; 6533 struct perf_event *sibling; 6534 6535 lockdep_assert_held(&ctx->mutex); 6536 6537 event = event->group_leader; 6538 6539 perf_event_for_each_child(event, func); 6540 for_each_sibling_event(sibling, event) 6541 perf_event_for_each_child(sibling, func); 6542 } 6543 6544 static void __perf_event_period(struct perf_event *event, 6545 struct perf_cpu_context *cpuctx, 6546 struct perf_event_context *ctx, 6547 void *info) 6548 { 6549 u64 value = *((u64 *)info); 6550 bool active; 6551 6552 if (event->attr.freq) { 6553 event->attr.sample_freq = value; 6554 } else { 6555 event->attr.sample_period = value; 6556 event->hw.sample_period = value; 6557 } 6558 6559 active = (event->state == PERF_EVENT_STATE_ACTIVE); 6560 if (active) { 6561 perf_pmu_disable(event->pmu); 6562 event->pmu->stop(event, PERF_EF_UPDATE); 6563 } 6564 6565 local64_set(&event->hw.period_left, 0); 6566 6567 if (active) { 6568 event->pmu->start(event, PERF_EF_RELOAD); 6569 /* 6570 * Once the period is force-reset, the event starts immediately. 6571 * But the event/group could be throttled. Unthrottle the 6572 * event/group now to avoid the next tick trying to unthrottle 6573 * while we already re-started the event/group. 6574 */ 6575 if (event->hw.interrupts == MAX_INTERRUPTS) 6576 perf_event_unthrottle_group(event, true); 6577 perf_pmu_enable(event->pmu); 6578 } 6579 } 6580 6581 static int perf_event_check_period(struct perf_event *event, u64 value) 6582 { 6583 return event->pmu->check_period(event, value); 6584 } 6585 6586 static int _perf_event_period(struct perf_event *event, u64 value) 6587 { 6588 if (!is_sampling_event(event)) 6589 return -EINVAL; 6590 6591 if (!value) 6592 return -EINVAL; 6593 6594 if (event->attr.freq) { 6595 if (value > sysctl_perf_event_sample_rate) 6596 return -EINVAL; 6597 } else { 6598 if (perf_event_check_period(event, value)) 6599 return -EINVAL; 6600 if (value & (1ULL << 63)) 6601 return -EINVAL; 6602 } 6603 6604 event_function_call(event, __perf_event_period, &value); 6605 6606 return 0; 6607 } 6608 6609 int perf_event_period(struct perf_event *event, u64 value) 6610 { 6611 struct perf_event_context *ctx; 6612 int ret; 6613 6614 ctx = perf_event_ctx_lock(event); 6615 ret = _perf_event_period(event, value); 6616 perf_event_ctx_unlock(event, ctx); 6617 6618 return ret; 6619 } 6620 EXPORT_SYMBOL_GPL(perf_event_period); 6621 6622 static const struct file_operations perf_fops; 6623 6624 static inline bool is_perf_file(struct fd f) 6625 { 6626 return !fd_empty(f) && fd_file(f)->f_op == &perf_fops; 6627 } 6628 6629 static int perf_event_set_output(struct perf_event *event, 6630 struct perf_event *output_event); 6631 static int perf_event_set_filter(struct perf_event *event, void __user *arg); 6632 static int perf_copy_attr(struct perf_event_attr __user *uattr, 6633 struct perf_event_attr *attr); 6634 static int __perf_event_set_bpf_prog(struct perf_event *event, 6635 struct bpf_prog *prog, 6636 u64 bpf_cookie); 6637 6638 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg) 6639 { 6640 void (*func)(struct perf_event *); 6641 u32 flags = arg; 6642 6643 if (event->state <= PERF_EVENT_STATE_REVOKED) 6644 return -ENODEV; 6645 6646 switch (cmd) { 6647 case PERF_EVENT_IOC_ENABLE: 6648 func = _perf_event_enable; 6649 break; 6650 case PERF_EVENT_IOC_DISABLE: 6651 func = _perf_event_disable; 6652 break; 6653 case PERF_EVENT_IOC_RESET: 6654 func = _perf_event_reset; 6655 break; 6656 6657 case PERF_EVENT_IOC_REFRESH: 6658 return _perf_event_refresh(event, arg); 6659 6660 case PERF_EVENT_IOC_PERIOD: 6661 { 6662 u64 value; 6663 6664 if (copy_from_user(&value, (u64 __user *)arg, sizeof(value))) 6665 return -EFAULT; 6666 6667 return _perf_event_period(event, value); 6668 } 6669 case PERF_EVENT_IOC_ID: 6670 { 6671 u64 id = primary_event_id(event); 6672 6673 if (copy_to_user((void __user *)arg, &id, sizeof(id))) 6674 return -EFAULT; 6675 return 0; 6676 } 6677 6678 case PERF_EVENT_IOC_SET_OUTPUT: 6679 { 6680 CLASS(fd, output)(arg); // arg == -1 => empty 6681 struct perf_event *output_event = NULL; 6682 if (arg != -1) { 6683 if (!is_perf_file(output)) 6684 return -EBADF; 6685 output_event = fd_file(output)->private_data; 6686 } 6687 return perf_event_set_output(event, output_event); 6688 } 6689 6690 case PERF_EVENT_IOC_SET_FILTER: 6691 return perf_event_set_filter(event, (void __user *)arg); 6692 6693 case PERF_EVENT_IOC_SET_BPF: 6694 { 6695 struct bpf_prog *prog; 6696 int err; 6697 6698 prog = bpf_prog_get(arg); 6699 if (IS_ERR(prog)) 6700 return PTR_ERR(prog); 6701 6702 err = __perf_event_set_bpf_prog(event, prog, 0); 6703 if (err) { 6704 bpf_prog_put(prog); 6705 return err; 6706 } 6707 6708 return 0; 6709 } 6710 6711 case PERF_EVENT_IOC_PAUSE_OUTPUT: { 6712 struct perf_buffer *rb; 6713 6714 rcu_read_lock(); 6715 rb = rcu_dereference(event->rb); 6716 if (!rb || !rb->nr_pages) { 6717 rcu_read_unlock(); 6718 return -EINVAL; 6719 } 6720 rb_toggle_paused(rb, !!arg); 6721 rcu_read_unlock(); 6722 return 0; 6723 } 6724 6725 case PERF_EVENT_IOC_QUERY_BPF: 6726 return perf_event_query_prog_array(event, (void __user *)arg); 6727 6728 case PERF_EVENT_IOC_MODIFY_ATTRIBUTES: { 6729 struct perf_event_attr new_attr; 6730 int err = perf_copy_attr((struct perf_event_attr __user *)arg, 6731 &new_attr); 6732 6733 if (err) 6734 return err; 6735 6736 return perf_event_modify_attr(event, &new_attr); 6737 } 6738 default: 6739 return -ENOTTY; 6740 } 6741 6742 if (flags & PERF_IOC_FLAG_GROUP) 6743 perf_event_for_each(event, func); 6744 else 6745 perf_event_for_each_child(event, func); 6746 6747 return 0; 6748 } 6749 6750 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 6751 { 6752 struct perf_event *event = file->private_data; 6753 struct perf_event_context *ctx; 6754 long ret; 6755 6756 /* Treat ioctl like writes as it is likely a mutating operation. */ 6757 ret = security_perf_event_write(event); 6758 if (ret) 6759 return ret; 6760 6761 ctx = perf_event_ctx_lock(event); 6762 ret = _perf_ioctl(event, cmd, arg); 6763 perf_event_ctx_unlock(event, ctx); 6764 6765 return ret; 6766 } 6767 6768 #ifdef CONFIG_COMPAT 6769 static long perf_compat_ioctl(struct file *file, unsigned int cmd, 6770 unsigned long arg) 6771 { 6772 switch (_IOC_NR(cmd)) { 6773 case _IOC_NR(PERF_EVENT_IOC_SET_FILTER): 6774 case _IOC_NR(PERF_EVENT_IOC_ID): 6775 case _IOC_NR(PERF_EVENT_IOC_QUERY_BPF): 6776 case _IOC_NR(PERF_EVENT_IOC_MODIFY_ATTRIBUTES): 6777 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ 6778 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) { 6779 cmd &= ~IOCSIZE_MASK; 6780 cmd |= sizeof(void *) << IOCSIZE_SHIFT; 6781 } 6782 break; 6783 } 6784 return perf_ioctl(file, cmd, arg); 6785 } 6786 #else 6787 # define perf_compat_ioctl NULL 6788 #endif 6789 6790 int perf_event_task_enable(void) 6791 { 6792 struct perf_event_context *ctx; 6793 struct perf_event *event; 6794 6795 mutex_lock(¤t->perf_event_mutex); 6796 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 6797 ctx = perf_event_ctx_lock(event); 6798 perf_event_for_each_child(event, _perf_event_enable); 6799 perf_event_ctx_unlock(event, ctx); 6800 } 6801 mutex_unlock(¤t->perf_event_mutex); 6802 6803 return 0; 6804 } 6805 6806 int perf_event_task_disable(void) 6807 { 6808 struct perf_event_context *ctx; 6809 struct perf_event *event; 6810 6811 mutex_lock(¤t->perf_event_mutex); 6812 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 6813 ctx = perf_event_ctx_lock(event); 6814 perf_event_for_each_child(event, _perf_event_disable); 6815 perf_event_ctx_unlock(event, ctx); 6816 } 6817 mutex_unlock(¤t->perf_event_mutex); 6818 6819 return 0; 6820 } 6821 6822 static int perf_event_index(struct perf_event *event) 6823 { 6824 if (event->hw.state & PERF_HES_STOPPED) 6825 return 0; 6826 6827 if (event->state != PERF_EVENT_STATE_ACTIVE) 6828 return 0; 6829 6830 return event->pmu->event_idx(event); 6831 } 6832 6833 static void perf_event_init_userpage(struct perf_event *event) 6834 { 6835 struct perf_event_mmap_page *userpg; 6836 struct perf_buffer *rb; 6837 6838 rcu_read_lock(); 6839 rb = rcu_dereference(event->rb); 6840 if (!rb) 6841 goto unlock; 6842 6843 userpg = rb->user_page; 6844 6845 /* Allow new userspace to detect that bit 0 is deprecated */ 6846 userpg->cap_bit0_is_deprecated = 1; 6847 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); 6848 userpg->data_offset = PAGE_SIZE; 6849 userpg->data_size = perf_data_size(rb); 6850 6851 unlock: 6852 rcu_read_unlock(); 6853 } 6854 6855 void __weak arch_perf_update_userpage( 6856 struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now) 6857 { 6858 } 6859 6860 /* 6861 * Callers need to ensure there can be no nesting of this function, otherwise 6862 * the seqlock logic goes bad. We can not serialize this because the arch 6863 * code calls this from NMI context. 6864 */ 6865 void perf_event_update_userpage(struct perf_event *event) 6866 { 6867 struct perf_event_mmap_page *userpg; 6868 struct perf_buffer *rb; 6869 u64 enabled, running, now; 6870 6871 rcu_read_lock(); 6872 rb = rcu_dereference(event->rb); 6873 if (!rb) 6874 goto unlock; 6875 6876 /* 6877 * Disable preemption to guarantee consistent time stamps are stored to 6878 * the user page. 6879 */ 6880 preempt_disable(); 6881 6882 /* 6883 * Compute total_time_enabled, total_time_running based on snapshot 6884 * values taken when the event was last scheduled in. 6885 * 6886 * We cannot simply call update_context_time() because doing so would 6887 * lead to deadlock when called from NMI context. 6888 */ 6889 calc_timer_values(event, &now, &enabled, &running); 6890 6891 userpg = rb->user_page; 6892 6893 ++userpg->lock; 6894 barrier(); 6895 userpg->index = perf_event_index(event); 6896 userpg->offset = perf_event_count(event, false); 6897 if (userpg->index) 6898 userpg->offset -= local64_read(&event->hw.prev_count); 6899 6900 userpg->time_enabled = enabled + 6901 atomic64_read(&event->child_total_time_enabled); 6902 6903 userpg->time_running = running + 6904 atomic64_read(&event->child_total_time_running); 6905 6906 arch_perf_update_userpage(event, userpg, now); 6907 6908 barrier(); 6909 ++userpg->lock; 6910 preempt_enable(); 6911 unlock: 6912 rcu_read_unlock(); 6913 } 6914 EXPORT_SYMBOL_GPL(perf_event_update_userpage); 6915 6916 static void ring_buffer_attach(struct perf_event *event, 6917 struct perf_buffer *rb) 6918 { 6919 struct perf_buffer *old_rb = NULL; 6920 unsigned long flags; 6921 6922 WARN_ON_ONCE(event->parent); 6923 6924 if (event->rb) { 6925 /* 6926 * Should be impossible, we set this when removing 6927 * event->rb_entry and wait/clear when adding event->rb_entry. 6928 */ 6929 WARN_ON_ONCE(event->rcu_pending); 6930 6931 old_rb = event->rb; 6932 spin_lock_irqsave(&old_rb->event_lock, flags); 6933 list_del_rcu(&event->rb_entry); 6934 spin_unlock_irqrestore(&old_rb->event_lock, flags); 6935 6936 event->rcu_batches = get_state_synchronize_rcu(); 6937 event->rcu_pending = 1; 6938 } 6939 6940 if (rb) { 6941 if (event->rcu_pending) { 6942 cond_synchronize_rcu(event->rcu_batches); 6943 event->rcu_pending = 0; 6944 } 6945 6946 spin_lock_irqsave(&rb->event_lock, flags); 6947 list_add_rcu(&event->rb_entry, &rb->event_list); 6948 spin_unlock_irqrestore(&rb->event_lock, flags); 6949 } 6950 6951 /* 6952 * Avoid racing with perf_mmap_close(AUX): stop the event 6953 * before swizzling the event::rb pointer; if it's getting 6954 * unmapped, its aux_mmap_count will be 0 and it won't 6955 * restart. See the comment in __perf_pmu_output_stop(). 6956 * 6957 * Data will inevitably be lost when set_output is done in 6958 * mid-air, but then again, whoever does it like this is 6959 * not in for the data anyway. 6960 */ 6961 if (has_aux(event)) 6962 perf_event_stop(event, 0); 6963 6964 rcu_assign_pointer(event->rb, rb); 6965 6966 if (old_rb) { 6967 ring_buffer_put(old_rb); 6968 /* 6969 * Since we detached before setting the new rb, so that we 6970 * could attach the new rb, we could have missed a wakeup. 6971 * Provide it now. 6972 */ 6973 wake_up_all(&event->waitq); 6974 } 6975 } 6976 6977 static void ring_buffer_wakeup(struct perf_event *event) 6978 { 6979 struct perf_buffer *rb; 6980 6981 if (event->parent) 6982 event = event->parent; 6983 6984 rcu_read_lock(); 6985 rb = rcu_dereference(event->rb); 6986 if (rb) { 6987 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) 6988 wake_up_all(&event->waitq); 6989 } 6990 rcu_read_unlock(); 6991 } 6992 6993 struct perf_buffer *ring_buffer_get(struct perf_event *event) 6994 { 6995 struct perf_buffer *rb; 6996 6997 if (event->parent) 6998 event = event->parent; 6999 7000 rcu_read_lock(); 7001 rb = rcu_dereference(event->rb); 7002 if (rb) { 7003 if (!refcount_inc_not_zero(&rb->refcount)) 7004 rb = NULL; 7005 } 7006 rcu_read_unlock(); 7007 7008 return rb; 7009 } 7010 7011 void ring_buffer_put(struct perf_buffer *rb) 7012 { 7013 if (!refcount_dec_and_test(&rb->refcount)) 7014 return; 7015 7016 WARN_ON_ONCE(!list_empty(&rb->event_list)); 7017 7018 call_rcu(&rb->rcu_head, rb_free_rcu); 7019 } 7020 7021 typedef void (*mapped_f)(struct perf_event *event, struct mm_struct *mm); 7022 7023 #define get_mapped(event, func) \ 7024 ({ struct pmu *pmu; \ 7025 mapped_f f = NULL; \ 7026 guard(rcu)(); \ 7027 pmu = READ_ONCE(event->pmu); \ 7028 if (pmu) \ 7029 f = pmu->func; \ 7030 f; \ 7031 }) 7032 7033 static void perf_mmap_open(struct vm_area_struct *vma) 7034 { 7035 struct perf_event *event = vma->vm_file->private_data; 7036 mapped_f mapped = get_mapped(event, event_mapped); 7037 7038 refcount_inc(&event->mmap_count); 7039 refcount_inc(&event->rb->mmap_count); 7040 7041 if (vma_start_pgoff(vma)) 7042 refcount_inc(&event->rb->aux_mmap_count); 7043 7044 if (mapped) 7045 mapped(event, vma->vm_mm); 7046 } 7047 7048 static void perf_pmu_output_stop(struct perf_event *event); 7049 static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer *rb); 7050 7051 /* 7052 * A buffer can be mmap()ed multiple times; either directly through the same 7053 * event, or through other events by use of perf_event_set_output(). 7054 * 7055 * In order to undo the VM accounting done by perf_mmap() we need to destroy 7056 * the buffer here, where we still have a VM context. This means we need 7057 * to detach all events redirecting to us. 7058 */ 7059 static void perf_mmap_close(struct vm_area_struct *vma) 7060 { 7061 struct perf_event *event = vma->vm_file->private_data; 7062 mapped_f unmapped = get_mapped(event, event_unmapped); 7063 struct perf_buffer *rb = ring_buffer_get(event); 7064 struct user_struct *mmap_user = rb->mmap_user; 7065 7066 /* FIXIES vs perf_pmu_unregister() */ 7067 if (unmapped) 7068 unmapped(event, vma->vm_mm); 7069 7070 /* 7071 * The AUX buffer is strictly a sub-buffer, serialize using aux_mutex 7072 * to avoid complications. 7073 */ 7074 if (rb_has_aux(rb) && vma_start_pgoff(vma) == rb->aux_pgoff && 7075 refcount_dec_and_mutex_lock(&rb->aux_mmap_count, &rb->aux_mutex)) { 7076 /* 7077 * Stop all AUX events that are writing to this buffer, 7078 * so that we can free its AUX pages and corresponding PMU 7079 * data. Note that after rb::aux_mmap_count dropped to zero, 7080 * they won't start any more (see perf_aux_output_begin()). 7081 */ 7082 perf_pmu_output_stop(event); 7083 7084 /* now it's safe to free the pages */ 7085 atomic_long_sub(rb->aux_nr_pages - rb->aux_mmap_locked, &mmap_user->perf_mlock); 7086 atomic64_sub(rb->aux_mmap_locked, &vma->vm_mm->pinned_vm); 7087 7088 /* this has to be the last one */ 7089 rb_free_aux(rb); 7090 WARN_ON_ONCE(refcount_read(&rb->aux_refcount)); 7091 7092 mutex_unlock(&rb->aux_mutex); 7093 } 7094 7095 /* 7096 * Drop references in reverse order of perf_mmap() to prevent 7097 * rb revival after rb->mmap_count reaches zero. 7098 */ 7099 if (refcount_dec_and_mutex_lock(&event->mmap_count, 7100 &event->mmap_mutex)) { 7101 ring_buffer_attach(event, NULL); 7102 mutex_unlock(&event->mmap_mutex); 7103 } 7104 7105 /* If there's still other mmap()s of this buffer, we're done. */ 7106 if (!refcount_dec_and_test(&rb->mmap_count)) 7107 goto out_put; 7108 7109 /* 7110 * No other mmap()s, detach from all other events that might redirect 7111 * into the now unreachable buffer. Somewhat complicated by the 7112 * fact that rb::event_lock otherwise nests inside mmap_mutex. 7113 */ 7114 again: 7115 rcu_read_lock(); 7116 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { 7117 if (!atomic_long_inc_not_zero(&event->refcount)) { 7118 /* 7119 * This event is en-route to free_event() which will 7120 * detach it and remove it from the list. 7121 */ 7122 continue; 7123 } 7124 rcu_read_unlock(); 7125 7126 mutex_lock(&event->mmap_mutex); 7127 /* 7128 * Check we didn't race with perf_event_set_output() which can 7129 * swizzle the rb from under us while we were waiting to 7130 * acquire mmap_mutex. 7131 * 7132 * If we find a different rb; ignore this event, a next 7133 * iteration will no longer find it on the list. We have to 7134 * still restart the iteration to make sure we're not now 7135 * iterating the wrong list. 7136 */ 7137 if (event->rb == rb) 7138 ring_buffer_attach(event, NULL); 7139 7140 mutex_unlock(&event->mmap_mutex); 7141 put_event(event); 7142 7143 /* 7144 * Restart the iteration; either we're on the wrong list or 7145 * destroyed its integrity by doing a deletion. 7146 */ 7147 goto again; 7148 } 7149 rcu_read_unlock(); 7150 7151 /* 7152 * It could be there's still a few 0-ref events on the list; they'll 7153 * get cleaned up by free_event() -- they'll also still have their 7154 * ref on the rb and will free it whenever they are done with it. 7155 * 7156 * Aside from that, this buffer is 'fully' detached and unmapped, 7157 * undo the VM accounting. 7158 */ 7159 perf_mmap_unaccount(vma, rb); 7160 7161 out_put: 7162 ring_buffer_put(rb); /* could be last */ 7163 } 7164 7165 static vm_fault_t perf_mmap_pfn_mkwrite(struct vm_fault *vmf) 7166 { 7167 /* The first page is the user control page, others are read-only. */ 7168 return vmf->pgoff == 0 ? 0 : VM_FAULT_SIGBUS; 7169 } 7170 7171 static int perf_mmap_may_split(struct vm_area_struct *vma, unsigned long addr) 7172 { 7173 /* 7174 * Forbid splitting perf mappings to prevent refcount leaks due to 7175 * the resulting non-matching offsets and sizes. See open()/close(). 7176 */ 7177 return -EINVAL; 7178 } 7179 7180 static const struct vm_operations_struct perf_mmap_vmops = { 7181 .open = perf_mmap_open, 7182 .close = perf_mmap_close, /* non mergeable */ 7183 .pfn_mkwrite = perf_mmap_pfn_mkwrite, 7184 .may_split = perf_mmap_may_split, 7185 }; 7186 7187 static int map_range(struct perf_buffer *rb, struct vm_area_struct *vma) 7188 { 7189 unsigned long nr_pages = vma_pages(vma); 7190 int err = 0; 7191 unsigned long pagenum; 7192 7193 guard(mutex)(&rb->aux_mutex); 7194 7195 /* 7196 * We map this as a VM_PFNMAP VMA. 7197 * 7198 * This is not ideal as this is designed broadly for mappings of PFNs 7199 * referencing memory-mapped I/O ranges or non-system RAM i.e. for which 7200 * !pfn_valid(pfn). 7201 * 7202 * We are mapping kernel-allocated memory (memory we manage ourselves) 7203 * which would more ideally be mapped using vm_insert_page() or a 7204 * similar mechanism, that is as a VM_MIXEDMAP mapping. 7205 * 7206 * However this won't work here, because: 7207 * 7208 * 1. It uses vma->vm_page_prot, but this field has not been completely 7209 * setup at the point of the f_op->mmp() hook, so we are unable to 7210 * indicate that this should be mapped CoW in order that the 7211 * mkwrite() hook can be invoked to make the first page R/W and the 7212 * rest R/O as desired. 7213 * 7214 * 2. Anything other than a VM_PFNMAP of valid PFNs will result in 7215 * vm_normal_page() returning a struct page * pointer, which means 7216 * vm_ops->page_mkwrite() will be invoked rather than 7217 * vm_ops->pfn_mkwrite(), and this means we have to set page->mapping 7218 * to work around retry logic in the fault handler, however this 7219 * field is no longer allowed to be used within struct page. 7220 * 7221 * 3. Having a struct page * made available in the fault logic also 7222 * means that the page gets put on the rmap and becomes 7223 * inappropriately accessible and subject to map and ref counting. 7224 * 7225 * Ideally we would have a mechanism that could explicitly express our 7226 * desires, but this is not currently the case, so we instead use 7227 * VM_PFNMAP. 7228 * 7229 * We manage the lifetime of these mappings with internal refcounts (see 7230 * perf_mmap_open() and perf_mmap_close()) so we ensure the lifetime of 7231 * this mapping is maintained correctly. 7232 */ 7233 for (pagenum = 0; pagenum < nr_pages; pagenum++) { 7234 unsigned long va = vma->vm_start + PAGE_SIZE * pagenum; 7235 struct page *page = perf_mmap_to_page(rb, 7236 vma_start_pgoff(vma) + pagenum); 7237 7238 if (page == NULL) { 7239 err = -EINVAL; 7240 break; 7241 } 7242 7243 /* Map readonly, perf_mmap_pfn_mkwrite() called on write fault. */ 7244 err = remap_pfn_range(vma, va, page_to_pfn(page), PAGE_SIZE, 7245 vm_get_page_prot(vma->vm_flags & ~VM_SHARED)); 7246 if (err) 7247 break; 7248 } 7249 7250 #ifdef CONFIG_MMU 7251 /* Clear any partial mappings on error. */ 7252 if (err) 7253 zap_vma_range(vma, vma->vm_start, nr_pages * PAGE_SIZE); 7254 #endif 7255 7256 return err; 7257 } 7258 7259 static bool perf_mmap_calc_limits(struct vm_area_struct *vma, long *user_extra, long *extra) 7260 { 7261 unsigned long user_locked, user_lock_limit, locked, lock_limit; 7262 struct user_struct *user = current_user(); 7263 7264 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); 7265 /* Increase the limit linearly with more CPUs */ 7266 user_lock_limit *= num_online_cpus(); 7267 7268 user_locked = atomic_long_read(&user->perf_mlock); 7269 7270 /* 7271 * sysctl_perf_event_mlock may have changed, so that 7272 * user->perf_mlock > user_lock_limit 7273 */ 7274 if (user_locked > user_lock_limit) 7275 user_locked = user_lock_limit; 7276 user_locked += *user_extra; 7277 7278 if (user_locked > user_lock_limit) { 7279 /* 7280 * charge perf_mlock until it hits user_lock_limit; 7281 * charge the rest from pinned_vm 7282 */ 7283 *extra = user_locked - user_lock_limit; 7284 *user_extra -= *extra; 7285 } 7286 7287 lock_limit = rlimit(RLIMIT_MEMLOCK); 7288 lock_limit >>= PAGE_SHIFT; 7289 locked = atomic64_read(&vma->vm_mm->pinned_vm) + *extra; 7290 7291 return locked <= lock_limit || !perf_is_paranoid() || capable(CAP_IPC_LOCK); 7292 } 7293 7294 static void perf_mmap_account(struct vm_area_struct *vma, long user_extra, long extra) 7295 { 7296 struct user_struct *user = current_user(); 7297 7298 atomic_long_add(user_extra, &user->perf_mlock); 7299 atomic64_add(extra, &vma->vm_mm->pinned_vm); 7300 } 7301 7302 static void perf_mmap_unaccount(struct vm_area_struct *vma, struct perf_buffer *rb) 7303 { 7304 struct user_struct *user = rb->mmap_user; 7305 7306 atomic_long_sub((perf_data_size(rb) >> PAGE_SHIFT) + 1 - rb->mmap_locked, 7307 &user->perf_mlock); 7308 atomic64_sub(rb->mmap_locked, &vma->vm_mm->pinned_vm); 7309 } 7310 7311 static int perf_mmap_rb(struct vm_area_struct *vma, struct perf_event *event, 7312 unsigned long nr_pages) 7313 { 7314 long extra = 0, user_extra = nr_pages; 7315 struct perf_buffer *rb; 7316 int rb_flags = 0; 7317 7318 nr_pages -= 1; 7319 7320 /* 7321 * If we have rb pages ensure they're a power-of-two number, so we 7322 * can do bitmasks instead of modulo. 7323 */ 7324 if (nr_pages != 0 && !is_power_of_2(nr_pages)) 7325 return -EINVAL; 7326 7327 WARN_ON_ONCE(event->ctx->parent_ctx); 7328 7329 if (event->rb) { 7330 if (data_page_nr(event->rb) != nr_pages) 7331 return -EINVAL; 7332 7333 /* 7334 * If this event doesn't have mmap_count, we're attempting to 7335 * create an alias of another event's mmap(); this would mean 7336 * both events will end up scribbling the same user_page; 7337 * which makes no sense. 7338 */ 7339 if (!refcount_read(&event->mmap_count)) 7340 return -EBUSY; 7341 7342 if (refcount_inc_not_zero(&event->rb->mmap_count)) { 7343 /* 7344 * Success -- managed to mmap() the same buffer 7345 * multiple times. 7346 */ 7347 perf_mmap_account(vma, user_extra, extra); 7348 refcount_inc(&event->mmap_count); 7349 return 0; 7350 } 7351 7352 /* 7353 * Raced against perf_mmap_close()'s 7354 * refcount_dec_and_mutex_lock() remove the 7355 * event and continue as if !event->rb 7356 */ 7357 ring_buffer_attach(event, NULL); 7358 } 7359 7360 if (!perf_mmap_calc_limits(vma, &user_extra, &extra)) 7361 return -EPERM; 7362 7363 if (vma->vm_flags & VM_WRITE) 7364 rb_flags |= RING_BUFFER_WRITABLE; 7365 7366 rb = rb_alloc(nr_pages, 7367 event->attr.watermark ? event->attr.wakeup_watermark : 0, 7368 event->cpu, rb_flags); 7369 7370 if (!rb) 7371 return -ENOMEM; 7372 7373 rb->mmap_locked = extra; 7374 7375 ring_buffer_attach(event, rb); 7376 7377 perf_event_update_time(event); 7378 perf_event_init_userpage(event); 7379 perf_event_update_userpage(event); 7380 7381 perf_mmap_account(vma, user_extra, extra); 7382 refcount_set(&event->mmap_count, 1); 7383 7384 return 0; 7385 } 7386 7387 static int perf_mmap_aux(struct vm_area_struct *vma, struct perf_event *event, 7388 unsigned long nr_pages) 7389 { 7390 const pgoff_t pgoff_start = vma_start_pgoff(vma); 7391 long extra = 0, user_extra = nr_pages; 7392 u64 aux_offset, aux_size; 7393 struct perf_buffer *rb; 7394 int ret, rb_flags = 0; 7395 7396 rb = event->rb; 7397 if (!rb) 7398 return -EINVAL; 7399 7400 guard(mutex)(&rb->aux_mutex); 7401 7402 /* 7403 * AUX area mapping: if rb->aux_nr_pages != 0, it's already 7404 * mapped, all subsequent mappings should have the same size 7405 * and offset. Must be above the normal perf buffer. 7406 */ 7407 aux_offset = READ_ONCE(rb->user_page->aux_offset); 7408 aux_size = READ_ONCE(rb->user_page->aux_size); 7409 7410 if (aux_offset < perf_data_size(rb) + PAGE_SIZE) 7411 return -EINVAL; 7412 7413 if (aux_offset != pgoff_start << PAGE_SHIFT) 7414 return -EINVAL; 7415 7416 /* already mapped with a different offset */ 7417 if (rb_has_aux(rb) && rb->aux_pgoff != pgoff_start) 7418 return -EINVAL; 7419 7420 if (aux_size != nr_pages * PAGE_SIZE) 7421 return -EINVAL; 7422 7423 /* already mapped with a different size */ 7424 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages) 7425 return -EINVAL; 7426 7427 if (!is_power_of_2(nr_pages)) 7428 return -EINVAL; 7429 7430 if (!refcount_inc_not_zero(&rb->mmap_count)) 7431 return -EINVAL; 7432 7433 if (rb_has_aux(rb)) { 7434 refcount_inc(&rb->aux_mmap_count); 7435 7436 } else { 7437 if (!perf_mmap_calc_limits(vma, &user_extra, &extra)) { 7438 refcount_dec(&rb->mmap_count); 7439 return -EPERM; 7440 } 7441 7442 WARN_ON(!rb && event->rb); 7443 7444 if (vma->vm_flags & VM_WRITE) 7445 rb_flags |= RING_BUFFER_WRITABLE; 7446 7447 ret = rb_alloc_aux(rb, event, pgoff_start, nr_pages, 7448 event->attr.aux_watermark, rb_flags); 7449 if (ret) { 7450 refcount_dec(&rb->mmap_count); 7451 return ret; 7452 } 7453 7454 refcount_set(&rb->aux_mmap_count, 1); 7455 rb->aux_mmap_locked = extra; 7456 } 7457 7458 perf_mmap_account(vma, user_extra, extra); 7459 refcount_inc(&event->mmap_count); 7460 7461 return 0; 7462 } 7463 7464 static int perf_mmap(struct file *file, struct vm_area_struct *vma) 7465 { 7466 struct perf_event *event = file->private_data; 7467 unsigned long vma_size, nr_pages; 7468 mapped_f mapped; 7469 int ret; 7470 7471 /* 7472 * Don't allow mmap() of inherited per-task counters. This would 7473 * create a performance issue due to all children writing to the 7474 * same rb. 7475 */ 7476 if (event->cpu == -1 && event->attr.inherit) 7477 return -EINVAL; 7478 7479 if (!(vma->vm_flags & VM_SHARED)) 7480 return -EINVAL; 7481 7482 ret = security_perf_event_read(event); 7483 if (ret) 7484 return ret; 7485 7486 vma_size = vma->vm_end - vma->vm_start; 7487 nr_pages = vma_size / PAGE_SIZE; 7488 7489 if (nr_pages > INT_MAX) 7490 return -ENOMEM; 7491 7492 if (vma_size != PAGE_SIZE * nr_pages) 7493 return -EINVAL; 7494 7495 scoped_guard (mutex, &event->mmap_mutex) { 7496 /* 7497 * This relies on __pmu_detach_event() taking mmap_mutex after marking 7498 * the event REVOKED. Either we observe the state, or __pmu_detach_event() 7499 * will detach the rb created here. 7500 */ 7501 if (event->state <= PERF_EVENT_STATE_REVOKED) 7502 return -ENODEV; 7503 7504 if (!vma_start_pgoff(vma)) 7505 ret = perf_mmap_rb(vma, event, nr_pages); 7506 else 7507 ret = perf_mmap_aux(vma, event, nr_pages); 7508 if (ret) 7509 return ret; 7510 7511 /* 7512 * Since pinned accounting is per vm we cannot allow fork() to copy our 7513 * vma. 7514 */ 7515 vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP); 7516 vma->vm_ops = &perf_mmap_vmops; 7517 7518 mapped = get_mapped(event, event_mapped); 7519 if (mapped) 7520 mapped(event, vma->vm_mm); 7521 7522 /* 7523 * Try to map it into the page table. On fail undo the above, 7524 * as the callsite expects full cleanup in this case and 7525 * therefore does not invoke vmops::close(). 7526 */ 7527 ret = map_range(event->rb, vma); 7528 if (likely(!ret)) 7529 return 0; 7530 7531 /* Error path */ 7532 7533 /* 7534 * If this is the first mmap(), then event->mmap_count should 7535 * be stable at 1. It is only modified by: 7536 * perf_mmap_{open,close}() and perf_mmap(). 7537 * 7538 * The former are not possible because this mmap() hasn't been 7539 * successful yet, and the latter is serialized by 7540 * event->mmap_mutex which we still hold (note that mmap_lock 7541 * is not strictly sufficient here, because the event fd can 7542 * be passed to another process through trivial means like 7543 * fork(), leading to concurrent mmap() from different mm). 7544 * 7545 * Make sure to remove event->rb before releasing 7546 * event->mmap_mutex, such that any concurrent mmap() will not 7547 * attempt use this failed buffer. 7548 */ 7549 if (refcount_read(&event->mmap_count) == 1) { 7550 /* 7551 * Minimal perf_mmap_close(); there can't be AUX or 7552 * other events on account of this being the first. 7553 */ 7554 mapped = get_mapped(event, event_unmapped); 7555 if (mapped) 7556 mapped(event, vma->vm_mm); 7557 perf_mmap_unaccount(vma, event->rb); 7558 ring_buffer_attach(event, NULL); /* drops last rb->refcount */ 7559 refcount_set(&event->mmap_count, 0); 7560 return ret; 7561 } 7562 7563 /* 7564 * Otherwise this is an already existing buffer, and there is 7565 * no race vs first exposure, so fall-through and call 7566 * perf_mmap_close(). 7567 */ 7568 } 7569 7570 perf_mmap_close(vma); 7571 return ret; 7572 } 7573 7574 static int perf_fasync(int fd, struct file *filp, int on) 7575 { 7576 struct inode *inode = file_inode(filp); 7577 struct perf_event *event = filp->private_data; 7578 int retval; 7579 7580 if (event->state <= PERF_EVENT_STATE_REVOKED) 7581 return -ENODEV; 7582 7583 inode_lock(inode); 7584 retval = fasync_helper(fd, filp, on, &event->fasync); 7585 inode_unlock(inode); 7586 7587 if (retval < 0) 7588 return retval; 7589 7590 return 0; 7591 } 7592 7593 static void perf_show_fdinfo(struct seq_file *m, struct file *f) 7594 { 7595 struct perf_event *event = f->private_data; 7596 struct perf_event_context *ctx; 7597 struct mutex *child_mutex; 7598 7599 ctx = perf_event_ctx_lock(event); 7600 child_mutex = event->parent ? &event->parent->child_mutex : &event->child_mutex; 7601 mutex_lock(child_mutex); 7602 7603 seq_printf(m, "perf_event_attr.type:\t%u\n", event->orig_type); 7604 if (event->pmu) 7605 seq_printf(m, "pmu_type:\t%u\n", event->pmu->type); 7606 seq_printf(m, "perf_event_attr.config:\t0x%llx\n", (unsigned long long)event->attr.config); 7607 seq_printf(m, "perf_event_attr.config1:\t0x%llx\n", 7608 (unsigned long long)event->attr.config1); 7609 seq_printf(m, "perf_event_attr.config2:\t0x%llx\n", 7610 (unsigned long long)event->attr.config2); 7611 seq_printf(m, "perf_event_attr.config3:\t0x%llx\n", 7612 (unsigned long long)event->attr.config3); 7613 seq_printf(m, "perf_event_attr.config4:\t0x%llx\n", 7614 (unsigned long long)event->attr.config4); 7615 7616 mutex_unlock(child_mutex); 7617 perf_event_ctx_unlock(event, ctx); 7618 } 7619 7620 static const struct file_operations perf_fops = { 7621 .release = perf_release, 7622 .read = perf_read, 7623 .poll = perf_poll, 7624 .unlocked_ioctl = perf_ioctl, 7625 .compat_ioctl = perf_compat_ioctl, 7626 .mmap = perf_mmap, 7627 .fasync = perf_fasync, 7628 .show_fdinfo = perf_show_fdinfo, 7629 }; 7630 7631 /* 7632 * Perf event wakeup 7633 * 7634 * If there's data, ensure we set the poll() state and publish everything 7635 * to user-space before waking everybody up. 7636 */ 7637 7638 void perf_event_wakeup(struct perf_event *event) 7639 { 7640 ring_buffer_wakeup(event); 7641 7642 if (event->pending_kill) { 7643 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill); 7644 event->pending_kill = 0; 7645 } 7646 } 7647 7648 static void perf_sigtrap(struct perf_event *event) 7649 { 7650 /* 7651 * Both perf_pending_task() and perf_pending_irq() can race with the 7652 * task exiting or exec-ing. We can determine if such a race has 7653 * occurred by checking if perf_event_exit_task(), which will set 7654 * ctx->task to TASK_TOMBSTONE, has already been called. 7655 */ 7656 if (event->ctx->task == TASK_TOMBSTONE) 7657 return; 7658 7659 /* 7660 * The event was removed after this signal was queued, e.g. by exec() 7661 * (remove_on_exec), which may also have closed its fd (close-on-exec). 7662 * The new program has default signal handlers, so a SIGTRAP would 7663 * kill it. 7664 */ 7665 if (READ_ONCE(event->state) <= PERF_EVENT_STATE_EXIT) 7666 return; 7667 7668 /* 7669 * We'd expect this to only occur if the irq_work is delayed and either 7670 * ctx->task or current has changed in the meantime. This can be the 7671 * case on architectures that do not implement arch_irq_work_raise(). 7672 */ 7673 if (WARN_ON_ONCE(event->ctx->task != current)) 7674 return; 7675 7676 send_sig_perf((void __user *)event->pending_addr, 7677 event->orig_type, event->attr.sig_data); 7678 } 7679 7680 /* 7681 * Deliver the pending work in-event-context or follow the context. 7682 */ 7683 static void __perf_pending_disable(struct perf_event *event) 7684 { 7685 int cpu = READ_ONCE(event->oncpu); 7686 7687 /* 7688 * If the event isn't running; we done. event_sched_out() will have 7689 * taken care of things. 7690 */ 7691 if (cpu < 0) 7692 return; 7693 7694 /* 7695 * Yay, we hit home and are in the context of the event. 7696 */ 7697 if (cpu == smp_processor_id()) { 7698 if (event->pending_disable) { 7699 event->pending_disable = 0; 7700 perf_event_disable_local(event); 7701 } 7702 return; 7703 } 7704 7705 /* 7706 * CPU-A CPU-B 7707 * 7708 * perf_event_disable_inatomic() 7709 * @pending_disable = 1; 7710 * irq_work_queue(); 7711 * 7712 * sched-out 7713 * @pending_disable = 0; 7714 * 7715 * sched-in 7716 * perf_event_disable_inatomic() 7717 * @pending_disable = 1; 7718 * irq_work_queue(); // FAILS 7719 * 7720 * irq_work_run() 7721 * perf_pending_disable() 7722 * 7723 * But the event runs on CPU-B and wants disabling there. 7724 */ 7725 irq_work_queue_on(&event->pending_disable_irq, cpu); 7726 } 7727 7728 static void perf_pending_disable(struct irq_work *entry) 7729 { 7730 struct perf_event *event = container_of(entry, struct perf_event, pending_disable_irq); 7731 int rctx; 7732 7733 /* 7734 * If we 'fail' here, that's OK, it means recursion is already disabled 7735 * and we won't recurse 'further'. 7736 */ 7737 rctx = perf_swevent_get_recursion_context(); 7738 __perf_pending_disable(event); 7739 if (rctx >= 0) 7740 perf_swevent_put_recursion_context(rctx); 7741 } 7742 7743 static void perf_pending_irq(struct irq_work *entry) 7744 { 7745 struct perf_event *event = container_of(entry, struct perf_event, pending_irq); 7746 int rctx; 7747 7748 /* 7749 * If we 'fail' here, that's OK, it means recursion is already disabled 7750 * and we won't recurse 'further'. 7751 */ 7752 rctx = perf_swevent_get_recursion_context(); 7753 7754 /* 7755 * The wakeup isn't bound to the context of the event -- it can happen 7756 * irrespective of where the event is. 7757 */ 7758 if (event->pending_wakeup) { 7759 event->pending_wakeup = 0; 7760 perf_event_wakeup(event); 7761 } 7762 7763 if (rctx >= 0) 7764 perf_swevent_put_recursion_context(rctx); 7765 } 7766 7767 static void perf_pending_task(struct callback_head *head) 7768 { 7769 struct perf_event *event = container_of(head, struct perf_event, pending_task); 7770 int rctx; 7771 7772 /* 7773 * If we 'fail' here, that's OK, it means recursion is already disabled 7774 * and we won't recurse 'further'. 7775 */ 7776 rctx = perf_swevent_get_recursion_context(); 7777 7778 if (event->pending_work) { 7779 event->pending_work = 0; 7780 perf_sigtrap(event); 7781 local_dec(&event->ctx->nr_no_switch_fast); 7782 } 7783 put_event(event); 7784 7785 if (rctx >= 0) 7786 perf_swevent_put_recursion_context(rctx); 7787 } 7788 7789 #ifdef CONFIG_GUEST_PERF_EVENTS 7790 struct perf_guest_info_callbacks __rcu *perf_guest_cbs; 7791 7792 DEFINE_STATIC_CALL_RET0(__perf_guest_state, *perf_guest_cbs->state); 7793 DEFINE_STATIC_CALL_RET0(__perf_guest_get_ip, *perf_guest_cbs->get_ip); 7794 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_intel_pt_intr, *perf_guest_cbs->handle_intel_pt_intr); 7795 DEFINE_STATIC_CALL_RET0(__perf_guest_handle_mediated_pmi, *perf_guest_cbs->handle_mediated_pmi); 7796 7797 void perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 7798 { 7799 if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs))) 7800 return; 7801 7802 rcu_assign_pointer(perf_guest_cbs, cbs); 7803 static_call_update(__perf_guest_state, cbs->state); 7804 static_call_update(__perf_guest_get_ip, cbs->get_ip); 7805 7806 /* Implementing ->handle_intel_pt_intr is optional. */ 7807 if (cbs->handle_intel_pt_intr) 7808 static_call_update(__perf_guest_handle_intel_pt_intr, 7809 cbs->handle_intel_pt_intr); 7810 7811 if (cbs->handle_mediated_pmi) 7812 static_call_update(__perf_guest_handle_mediated_pmi, 7813 cbs->handle_mediated_pmi); 7814 } 7815 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); 7816 7817 void perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 7818 { 7819 if (WARN_ON_ONCE(rcu_access_pointer(perf_guest_cbs) != cbs)) 7820 return; 7821 7822 rcu_assign_pointer(perf_guest_cbs, NULL); 7823 static_call_update(__perf_guest_state, (void *)&__static_call_return0); 7824 static_call_update(__perf_guest_get_ip, (void *)&__static_call_return0); 7825 static_call_update(__perf_guest_handle_intel_pt_intr, (void *)&__static_call_return0); 7826 static_call_update(__perf_guest_handle_mediated_pmi, (void *)&__static_call_return0); 7827 synchronize_rcu(); 7828 } 7829 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); 7830 #endif 7831 7832 static bool should_sample_guest(struct perf_event *event) 7833 { 7834 return !event->attr.exclude_guest && perf_guest_state(); 7835 } 7836 7837 unsigned long perf_misc_flags(struct perf_event *event, 7838 struct pt_regs *regs) 7839 { 7840 if (should_sample_guest(event)) 7841 return perf_arch_guest_misc_flags(regs); 7842 7843 return perf_arch_misc_flags(regs); 7844 } 7845 7846 unsigned long perf_instruction_pointer(struct perf_event *event, 7847 struct pt_regs *regs) 7848 { 7849 /* 7850 * Hardware skid can lead to a scenario where a PMI is 7851 * delivered after the CPU has already entered kernel mode. 7852 * In that case, user-space sampling must not expose kernel 7853 * register state. 7854 */ 7855 if (should_sample_guest(event)) { 7856 return event->attr.exclude_kernel && 7857 !(perf_guest_state() & PERF_GUEST_USER) ? 7858 0 : perf_guest_get_ip(); 7859 } 7860 7861 return event->attr.exclude_kernel && !user_mode(regs) ? 7862 0 : perf_arch_instruction_pointer(regs); 7863 } 7864 7865 u64 __weak perf_reg_value(struct pt_regs *regs, int idx) 7866 { 7867 return 0; 7868 } 7869 7870 int __weak perf_reg_validate(u64 mask, bool simd_enabled) 7871 { 7872 return mask ? -ENOSYS : 0; 7873 } 7874 7875 u64 __weak perf_reg_abi(struct task_struct *task) 7876 { 7877 return PERF_SAMPLE_REGS_ABI_NONE; 7878 } 7879 7880 void __weak perf_get_regs_user(struct perf_regs *regs_user, 7881 struct pt_regs *regs) 7882 { 7883 regs_user->regs = task_pt_regs(current); 7884 regs_user->abi = perf_reg_abi(current); 7885 } 7886 7887 #define word_for_each_set_bit(bit, val) \ 7888 for (unsigned long long __v = (val); \ 7889 __v && ((bit = __builtin_ctzll(__v)), 1); \ 7890 __v &= __v - 1) 7891 7892 static void 7893 perf_output_sample_regs(struct perf_output_handle *handle, 7894 struct pt_regs *regs, u64 mask) 7895 { 7896 int bit; 7897 7898 word_for_each_set_bit(bit, mask) { 7899 u64 val = perf_reg_value(regs, bit); 7900 perf_output_put(handle, val); 7901 } 7902 } 7903 7904 static void 7905 perf_output_sample_simd_regs(struct perf_output_handle *handle, 7906 struct perf_event *event, 7907 struct pt_regs *regs, 7908 u64 mask, u32 pred_mask) 7909 { 7910 u64 pred_qwords = event->attr.sample_simd_pred_reg_qwords; 7911 u64 vec_qwords = event->attr.sample_simd_vec_reg_qwords; 7912 u64 nr_vectors = hweight64(mask); 7913 u64 nr_pred = hweight32(pred_mask); 7914 int bit; 7915 7916 perf_output_put(handle, nr_vectors); 7917 perf_output_put(handle, vec_qwords); 7918 perf_output_put(handle, nr_pred); 7919 perf_output_put(handle, pred_qwords); 7920 7921 if (nr_vectors) { 7922 word_for_each_set_bit(bit, mask) { 7923 for (int i = 0; i < vec_qwords; i++) { 7924 u64 val = perf_simd_reg_value(regs, bit, 7925 i, false); 7926 perf_output_put(handle, val); 7927 } 7928 } 7929 } 7930 if (nr_pred) { 7931 word_for_each_set_bit(bit, pred_mask) { 7932 for (int i = 0; i < pred_qwords; i++) { 7933 u64 val = perf_simd_reg_value(regs, bit, 7934 i, true); 7935 perf_output_put(handle, val); 7936 } 7937 } 7938 } 7939 } 7940 7941 void perf_sample_regs_user(struct perf_regs *regs_user, 7942 struct pt_regs *regs) 7943 { 7944 if (user_mode(regs)) { 7945 regs_user->abi = perf_reg_abi(current); 7946 regs_user->regs = regs; 7947 } else if (is_user_task(current)) { 7948 perf_get_regs_user(regs_user, regs); 7949 } else { 7950 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; 7951 regs_user->regs = NULL; 7952 } 7953 } 7954 7955 void perf_sample_regs_intr(struct perf_regs *regs_intr, 7956 struct pt_regs *regs, 7957 bool exclude_kernel) 7958 { 7959 /* 7960 * Hardware skid can lead to a scenario where a PMI is 7961 * delivered after the CPU has already entered kernel mode. 7962 * In that case, user-space sampling must not expose kernel 7963 * register state. 7964 */ 7965 if (exclude_kernel && !user_mode(regs)) { 7966 regs_intr->abi = PERF_SAMPLE_REGS_ABI_NONE; 7967 regs_intr->regs = NULL; 7968 } else { 7969 regs_intr->regs = regs; 7970 regs_intr->abi = perf_reg_abi(current); 7971 } 7972 } 7973 7974 int __weak perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, 7975 u16 pred_qwords, u32 pred_mask) 7976 { 7977 return -EINVAL; 7978 } 7979 7980 u64 __weak perf_simd_reg_value(struct pt_regs *regs, int idx, 7981 u16 qwords_idx, bool pred) 7982 { 7983 return 0; 7984 } 7985 7986 /* 7987 * Get remaining task size from user stack pointer. 7988 * 7989 * It'd be better to take stack vma map and limit this more 7990 * precisely, but there's no way to get it safely under interrupt, 7991 * so using TASK_SIZE as limit. 7992 */ 7993 static u64 perf_ustack_task_size(struct pt_regs *regs) 7994 { 7995 unsigned long addr = perf_user_stack_pointer(regs); 7996 7997 if (!addr || addr >= TASK_SIZE) 7998 return 0; 7999 8000 return TASK_SIZE - addr; 8001 } 8002 8003 static u16 8004 perf_sample_ustack_size(u16 stack_size, u16 header_size, 8005 struct pt_regs *regs) 8006 { 8007 u64 task_size; 8008 8009 /* No regs, no stack pointer, no dump. */ 8010 if (!regs) 8011 return 0; 8012 8013 /* No mm, no stack, no dump. */ 8014 if (!current->mm) 8015 return 0; 8016 8017 /* 8018 * Check if we fit in with the requested stack size into the: 8019 * - TASK_SIZE 8020 * If we don't, we limit the size to the TASK_SIZE. 8021 * 8022 * - remaining sample size 8023 * If we don't, we customize the stack size to 8024 * fit in to the remaining sample size. 8025 */ 8026 8027 task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs)); 8028 stack_size = min(stack_size, (u16) task_size); 8029 8030 /* Current header size plus static size and dynamic size. */ 8031 header_size += 2 * sizeof(u64); 8032 8033 /* Do we fit in with the current stack dump size? */ 8034 if ((u16) (header_size + stack_size) < header_size) { 8035 /* 8036 * If we overflow the maximum size for the sample, 8037 * we customize the stack dump size to fit in. 8038 */ 8039 stack_size = USHRT_MAX - header_size - sizeof(u64); 8040 stack_size = round_up(stack_size, sizeof(u64)); 8041 } 8042 8043 return stack_size; 8044 } 8045 8046 static void 8047 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size, 8048 struct pt_regs *regs) 8049 { 8050 /* Case of a kernel thread, nothing to dump */ 8051 if (!regs) { 8052 u64 size = 0; 8053 perf_output_put(handle, size); 8054 } else { 8055 unsigned long sp; 8056 unsigned int rem; 8057 u64 dyn_size; 8058 8059 /* 8060 * We dump: 8061 * static size 8062 * - the size requested by user or the best one we can fit 8063 * in to the sample max size 8064 * data 8065 * - user stack dump data 8066 * dynamic size 8067 * - the actual dumped size 8068 */ 8069 8070 /* Static size. */ 8071 perf_output_put(handle, dump_size); 8072 8073 /* Data. */ 8074 sp = perf_user_stack_pointer(regs); 8075 rem = __output_copy_user(handle, (void *) sp, dump_size); 8076 dyn_size = dump_size - rem; 8077 8078 perf_output_skip(handle, rem); 8079 8080 /* Dynamic size. */ 8081 perf_output_put(handle, dyn_size); 8082 } 8083 } 8084 8085 static unsigned long perf_prepare_sample_aux(struct perf_event *event, 8086 struct perf_sample_data *data, 8087 size_t size) 8088 { 8089 struct perf_event *sampler = event->aux_event; 8090 struct perf_buffer *rb; 8091 8092 data->aux_size = 0; 8093 8094 if (!sampler) 8095 goto out; 8096 8097 if (WARN_ON_ONCE(READ_ONCE(sampler->state) != PERF_EVENT_STATE_ACTIVE)) 8098 goto out; 8099 8100 if (WARN_ON_ONCE(READ_ONCE(sampler->oncpu) != smp_processor_id())) 8101 goto out; 8102 8103 rb = ring_buffer_get(sampler); 8104 if (!rb) 8105 goto out; 8106 8107 /* 8108 * If this is an NMI hit inside sampling code, don't take 8109 * the sample. See also perf_aux_sample_output(). 8110 */ 8111 if (READ_ONCE(rb->aux_in_sampling)) { 8112 data->aux_size = 0; 8113 } else { 8114 size = min_t(size_t, size, perf_aux_size(rb)); 8115 data->aux_size = ALIGN(size, sizeof(u64)); 8116 } 8117 ring_buffer_put(rb); 8118 8119 out: 8120 return data->aux_size; 8121 } 8122 8123 static long perf_pmu_snapshot_aux(struct perf_buffer *rb, 8124 struct perf_event *event, 8125 struct perf_output_handle *handle, 8126 unsigned long size) 8127 { 8128 unsigned long flags; 8129 long ret; 8130 8131 /* 8132 * Normal ->start()/->stop() callbacks run in IRQ mode in scheduler 8133 * paths. If we start calling them in NMI context, they may race with 8134 * the IRQ ones, that is, for example, re-starting an event that's just 8135 * been stopped, which is why we're using a separate callback that 8136 * doesn't change the event state. 8137 * 8138 * IRQs need to be disabled to prevent IPIs from racing with us. 8139 */ 8140 local_irq_save(flags); 8141 /* 8142 * Guard against NMI hits inside the critical section; 8143 * see also perf_prepare_sample_aux(). 8144 */ 8145 WRITE_ONCE(rb->aux_in_sampling, 1); 8146 barrier(); 8147 8148 ret = event->pmu->snapshot_aux(event, handle, size); 8149 8150 barrier(); 8151 WRITE_ONCE(rb->aux_in_sampling, 0); 8152 local_irq_restore(flags); 8153 8154 return ret; 8155 } 8156 8157 static void perf_aux_sample_output(struct perf_event *event, 8158 struct perf_output_handle *handle, 8159 struct perf_sample_data *data) 8160 { 8161 struct perf_event *sampler = event->aux_event; 8162 struct perf_buffer *rb; 8163 unsigned long pad; 8164 long size; 8165 8166 if (WARN_ON_ONCE(!sampler || !data->aux_size)) 8167 return; 8168 8169 rb = ring_buffer_get(sampler); 8170 if (!rb) 8171 return; 8172 8173 size = perf_pmu_snapshot_aux(rb, sampler, handle, data->aux_size); 8174 8175 /* 8176 * An error here means that perf_output_copy() failed (returned a 8177 * non-zero surplus that it didn't copy), which in its current 8178 * enlightened implementation is not possible. If that changes, we'd 8179 * like to know. 8180 */ 8181 if (WARN_ON_ONCE(size < 0)) 8182 goto out_put; 8183 8184 /* 8185 * The pad comes from ALIGN()ing data->aux_size up to u64 in 8186 * perf_prepare_sample_aux(), so should not be more than that. 8187 */ 8188 pad = data->aux_size - size; 8189 if (WARN_ON_ONCE(pad >= sizeof(u64))) 8190 pad = 8; 8191 8192 if (pad) { 8193 u64 zero = 0; 8194 perf_output_copy(handle, &zero, pad); 8195 } 8196 8197 out_put: 8198 ring_buffer_put(rb); 8199 } 8200 8201 /* 8202 * A set of common sample data types saved even for non-sample records 8203 * when event->attr.sample_id_all is set. 8204 */ 8205 #define PERF_SAMPLE_ID_ALL (PERF_SAMPLE_TID | PERF_SAMPLE_TIME | \ 8206 PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID | \ 8207 PERF_SAMPLE_CPU | PERF_SAMPLE_IDENTIFIER) 8208 8209 static void __perf_event_header__init_id(struct perf_sample_data *data, 8210 struct perf_event *event, 8211 u64 sample_type) 8212 { 8213 data->type = event->attr.sample_type; 8214 data->sample_flags |= data->type & PERF_SAMPLE_ID_ALL; 8215 8216 if (sample_type & PERF_SAMPLE_TID) { 8217 /* namespace issues */ 8218 data->tid_entry.pid = perf_event_pid(event, current); 8219 data->tid_entry.tid = perf_event_tid(event, current); 8220 } 8221 8222 if (sample_type & PERF_SAMPLE_TIME) 8223 data->time = perf_event_clock(event); 8224 8225 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 8226 data->id = primary_event_id(event); 8227 8228 if (sample_type & PERF_SAMPLE_STREAM_ID) 8229 data->stream_id = event->id; 8230 8231 if (sample_type & PERF_SAMPLE_CPU) { 8232 data->cpu_entry.cpu = raw_smp_processor_id(); 8233 data->cpu_entry.reserved = 0; 8234 } 8235 } 8236 8237 void perf_event_header__init(struct perf_event_header *header, 8238 struct perf_sample_data *data, 8239 u32 type, u16 misc, u16 size, 8240 struct perf_event *event) 8241 { 8242 header->type = type; 8243 header->misc = misc; 8244 header->size = size; 8245 8246 if (event->attr.sample_id_all) { 8247 header->size += event->id_header_size; 8248 __perf_event_header__init_id(data, event, event->attr.sample_type); 8249 } 8250 } 8251 8252 static void __perf_event__output_id_sample(struct perf_output_handle *handle, 8253 struct perf_sample_data *data) 8254 { 8255 u64 sample_type = data->type; 8256 8257 if (sample_type & PERF_SAMPLE_TID) 8258 perf_output_put(handle, data->tid_entry); 8259 8260 if (sample_type & PERF_SAMPLE_TIME) 8261 perf_output_put(handle, data->time); 8262 8263 if (sample_type & PERF_SAMPLE_ID) 8264 perf_output_put(handle, data->id); 8265 8266 if (sample_type & PERF_SAMPLE_STREAM_ID) 8267 perf_output_put(handle, data->stream_id); 8268 8269 if (sample_type & PERF_SAMPLE_CPU) 8270 perf_output_put(handle, data->cpu_entry); 8271 8272 if (sample_type & PERF_SAMPLE_IDENTIFIER) 8273 perf_output_put(handle, data->id); 8274 } 8275 8276 void perf_event__output_id_sample(struct perf_event *event, 8277 struct perf_output_handle *handle, 8278 struct perf_sample_data *sample) 8279 { 8280 if (event->attr.sample_id_all) 8281 __perf_event__output_id_sample(handle, sample); 8282 } 8283 8284 static void perf_output_read_one(struct perf_output_handle *handle, 8285 struct perf_event *event, 8286 u64 enabled, u64 running) 8287 { 8288 u64 read_format = event->attr.read_format; 8289 u64 values[5]; 8290 int n = 0; 8291 8292 values[n++] = perf_event_count(event, has_inherit_and_sample_read(&event->attr)); 8293 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 8294 values[n++] = enabled + 8295 atomic64_read(&event->child_total_time_enabled); 8296 } 8297 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 8298 values[n++] = running + 8299 atomic64_read(&event->child_total_time_running); 8300 } 8301 if (read_format & PERF_FORMAT_ID) 8302 values[n++] = primary_event_id(event); 8303 if (read_format & PERF_FORMAT_LOST) 8304 values[n++] = atomic64_read(&event->lost_samples); 8305 8306 __output_copy(handle, values, n * sizeof(u64)); 8307 } 8308 8309 static void perf_output_read_group(struct perf_output_handle *handle, 8310 struct perf_event *event, 8311 u64 enabled, u64 running) 8312 { 8313 struct perf_event *leader = event->group_leader, *sub; 8314 u64 read_format = event->attr.read_format; 8315 unsigned long flags; 8316 u64 values[6]; 8317 int n = 0; 8318 bool self = has_inherit_and_sample_read(&event->attr); 8319 8320 /* 8321 * Disabling interrupts avoids all counter scheduling 8322 * (context switches, timer based rotation and IPIs). 8323 */ 8324 local_irq_save(flags); 8325 8326 values[n++] = 1 + leader->nr_siblings; 8327 8328 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 8329 values[n++] = enabled; 8330 8331 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 8332 values[n++] = running; 8333 8334 if ((leader != event) && !handle->skip_read) 8335 perf_pmu_read(leader); 8336 8337 values[n++] = perf_event_count(leader, self); 8338 if (read_format & PERF_FORMAT_ID) 8339 values[n++] = primary_event_id(leader); 8340 if (read_format & PERF_FORMAT_LOST) 8341 values[n++] = atomic64_read(&leader->lost_samples); 8342 8343 __output_copy(handle, values, n * sizeof(u64)); 8344 8345 for_each_sibling_event(sub, leader) { 8346 n = 0; 8347 8348 if ((sub != event) && !handle->skip_read) 8349 perf_pmu_read(sub); 8350 8351 values[n++] = perf_event_count(sub, self); 8352 if (read_format & PERF_FORMAT_ID) 8353 values[n++] = primary_event_id(sub); 8354 if (read_format & PERF_FORMAT_LOST) 8355 values[n++] = atomic64_read(&sub->lost_samples); 8356 8357 __output_copy(handle, values, n * sizeof(u64)); 8358 } 8359 8360 local_irq_restore(flags); 8361 } 8362 8363 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\ 8364 PERF_FORMAT_TOTAL_TIME_RUNNING) 8365 8366 /* 8367 * XXX PERF_SAMPLE_READ vs inherited events seems difficult. 8368 * 8369 * The problem is that its both hard and excessively expensive to iterate the 8370 * child list, not to mention that its impossible to IPI the children running 8371 * on another CPU, from interrupt/NMI context. 8372 * 8373 * Instead the combination of PERF_SAMPLE_READ and inherit will track per-thread 8374 * counts rather than attempting to accumulate some value across all children on 8375 * all cores. 8376 */ 8377 static void perf_output_read(struct perf_output_handle *handle, 8378 struct perf_event *event) 8379 { 8380 u64 enabled = 0, running = 0, now; 8381 u64 read_format = event->attr.read_format; 8382 8383 /* 8384 * Compute total_time_enabled, total_time_running based on snapshot 8385 * values taken when the event was last scheduled in. 8386 * 8387 * We cannot simply call update_context_time() because doing so would 8388 * lead to deadlock when called from NMI context. 8389 */ 8390 if (read_format & PERF_FORMAT_TOTAL_TIMES) 8391 calc_timer_values(event, &now, &enabled, &running); 8392 8393 if (event->attr.read_format & PERF_FORMAT_GROUP) 8394 perf_output_read_group(handle, event, enabled, running); 8395 else 8396 perf_output_read_one(handle, event, enabled, running); 8397 } 8398 8399 void perf_output_sample(struct perf_output_handle *handle, 8400 struct perf_event_header *header, 8401 struct perf_sample_data *data, 8402 struct perf_event *event) 8403 { 8404 u64 sample_type = data->type; 8405 8406 if (data->sample_flags & PERF_SAMPLE_READ) 8407 handle->skip_read = 1; 8408 8409 perf_output_put(handle, *header); 8410 8411 if (sample_type & PERF_SAMPLE_IDENTIFIER) 8412 perf_output_put(handle, data->id); 8413 8414 if (sample_type & PERF_SAMPLE_IP) 8415 perf_output_put(handle, data->ip); 8416 8417 if (sample_type & PERF_SAMPLE_TID) 8418 perf_output_put(handle, data->tid_entry); 8419 8420 if (sample_type & PERF_SAMPLE_TIME) 8421 perf_output_put(handle, data->time); 8422 8423 if (sample_type & PERF_SAMPLE_ADDR) 8424 perf_output_put(handle, data->addr); 8425 8426 if (sample_type & PERF_SAMPLE_ID) 8427 perf_output_put(handle, data->id); 8428 8429 if (sample_type & PERF_SAMPLE_STREAM_ID) 8430 perf_output_put(handle, data->stream_id); 8431 8432 if (sample_type & PERF_SAMPLE_CPU) 8433 perf_output_put(handle, data->cpu_entry); 8434 8435 if (sample_type & PERF_SAMPLE_PERIOD) 8436 perf_output_put(handle, data->period); 8437 8438 if (sample_type & PERF_SAMPLE_READ) 8439 perf_output_read(handle, event); 8440 8441 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 8442 int size = 1; 8443 8444 size += data->callchain->nr; 8445 size *= sizeof(u64); 8446 __output_copy(handle, data->callchain, size); 8447 } 8448 8449 if (sample_type & PERF_SAMPLE_RAW) { 8450 struct perf_raw_record *raw = data->raw; 8451 8452 if (raw) { 8453 struct perf_raw_frag *frag = &raw->frag; 8454 8455 perf_output_put(handle, raw->size); 8456 do { 8457 if (frag->copy) { 8458 __output_custom(handle, frag->copy, 8459 frag->data, frag->size); 8460 } else { 8461 __output_copy(handle, frag->data, 8462 frag->size); 8463 } 8464 if (perf_raw_frag_last(frag)) 8465 break; 8466 frag = frag->next; 8467 } while (1); 8468 if (frag->pad) 8469 __output_skip(handle, NULL, frag->pad); 8470 } else { 8471 struct { 8472 u32 size; 8473 u32 data; 8474 } raw = { 8475 .size = sizeof(u32), 8476 .data = 0, 8477 }; 8478 perf_output_put(handle, raw); 8479 } 8480 } 8481 8482 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 8483 if (data->br_stack) { 8484 size_t size; 8485 8486 size = data->br_stack->nr 8487 * sizeof(struct perf_branch_entry); 8488 8489 perf_output_put(handle, data->br_stack->nr); 8490 if (branch_sample_hw_index(event)) 8491 perf_output_put(handle, data->br_stack->hw_idx); 8492 perf_output_copy(handle, data->br_stack->entries, size); 8493 /* 8494 * Add the extension space which is appended 8495 * right after the struct perf_branch_stack. 8496 */ 8497 if (data->br_stack_cntr) { 8498 size = data->br_stack->nr * sizeof(u64); 8499 perf_output_copy(handle, data->br_stack_cntr, size); 8500 } 8501 } else { 8502 /* 8503 * we always store at least the value of nr 8504 */ 8505 u64 nr = 0; 8506 perf_output_put(handle, nr); 8507 } 8508 } 8509 8510 if (sample_type & PERF_SAMPLE_REGS_USER) { 8511 u64 abi = data->regs_user.abi; 8512 8513 /* 8514 * If there are no regs to dump, notice it through 8515 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 8516 */ 8517 perf_output_put(handle, abi); 8518 8519 if (abi) { 8520 struct perf_event_attr *attr = &event->attr; 8521 u64 mask = attr->sample_regs_user; 8522 perf_output_sample_regs(handle, 8523 data->regs_user.regs, 8524 mask); 8525 if (abi & PERF_SAMPLE_REGS_ABI_SIMD) { 8526 perf_output_sample_simd_regs(handle, event, 8527 data->regs_user.regs, 8528 attr->sample_simd_vec_reg_user, 8529 attr->sample_simd_pred_reg_user); 8530 } 8531 } 8532 } 8533 8534 if (sample_type & PERF_SAMPLE_STACK_USER) { 8535 perf_output_sample_ustack(handle, 8536 data->stack_user_size, 8537 data->regs_user.regs); 8538 } 8539 8540 if (sample_type & PERF_SAMPLE_WEIGHT_TYPE) 8541 perf_output_put(handle, data->weight.full); 8542 8543 if (sample_type & PERF_SAMPLE_DATA_SRC) 8544 perf_output_put(handle, data->data_src.val); 8545 8546 if (sample_type & PERF_SAMPLE_TRANSACTION) 8547 perf_output_put(handle, data->txn); 8548 8549 if (sample_type & PERF_SAMPLE_REGS_INTR) { 8550 u64 abi = data->regs_intr.abi; 8551 /* 8552 * If there are no regs to dump, notice it through 8553 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 8554 */ 8555 perf_output_put(handle, abi); 8556 8557 if (abi) { 8558 struct perf_event_attr *attr = &event->attr; 8559 u64 mask = attr->sample_regs_intr; 8560 8561 perf_output_sample_regs(handle, 8562 data->regs_intr.regs, 8563 mask); 8564 if (abi & PERF_SAMPLE_REGS_ABI_SIMD) { 8565 perf_output_sample_simd_regs(handle, event, 8566 data->regs_intr.regs, 8567 attr->sample_simd_vec_reg_intr, 8568 attr->sample_simd_pred_reg_intr); 8569 } 8570 } 8571 } 8572 8573 if (sample_type & PERF_SAMPLE_PHYS_ADDR) 8574 perf_output_put(handle, data->phys_addr); 8575 8576 if (sample_type & PERF_SAMPLE_CGROUP) 8577 perf_output_put(handle, data->cgroup); 8578 8579 if (sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) 8580 perf_output_put(handle, data->data_page_size); 8581 8582 if (sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) 8583 perf_output_put(handle, data->code_page_size); 8584 8585 if (sample_type & PERF_SAMPLE_AUX) { 8586 perf_output_put(handle, data->aux_size); 8587 8588 if (data->aux_size) 8589 perf_aux_sample_output(event, handle, data); 8590 } 8591 8592 if (!event->attr.watermark) { 8593 int wakeup_events = event->attr.wakeup_events; 8594 8595 if (wakeup_events) { 8596 struct perf_buffer *rb = handle->rb; 8597 int events = local_inc_return(&rb->events); 8598 8599 if (events >= wakeup_events) { 8600 local_sub(wakeup_events, &rb->events); 8601 local_inc(&rb->wakeup); 8602 } 8603 } 8604 } 8605 } 8606 8607 static u64 perf_virt_to_phys(u64 virt) 8608 { 8609 u64 phys_addr = 0; 8610 8611 if (!virt) 8612 return 0; 8613 8614 if (virt >= TASK_SIZE) { 8615 /* If it's vmalloc()d memory, leave phys_addr as 0 */ 8616 if (virt_addr_valid((void *)(uintptr_t)virt) && 8617 !(virt >= VMALLOC_START && virt < VMALLOC_END)) 8618 phys_addr = (u64)virt_to_phys((void *)(uintptr_t)virt); 8619 } else { 8620 /* 8621 * Walking the pages tables for user address. 8622 * Interrupts are disabled, so it prevents any tear down 8623 * of the page tables. 8624 * Try IRQ-safe get_user_page_fast_only first. 8625 * If failed, leave phys_addr as 0. 8626 */ 8627 if (is_user_task(current)) { 8628 struct page *p; 8629 8630 pagefault_disable(); 8631 if (get_user_page_fast_only(virt, 0, &p)) { 8632 phys_addr = page_to_phys(p) + virt % PAGE_SIZE; 8633 put_page(p); 8634 } 8635 pagefault_enable(); 8636 } 8637 } 8638 8639 return phys_addr; 8640 } 8641 8642 /* 8643 * Return the pagetable size of a given virtual address. 8644 */ 8645 static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) 8646 { 8647 u64 size = 0; 8648 8649 #ifdef CONFIG_HAVE_GUP_FAST 8650 pgd_t *pgdp, pgd; 8651 p4d_t *p4dp, p4d; 8652 pud_t *pudp, pud; 8653 pmd_t *pmdp, pmd; 8654 pte_t *ptep, pte; 8655 8656 pgdp = pgd_offset(mm, addr); 8657 pgd = pgdp_get(pgdp); 8658 if (pgd_none(pgd)) 8659 return 0; 8660 8661 if (pgd_leaf(pgd)) 8662 return pgd_leaf_size(pgd); 8663 8664 p4dp = p4d_offset_lockless(pgdp, pgd, addr); 8665 p4d = p4dp_get(p4dp); 8666 if (!p4d_present(p4d)) 8667 return 0; 8668 8669 if (p4d_leaf(p4d)) 8670 return p4d_leaf_size(p4d); 8671 8672 pudp = pud_offset_lockless(p4dp, p4d, addr); 8673 pud = pudp_get(pudp); 8674 if (!pud_present(pud)) 8675 return 0; 8676 8677 if (pud_leaf(pud)) 8678 return pud_leaf_size(pud); 8679 8680 pmdp = pmd_offset_lockless(pudp, pud, addr); 8681 again: 8682 pmd = pmdp_get_lockless(pmdp); 8683 if (!pmd_present(pmd)) 8684 return 0; 8685 8686 if (pmd_leaf(pmd)) 8687 return pmd_leaf_size(pmd); 8688 8689 ptep = pte_offset_map(&pmd, addr); 8690 if (!ptep) 8691 goto again; 8692 8693 pte = ptep_get_lockless(ptep); 8694 if (pte_present(pte)) 8695 size = __pte_leaf_size(pmd, pte); 8696 pte_unmap(ptep); 8697 #endif /* CONFIG_HAVE_GUP_FAST */ 8698 8699 return size; 8700 } 8701 8702 static u64 perf_get_page_size(unsigned long addr) 8703 { 8704 struct mm_struct *mm; 8705 unsigned long flags; 8706 u64 size; 8707 8708 if (!addr) 8709 return 0; 8710 8711 /* 8712 * Software page-table walkers must disable IRQs, 8713 * which prevents any tear down of the page tables. 8714 */ 8715 local_irq_save(flags); 8716 8717 mm = current->mm; 8718 if (!mm) { 8719 /* 8720 * For kernel threads and the like, use init_mm so that 8721 * we can find kernel memory. 8722 */ 8723 mm = &init_mm; 8724 } 8725 8726 size = perf_get_pgtable_size(mm, addr); 8727 8728 local_irq_restore(flags); 8729 8730 return size; 8731 } 8732 8733 static struct perf_callchain_entry __empty_callchain = { .nr = 0, }; 8734 8735 static struct unwind_work perf_unwind_work; 8736 8737 struct perf_callchain_entry * 8738 perf_callchain(struct perf_event *event, struct pt_regs *regs) 8739 { 8740 bool kernel = !event->attr.exclude_callchain_kernel; 8741 bool user = !event->attr.exclude_callchain_user && 8742 is_user_task(current); 8743 /* Disallow cross-task user callchains. */ 8744 bool crosstask = event->ctx->task && event->ctx->task != current; 8745 bool defer_user = IS_ENABLED(CONFIG_UNWIND_USER) && user && 8746 event->attr.defer_callchain; 8747 const u32 max_stack = event->attr.sample_max_stack; 8748 struct perf_callchain_entry *callchain; 8749 u64 defer_cookie; 8750 8751 if (!current->mm) 8752 user = false; 8753 8754 if (!kernel && !user) 8755 return &__empty_callchain; 8756 8757 if (!(user && defer_user && !crosstask && 8758 unwind_deferred_request(&perf_unwind_work, &defer_cookie) >= 0)) 8759 defer_cookie = 0; 8760 8761 callchain = get_perf_callchain(regs, kernel, user, max_stack, 8762 crosstask, true, defer_cookie); 8763 8764 return callchain ?: &__empty_callchain; 8765 } 8766 8767 static __always_inline u64 __cond_set(u64 flags, u64 s, u64 d) 8768 { 8769 return d * !!(flags & s); 8770 } 8771 8772 u64 perf_update_xregs_size(struct perf_event *event, bool intr) 8773 { 8774 u16 pred_qwords = event->attr.sample_simd_pred_reg_qwords; 8775 u16 vec_qwords = event->attr.sample_simd_vec_reg_qwords; 8776 u64 pred_mask; 8777 u64 mask; 8778 int size; 8779 8780 if (intr) { 8781 mask = event->attr.sample_simd_vec_reg_intr; 8782 pred_mask = event->attr.sample_simd_pred_reg_intr; 8783 } else { 8784 mask = event->attr.sample_simd_vec_reg_user; 8785 pred_mask = event->attr.sample_simd_pred_reg_user; 8786 } 8787 8788 size = sizeof(u64) * 4; 8789 size += (hweight64(mask) * vec_qwords + 8790 hweight64(pred_mask) * pred_qwords) * sizeof(u64); 8791 8792 return size; 8793 } 8794 8795 void perf_prepare_sample(struct perf_sample_data *data, 8796 struct perf_event *event, 8797 struct pt_regs *regs) 8798 { 8799 u64 sample_type = event->attr.sample_type; 8800 u64 filtered_sample_type; 8801 8802 /* 8803 * Add the sample flags that are dependent to others. And clear the 8804 * sample flags that have already been done by the PMU driver. 8805 */ 8806 filtered_sample_type = sample_type; 8807 filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_CODE_PAGE_SIZE, 8808 PERF_SAMPLE_IP); 8809 filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_DATA_PAGE_SIZE | 8810 PERF_SAMPLE_PHYS_ADDR, PERF_SAMPLE_ADDR); 8811 filtered_sample_type |= __cond_set(sample_type, PERF_SAMPLE_STACK_USER, 8812 PERF_SAMPLE_REGS_USER); 8813 filtered_sample_type &= ~data->sample_flags; 8814 8815 if (filtered_sample_type == 0) { 8816 /* Make sure it has the correct data->type for output */ 8817 data->type = event->attr.sample_type; 8818 return; 8819 } 8820 8821 __perf_event_header__init_id(data, event, filtered_sample_type); 8822 8823 if (filtered_sample_type & PERF_SAMPLE_IP) { 8824 data->ip = perf_instruction_pointer(event, regs); 8825 data->sample_flags |= PERF_SAMPLE_IP; 8826 } 8827 8828 if (filtered_sample_type & PERF_SAMPLE_CALLCHAIN) 8829 perf_sample_save_callchain(data, event, regs); 8830 8831 if (filtered_sample_type & PERF_SAMPLE_RAW) { 8832 data->raw = NULL; 8833 data->dyn_size += sizeof(u64); 8834 data->sample_flags |= PERF_SAMPLE_RAW; 8835 } 8836 8837 if (filtered_sample_type & PERF_SAMPLE_BRANCH_STACK) { 8838 data->br_stack = NULL; 8839 data->dyn_size += sizeof(u64); 8840 data->sample_flags |= PERF_SAMPLE_BRANCH_STACK; 8841 } 8842 8843 if (filtered_sample_type & PERF_SAMPLE_REGS_USER) 8844 perf_sample_regs_user(&data->regs_user, regs); 8845 8846 /* 8847 * It cannot use the filtered_sample_type here as REGS_USER can be set 8848 * by STACK_USER (using __cond_set() above) and we don't want to update 8849 * the dyn_size if it's not requested by users. 8850 */ 8851 if ((sample_type & ~data->sample_flags) & PERF_SAMPLE_REGS_USER) { 8852 /* regs dump ABI info */ 8853 int size = sizeof(u64); 8854 8855 if (data->regs_user.regs) { 8856 u64 mask = event->attr.sample_regs_user; 8857 8858 size += hweight64(mask) * sizeof(u64); 8859 if (event_has_simd_regs(event)) { 8860 size += perf_update_xregs_size(event, false); 8861 data->regs_user.abi |= PERF_SAMPLE_REGS_ABI_SIMD; 8862 } 8863 } 8864 8865 data->dyn_size += size; 8866 data->sample_flags |= PERF_SAMPLE_REGS_USER; 8867 } 8868 8869 if (filtered_sample_type & PERF_SAMPLE_STACK_USER) { 8870 /* 8871 * Either we need PERF_SAMPLE_STACK_USER bit to be always 8872 * processed as the last one or have additional check added 8873 * in case new sample type is added, because we could eat 8874 * up the rest of the sample size. 8875 */ 8876 u16 stack_size = event->attr.sample_stack_user; 8877 u16 header_size = perf_sample_data_size(data, event); 8878 u16 size = sizeof(u64); 8879 8880 stack_size = perf_sample_ustack_size(stack_size, header_size, 8881 data->regs_user.regs); 8882 8883 /* 8884 * If there is something to dump, add space for the dump 8885 * itself and for the field that tells the dynamic size, 8886 * which is how many have been actually dumped. 8887 */ 8888 if (stack_size) 8889 size += sizeof(u64) + stack_size; 8890 8891 data->stack_user_size = stack_size; 8892 data->dyn_size += size; 8893 data->sample_flags |= PERF_SAMPLE_STACK_USER; 8894 } 8895 8896 if (filtered_sample_type & PERF_SAMPLE_WEIGHT_TYPE) { 8897 data->weight.full = 0; 8898 data->sample_flags |= PERF_SAMPLE_WEIGHT_TYPE; 8899 } 8900 8901 if (filtered_sample_type & PERF_SAMPLE_DATA_SRC) { 8902 data->data_src.val = PERF_MEM_NA; 8903 data->sample_flags |= PERF_SAMPLE_DATA_SRC; 8904 } 8905 8906 if (filtered_sample_type & PERF_SAMPLE_TRANSACTION) { 8907 data->txn = 0; 8908 data->sample_flags |= PERF_SAMPLE_TRANSACTION; 8909 } 8910 8911 if (filtered_sample_type & PERF_SAMPLE_ADDR) { 8912 data->addr = 0; 8913 data->sample_flags |= PERF_SAMPLE_ADDR; 8914 } 8915 8916 if (filtered_sample_type & PERF_SAMPLE_REGS_INTR) { 8917 /* regs dump ABI info */ 8918 int size = sizeof(u64); 8919 8920 perf_sample_regs_intr(&data->regs_intr, regs, 8921 event->attr.exclude_kernel); 8922 8923 if (data->regs_intr.regs) { 8924 u64 mask = event->attr.sample_regs_intr; 8925 8926 size += hweight64(mask) * sizeof(u64); 8927 if (event_has_simd_regs(event)) { 8928 size += perf_update_xregs_size(event, true); 8929 data->regs_intr.abi |= PERF_SAMPLE_REGS_ABI_SIMD; 8930 } 8931 } 8932 8933 data->dyn_size += size; 8934 data->sample_flags |= PERF_SAMPLE_REGS_INTR; 8935 } 8936 8937 if (filtered_sample_type & PERF_SAMPLE_PHYS_ADDR) { 8938 data->phys_addr = perf_virt_to_phys(data->addr); 8939 data->sample_flags |= PERF_SAMPLE_PHYS_ADDR; 8940 } 8941 8942 #ifdef CONFIG_CGROUP_PERF 8943 if (filtered_sample_type & PERF_SAMPLE_CGROUP) { 8944 struct cgroup *cgrp; 8945 8946 /* protected by RCU */ 8947 cgrp = task_css_check(current, perf_event_cgrp_id, 1)->cgroup; 8948 data->cgroup = cgroup_id(cgrp); 8949 data->sample_flags |= PERF_SAMPLE_CGROUP; 8950 } 8951 #endif 8952 8953 /* 8954 * PERF_DATA_PAGE_SIZE requires PERF_SAMPLE_ADDR. If the user doesn't 8955 * require PERF_SAMPLE_ADDR, kernel implicitly retrieve the data->addr, 8956 * but the value will not dump to the userspace. 8957 */ 8958 if (filtered_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) { 8959 data->data_page_size = perf_get_page_size(data->addr); 8960 data->sample_flags |= PERF_SAMPLE_DATA_PAGE_SIZE; 8961 } 8962 8963 if (filtered_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) { 8964 data->code_page_size = perf_get_page_size(data->ip); 8965 data->sample_flags |= PERF_SAMPLE_CODE_PAGE_SIZE; 8966 } 8967 8968 if (filtered_sample_type & PERF_SAMPLE_AUX) { 8969 u64 size; 8970 u16 header_size = perf_sample_data_size(data, event); 8971 8972 header_size += sizeof(u64); /* size */ 8973 8974 /* 8975 * Given the 16bit nature of header::size, an AUX sample can 8976 * easily overflow it, what with all the preceding sample bits. 8977 * Make sure this doesn't happen by using up to U16_MAX bytes 8978 * per sample in total (rounded down to 8 byte boundary). 8979 */ 8980 size = min_t(size_t, U16_MAX - header_size, 8981 event->attr.aux_sample_size); 8982 size = rounddown(size, 8); 8983 size = perf_prepare_sample_aux(event, data, size); 8984 8985 WARN_ON_ONCE(size + header_size > U16_MAX); 8986 data->dyn_size += size + sizeof(u64); /* size above */ 8987 data->sample_flags |= PERF_SAMPLE_AUX; 8988 } 8989 } 8990 8991 void perf_prepare_header(struct perf_event_header *header, 8992 struct perf_sample_data *data, 8993 struct perf_event *event, 8994 struct pt_regs *regs) 8995 { 8996 header->type = PERF_RECORD_SAMPLE; 8997 header->size = perf_sample_data_size(data, event); 8998 header->misc = perf_misc_flags(event, regs); 8999 9000 /* 9001 * If you're adding more sample types here, you likely need to do 9002 * something about the overflowing header::size, like repurpose the 9003 * lowest 3 bits of size, which should be always zero at the moment. 9004 * This raises a more important question, do we really need 512k sized 9005 * samples and why, so good argumentation is in order for whatever you 9006 * do here next. 9007 */ 9008 WARN_ON_ONCE(header->size & 7); 9009 } 9010 9011 static void __perf_event_aux_pause(struct perf_event *event, bool pause) 9012 { 9013 if (pause) { 9014 if (!event->hw.aux_paused) { 9015 event->hw.aux_paused = 1; 9016 event->pmu->stop(event, PERF_EF_PAUSE); 9017 } 9018 } else { 9019 if (event->hw.aux_paused) { 9020 event->hw.aux_paused = 0; 9021 event->pmu->start(event, PERF_EF_RESUME); 9022 } 9023 } 9024 } 9025 9026 static void perf_event_aux_pause(struct perf_event *event, bool pause) 9027 { 9028 struct perf_buffer *rb; 9029 9030 if (WARN_ON_ONCE(!event)) 9031 return; 9032 9033 rb = ring_buffer_get(event); 9034 if (!rb) 9035 return; 9036 9037 scoped_guard (irqsave) { 9038 /* 9039 * Guard against self-recursion here. Another event could trip 9040 * this same from NMI context. 9041 */ 9042 if (READ_ONCE(rb->aux_in_pause_resume)) 9043 break; 9044 9045 WRITE_ONCE(rb->aux_in_pause_resume, 1); 9046 barrier(); 9047 __perf_event_aux_pause(event, pause); 9048 barrier(); 9049 WRITE_ONCE(rb->aux_in_pause_resume, 0); 9050 } 9051 ring_buffer_put(rb); 9052 } 9053 9054 static __always_inline int 9055 __perf_event_output(struct perf_event *event, 9056 struct perf_sample_data *data, 9057 struct pt_regs *regs, 9058 int (*output_begin)(struct perf_output_handle *, 9059 struct perf_sample_data *, 9060 struct perf_event *, 9061 unsigned int)) 9062 { 9063 struct perf_output_handle handle; 9064 struct perf_event_header header; 9065 int err; 9066 9067 /* protect the callchain buffers */ 9068 rcu_read_lock(); 9069 9070 perf_prepare_sample(data, event, regs); 9071 perf_prepare_header(&header, data, event, regs); 9072 9073 err = output_begin(&handle, data, event, header.size); 9074 if (err) 9075 goto exit; 9076 9077 perf_output_sample(&handle, &header, data, event); 9078 9079 perf_output_end(&handle); 9080 9081 exit: 9082 rcu_read_unlock(); 9083 return err; 9084 } 9085 9086 void 9087 perf_event_output_forward(struct perf_event *event, 9088 struct perf_sample_data *data, 9089 struct pt_regs *regs) 9090 { 9091 __perf_event_output(event, data, regs, perf_output_begin_forward); 9092 } 9093 9094 void 9095 perf_event_output_backward(struct perf_event *event, 9096 struct perf_sample_data *data, 9097 struct pt_regs *regs) 9098 { 9099 __perf_event_output(event, data, regs, perf_output_begin_backward); 9100 } 9101 9102 int 9103 perf_event_output(struct perf_event *event, 9104 struct perf_sample_data *data, 9105 struct pt_regs *regs) 9106 { 9107 return __perf_event_output(event, data, regs, perf_output_begin); 9108 } 9109 9110 /* 9111 * read event_id 9112 */ 9113 9114 struct perf_read_event { 9115 struct perf_event_header header; 9116 9117 u32 pid; 9118 u32 tid; 9119 }; 9120 9121 static void 9122 perf_event_read_event(struct perf_event *event, 9123 struct task_struct *task) 9124 { 9125 struct perf_output_handle handle; 9126 struct perf_sample_data sample; 9127 struct perf_read_event read_event = { 9128 .pid = perf_event_pid(event, task), 9129 .tid = perf_event_tid(event, task), 9130 }; 9131 int ret; 9132 9133 perf_event_header__init(&read_event.header, &sample, 9134 PERF_RECORD_READ, 9135 /* misc= */ 0, 9136 sizeof(read_event) + event->read_size, 9137 event); 9138 ret = perf_output_begin(&handle, &sample, event, read_event.header.size); 9139 if (ret) 9140 return; 9141 9142 perf_output_put(&handle, read_event); 9143 perf_output_read(&handle, event); 9144 perf_event__output_id_sample(event, &handle, &sample); 9145 9146 perf_output_end(&handle); 9147 } 9148 9149 typedef void (perf_iterate_f)(struct perf_event *event, void *data); 9150 9151 static void 9152 perf_iterate_ctx(struct perf_event_context *ctx, 9153 perf_iterate_f output, 9154 void *data, bool all) 9155 { 9156 struct perf_event *event; 9157 9158 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 9159 if (!all) { 9160 if (event->state < PERF_EVENT_STATE_INACTIVE) 9161 continue; 9162 if (!event_filter_match(event)) 9163 continue; 9164 } 9165 9166 output(event, data); 9167 } 9168 } 9169 9170 static void perf_iterate_sb_cpu(perf_iterate_f output, void *data) 9171 { 9172 struct pmu_event_list *pel = this_cpu_ptr(&pmu_sb_events); 9173 struct perf_event *event; 9174 9175 list_for_each_entry_rcu(event, &pel->list, sb_list) { 9176 /* 9177 * Skip events that are not fully formed yet; ensure that 9178 * if we observe event->ctx, both event and ctx will be 9179 * complete enough. See perf_install_in_context(). 9180 */ 9181 if (!smp_load_acquire(&event->ctx)) 9182 continue; 9183 9184 if (event->state < PERF_EVENT_STATE_INACTIVE) 9185 continue; 9186 if (!event_filter_match(event)) 9187 continue; 9188 output(event, data); 9189 } 9190 } 9191 9192 /* 9193 * Iterate all events that need to receive side-band events. 9194 * 9195 * For new callers; ensure that account_pmu_sb_event() includes 9196 * your event, otherwise it might not get delivered. 9197 */ 9198 static void 9199 perf_iterate_sb(perf_iterate_f output, void *data, 9200 struct perf_event_context *task_ctx) 9201 { 9202 struct perf_event_context *ctx; 9203 9204 rcu_read_lock(); 9205 preempt_disable(); 9206 9207 /* 9208 * If we have task_ctx != NULL we only notify the task context itself. 9209 * The task_ctx is set only for EXIT events before releasing task 9210 * context. 9211 */ 9212 if (task_ctx) { 9213 perf_iterate_ctx(task_ctx, output, data, false); 9214 goto done; 9215 } 9216 9217 perf_iterate_sb_cpu(output, data); 9218 9219 ctx = rcu_dereference(current->perf_event_ctxp); 9220 if (ctx) 9221 perf_iterate_ctx(ctx, output, data, false); 9222 done: 9223 preempt_enable(); 9224 rcu_read_unlock(); 9225 } 9226 9227 /* 9228 * Clear all file-based filters at exec, they'll have to be 9229 * re-instated when/if these objects are mmapped again. 9230 */ 9231 static void perf_event_addr_filters_exec(struct perf_event *event, void *data) 9232 { 9233 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 9234 struct perf_addr_filter *filter; 9235 unsigned int restart = 0, count = 0; 9236 unsigned long flags; 9237 9238 if (!has_addr_filter(event)) 9239 return; 9240 9241 raw_spin_lock_irqsave(&ifh->lock, flags); 9242 list_for_each_entry(filter, &ifh->list, entry) { 9243 if (filter->path.dentry) { 9244 event->addr_filter_ranges[count].start = 0; 9245 event->addr_filter_ranges[count].size = 0; 9246 restart++; 9247 } 9248 9249 count++; 9250 } 9251 9252 if (restart) 9253 event->addr_filters_gen++; 9254 raw_spin_unlock_irqrestore(&ifh->lock, flags); 9255 9256 if (restart) 9257 perf_event_stop(event, 1); 9258 } 9259 9260 void perf_event_exec(void) 9261 { 9262 struct perf_event_context *ctx; 9263 9264 ctx = perf_pin_task_context(current); 9265 if (!ctx) 9266 return; 9267 9268 perf_event_enable_on_exec(ctx); 9269 perf_event_remove_on_exec(ctx); 9270 scoped_guard(rcu) 9271 perf_iterate_ctx(ctx, perf_event_addr_filters_exec, NULL, true); 9272 9273 perf_unpin_context(ctx); 9274 put_ctx(ctx); 9275 } 9276 9277 struct remote_output { 9278 struct perf_buffer *rb; 9279 int err; 9280 }; 9281 9282 static void __perf_event_output_stop(struct perf_event *event, void *data) 9283 { 9284 struct perf_event *parent = event->parent; 9285 struct remote_output *ro = data; 9286 struct perf_buffer *rb = ro->rb; 9287 struct stop_event_data sd = { 9288 .event = event, 9289 }; 9290 9291 if (!has_aux(event)) 9292 return; 9293 9294 if (!parent) 9295 parent = event; 9296 9297 /* 9298 * In case of inheritance, it will be the parent that links to the 9299 * ring-buffer, but it will be the child that's actually using it. 9300 * 9301 * We are using event::rb to determine if the event should be stopped, 9302 * however this may race with ring_buffer_attach() (through set_output), 9303 * which will make us skip the event that actually needs to be stopped. 9304 * So ring_buffer_attach() has to stop an aux event before re-assigning 9305 * its rb pointer. 9306 */ 9307 if (rcu_dereference(parent->rb) == rb) 9308 ro->err = __perf_event_stop(&sd); 9309 } 9310 9311 static int __perf_pmu_output_stop(void *info) 9312 { 9313 struct perf_event *event = info; 9314 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 9315 struct remote_output ro = { 9316 .rb = event->rb, 9317 }; 9318 9319 rcu_read_lock(); 9320 perf_iterate_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false); 9321 if (cpuctx->task_ctx) 9322 perf_iterate_ctx(cpuctx->task_ctx, __perf_event_output_stop, 9323 &ro, false); 9324 rcu_read_unlock(); 9325 9326 return ro.err; 9327 } 9328 9329 static void perf_pmu_output_stop(struct perf_event *event) 9330 { 9331 struct perf_event *iter; 9332 int err, cpu; 9333 9334 restart: 9335 rcu_read_lock(); 9336 list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) { 9337 /* 9338 * For per-CPU events, we need to make sure that neither they 9339 * nor their children are running; for cpu==-1 events it's 9340 * sufficient to stop the event itself if it's active, since 9341 * it can't have children. 9342 */ 9343 cpu = iter->cpu; 9344 if (cpu == -1) 9345 cpu = READ_ONCE(iter->oncpu); 9346 9347 if (cpu == -1) 9348 continue; 9349 9350 err = cpu_function_call(cpu, __perf_pmu_output_stop, event); 9351 if (err == -EAGAIN) { 9352 rcu_read_unlock(); 9353 goto restart; 9354 } 9355 } 9356 rcu_read_unlock(); 9357 } 9358 9359 /* 9360 * task tracking -- fork/exit 9361 * 9362 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task 9363 */ 9364 9365 struct perf_task_event { 9366 struct task_struct *task; 9367 struct perf_event_context *task_ctx; 9368 9369 struct { 9370 struct perf_event_header header; 9371 9372 u32 pid; 9373 u32 ppid; 9374 u32 tid; 9375 u32 ptid; 9376 u64 time; 9377 } event_id; 9378 int new; 9379 }; 9380 9381 static int perf_event_task_match(struct perf_event *event) 9382 { 9383 return event->attr.comm || event->attr.mmap || 9384 event->attr.mmap2 || event->attr.mmap_data || 9385 event->attr.task; 9386 } 9387 9388 static void perf_event_task_output(struct perf_event *event, 9389 void *data) 9390 { 9391 struct perf_task_event *task_event = data; 9392 struct perf_output_handle handle; 9393 struct perf_sample_data sample; 9394 struct task_struct *task = task_event->task; 9395 int ret; 9396 9397 if (!perf_event_task_match(event)) 9398 return; 9399 9400 perf_event_header__init(&task_event->event_id.header, &sample, 9401 task_event->new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, 9402 /* misc= */ 0, 9403 sizeof(task_event->event_id), 9404 event); 9405 9406 ret = perf_output_begin(&handle, &sample, event, 9407 task_event->event_id.header.size); 9408 if (ret) 9409 return; 9410 9411 task_event->event_id.pid = perf_event_pid(event, task); 9412 task_event->event_id.tid = perf_event_tid(event, task); 9413 9414 if (task_event->event_id.header.type == PERF_RECORD_EXIT) { 9415 task_event->event_id.ppid = perf_event_pid(event, 9416 task->real_parent); 9417 task_event->event_id.ptid = perf_event_pid(event, 9418 task->real_parent); 9419 } else { /* PERF_RECORD_FORK */ 9420 task_event->event_id.ppid = perf_event_pid(event, current); 9421 task_event->event_id.ptid = perf_event_tid(event, current); 9422 } 9423 9424 task_event->event_id.time = perf_event_clock(event); 9425 9426 perf_output_put(&handle, task_event->event_id); 9427 9428 perf_event__output_id_sample(event, &handle, &sample); 9429 9430 perf_output_end(&handle); 9431 } 9432 9433 static void perf_event_task(struct task_struct *task, 9434 struct perf_event_context *task_ctx, 9435 int new) 9436 { 9437 struct perf_task_event task_event; 9438 9439 if (!atomic_read(&nr_comm_events) && 9440 !atomic_read(&nr_mmap_events) && 9441 !atomic_read(&nr_task_events)) 9442 return; 9443 9444 task_event = (struct perf_task_event){ 9445 .task = task, 9446 .task_ctx = task_ctx, 9447 .new = new, 9448 }; 9449 9450 perf_iterate_sb(perf_event_task_output, 9451 &task_event, 9452 task_ctx); 9453 } 9454 9455 /* 9456 * Allocate data for a new task when profiling system-wide 9457 * events which require PMU specific data 9458 */ 9459 static void 9460 perf_event_alloc_task_data(struct task_struct *child, 9461 struct task_struct *parent) 9462 { 9463 struct kmem_cache *ctx_cache = NULL; 9464 struct perf_ctx_data *cd; 9465 9466 if (!refcount_read(&global_ctx_data_ref)) 9467 return; 9468 9469 scoped_guard (rcu) { 9470 cd = rcu_dereference(parent->perf_ctx_data); 9471 if (cd) 9472 ctx_cache = cd->ctx_cache; 9473 } 9474 9475 if (!ctx_cache) 9476 return; 9477 9478 guard(percpu_read)(&global_ctx_data_rwsem); 9479 scoped_guard (rcu) { 9480 cd = rcu_dereference(child->perf_ctx_data); 9481 if (!cd) { 9482 /* 9483 * A system-wide event may be unaccount, 9484 * when attaching the perf_ctx_data. 9485 */ 9486 if (!refcount_read(&global_ctx_data_ref)) 9487 return; 9488 goto attach; 9489 } 9490 9491 if (!cd->global) { 9492 cd->global = 1; 9493 refcount_inc(&cd->refcount); 9494 } 9495 } 9496 9497 return; 9498 attach: 9499 attach_task_ctx_data(child, ctx_cache, true, GFP_KERNEL); 9500 } 9501 9502 void perf_event_fork(struct task_struct *task) 9503 { 9504 perf_event_task(task, NULL, 1); 9505 perf_event_namespaces(task); 9506 perf_event_alloc_task_data(task, current); 9507 } 9508 9509 /* 9510 * comm tracking 9511 */ 9512 9513 struct perf_comm_event { 9514 struct task_struct *task; 9515 char *comm; 9516 int comm_size; 9517 9518 struct { 9519 struct perf_event_header header; 9520 9521 u32 pid; 9522 u32 tid; 9523 } event_id; 9524 bool exec; 9525 }; 9526 9527 static int perf_event_comm_match(struct perf_event *event) 9528 { 9529 return event->attr.comm; 9530 } 9531 9532 static void perf_event_comm_output(struct perf_event *event, 9533 void *data) 9534 { 9535 struct perf_comm_event *comm_event = data; 9536 struct perf_output_handle handle; 9537 struct perf_sample_data sample; 9538 int ret; 9539 9540 if (!perf_event_comm_match(event)) 9541 return; 9542 9543 perf_event_header__init(&comm_event->event_id.header, &sample, 9544 PERF_RECORD_COMM, 9545 comm_event->exec ? PERF_RECORD_MISC_COMM_EXEC : 0, 9546 sizeof(comm_event->event_id) + comm_event->comm_size, 9547 event); 9548 ret = perf_output_begin(&handle, &sample, event, 9549 comm_event->event_id.header.size); 9550 9551 if (ret) 9552 return; 9553 9554 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); 9555 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); 9556 9557 perf_output_put(&handle, comm_event->event_id); 9558 __output_copy(&handle, comm_event->comm, 9559 comm_event->comm_size); 9560 9561 perf_event__output_id_sample(event, &handle, &sample); 9562 9563 perf_output_end(&handle); 9564 } 9565 9566 static void perf_event_comm_event(struct perf_comm_event *comm_event) 9567 { 9568 char comm[TASK_COMM_LEN]; 9569 unsigned int size; 9570 9571 memset(comm, 0, sizeof(comm)); 9572 strscpy(comm, comm_event->task->comm); 9573 size = ALIGN(strlen(comm)+1, sizeof(u64)); 9574 9575 comm_event->comm = comm; 9576 comm_event->comm_size = size; 9577 9578 perf_iterate_sb(perf_event_comm_output, 9579 comm_event, 9580 NULL); 9581 } 9582 9583 void perf_event_comm(struct task_struct *task, bool exec) 9584 { 9585 struct perf_comm_event comm_event; 9586 9587 if (!atomic_read(&nr_comm_events)) 9588 return; 9589 9590 comm_event = (struct perf_comm_event){ 9591 .task = task, 9592 /* .comm */ 9593 /* .comm_size */ 9594 /* .event_id */ 9595 .exec = exec, 9596 }; 9597 9598 perf_event_comm_event(&comm_event); 9599 } 9600 9601 /* 9602 * namespaces tracking 9603 */ 9604 9605 struct perf_namespaces_event { 9606 struct task_struct *task; 9607 9608 struct { 9609 struct perf_event_header header; 9610 9611 u32 pid; 9612 u32 tid; 9613 u64 nr_namespaces; 9614 struct perf_ns_link_info link_info[NR_NAMESPACES]; 9615 } event_id; 9616 }; 9617 9618 static int perf_event_namespaces_match(struct perf_event *event) 9619 { 9620 return event->attr.namespaces; 9621 } 9622 9623 static void perf_event_namespaces_output(struct perf_event *event, 9624 void *data) 9625 { 9626 struct perf_namespaces_event *namespaces_event = data; 9627 struct perf_output_handle handle; 9628 struct perf_sample_data sample; 9629 int ret; 9630 9631 if (!perf_event_namespaces_match(event)) 9632 return; 9633 9634 perf_event_header__init(&namespaces_event->event_id.header, &sample, 9635 PERF_RECORD_NAMESPACES, 9636 /* misc= */ 0, 9637 sizeof(namespaces_event->event_id), 9638 event); 9639 ret = perf_output_begin(&handle, &sample, event, 9640 namespaces_event->event_id.header.size); 9641 if (ret) 9642 return; 9643 9644 namespaces_event->event_id.pid = perf_event_pid(event, 9645 namespaces_event->task); 9646 namespaces_event->event_id.tid = perf_event_tid(event, 9647 namespaces_event->task); 9648 9649 perf_output_put(&handle, namespaces_event->event_id); 9650 9651 perf_event__output_id_sample(event, &handle, &sample); 9652 9653 perf_output_end(&handle); 9654 } 9655 9656 static void perf_fill_ns_link_info(struct perf_ns_link_info *ns_link_info, 9657 struct task_struct *task, 9658 const struct proc_ns_operations *ns_ops) 9659 { 9660 struct path ns_path; 9661 struct inode *ns_inode; 9662 int error; 9663 9664 error = ns_get_path(&ns_path, task, ns_ops); 9665 if (!error) { 9666 ns_inode = ns_path.dentry->d_inode; 9667 ns_link_info->dev = new_encode_dev(ns_inode->i_sb->s_dev); 9668 ns_link_info->ino = ns_inode->i_ino; 9669 path_put(&ns_path); 9670 } 9671 } 9672 9673 void perf_event_namespaces(struct task_struct *task) 9674 { 9675 struct perf_namespaces_event namespaces_event; 9676 struct perf_ns_link_info *ns_link_info; 9677 9678 if (!atomic_read(&nr_namespaces_events)) 9679 return; 9680 9681 namespaces_event = (struct perf_namespaces_event){ 9682 .task = task, 9683 .event_id = { 9684 /* .header */ 9685 /* .pid */ 9686 /* .tid */ 9687 .nr_namespaces = NR_NAMESPACES, 9688 /* .link_info[NR_NAMESPACES] */ 9689 }, 9690 }; 9691 9692 ns_link_info = namespaces_event.event_id.link_info; 9693 9694 perf_fill_ns_link_info(&ns_link_info[MNT_NS_INDEX], 9695 task, &mntns_operations); 9696 9697 #ifdef CONFIG_USER_NS 9698 perf_fill_ns_link_info(&ns_link_info[USER_NS_INDEX], 9699 task, &userns_operations); 9700 #endif 9701 #ifdef CONFIG_NET_NS 9702 perf_fill_ns_link_info(&ns_link_info[NET_NS_INDEX], 9703 task, &netns_operations); 9704 #endif 9705 #ifdef CONFIG_UTS_NS 9706 perf_fill_ns_link_info(&ns_link_info[UTS_NS_INDEX], 9707 task, &utsns_operations); 9708 #endif 9709 #ifdef CONFIG_IPC_NS 9710 perf_fill_ns_link_info(&ns_link_info[IPC_NS_INDEX], 9711 task, &ipcns_operations); 9712 #endif 9713 #ifdef CONFIG_PID_NS 9714 perf_fill_ns_link_info(&ns_link_info[PID_NS_INDEX], 9715 task, &pidns_operations); 9716 #endif 9717 #ifdef CONFIG_CGROUPS 9718 perf_fill_ns_link_info(&ns_link_info[CGROUP_NS_INDEX], 9719 task, &cgroupns_operations); 9720 #endif 9721 9722 perf_iterate_sb(perf_event_namespaces_output, 9723 &namespaces_event, 9724 NULL); 9725 } 9726 9727 /* 9728 * cgroup tracking 9729 */ 9730 #ifdef CONFIG_CGROUP_PERF 9731 9732 struct perf_cgroup_event { 9733 char *path; 9734 int path_size; 9735 struct { 9736 struct perf_event_header header; 9737 u64 id; 9738 char path[]; 9739 } event_id; 9740 }; 9741 9742 static int perf_event_cgroup_match(struct perf_event *event) 9743 { 9744 return event->attr.cgroup; 9745 } 9746 9747 static void perf_event_cgroup_output(struct perf_event *event, void *data) 9748 { 9749 struct perf_cgroup_event *cgroup_event = data; 9750 struct perf_output_handle handle; 9751 struct perf_sample_data sample; 9752 int ret; 9753 u16 size = sizeof(cgroup_event->event_id) + cgroup_event->path_size; 9754 9755 if (!perf_event_cgroup_match(event)) 9756 return; 9757 9758 perf_event_header__init(&cgroup_event->event_id.header, &sample, 9759 PERF_RECORD_CGROUP, /* misc= */ 0, size, 9760 event); 9761 ret = perf_output_begin(&handle, &sample, event, 9762 cgroup_event->event_id.header.size); 9763 if (ret) 9764 return; 9765 9766 perf_output_put(&handle, cgroup_event->event_id); 9767 __output_copy(&handle, cgroup_event->path, cgroup_event->path_size); 9768 9769 perf_event__output_id_sample(event, &handle, &sample); 9770 9771 perf_output_end(&handle); 9772 } 9773 9774 static void perf_event_cgroup(struct cgroup *cgrp) 9775 { 9776 struct perf_cgroup_event cgroup_event; 9777 char path_enomem[16] = "//enomem"; 9778 char *pathname; 9779 size_t size; 9780 9781 if (!atomic_read(&nr_cgroup_events)) 9782 return; 9783 9784 cgroup_event = (struct perf_cgroup_event){ 9785 .event_id = { 9786 .id = cgroup_id(cgrp), 9787 }, 9788 }; 9789 9790 pathname = kmalloc(PATH_MAX, GFP_KERNEL); 9791 if (pathname == NULL) { 9792 cgroup_event.path = path_enomem; 9793 } else { 9794 /* just to be sure to have enough space for alignment */ 9795 cgroup_path(cgrp, pathname, PATH_MAX - sizeof(u64)); 9796 cgroup_event.path = pathname; 9797 } 9798 9799 /* 9800 * Since our buffer works in 8 byte units we need to align our string 9801 * size to a multiple of 8. However, we must guarantee the tail end is 9802 * zero'd out to avoid leaking random bits to userspace. 9803 */ 9804 size = strlen(cgroup_event.path) + 1; 9805 while (!IS_ALIGNED(size, sizeof(u64))) 9806 cgroup_event.path[size++] = '\0'; 9807 9808 cgroup_event.path_size = size; 9809 9810 perf_iterate_sb(perf_event_cgroup_output, 9811 &cgroup_event, 9812 NULL); 9813 9814 kfree(pathname); 9815 } 9816 9817 #endif 9818 9819 /* 9820 * mmap tracking 9821 */ 9822 9823 struct perf_mmap_event { 9824 struct vm_area_struct *vma; 9825 9826 const char *file_name; 9827 int file_size; 9828 int maj, min; 9829 u64 ino; 9830 u64 ino_generation; 9831 u32 prot, flags; 9832 u8 build_id[BUILD_ID_SIZE_MAX]; 9833 u32 build_id_size; 9834 9835 struct { 9836 struct perf_event_header header; 9837 9838 u32 pid; 9839 u32 tid; 9840 u64 start; 9841 u64 len; 9842 u64 pgoff; 9843 } event_id; 9844 }; 9845 9846 static int perf_event_mmap_match(struct perf_event *event, 9847 void *data) 9848 { 9849 struct perf_mmap_event *mmap_event = data; 9850 struct vm_area_struct *vma = mmap_event->vma; 9851 int executable = vma->vm_flags & VM_EXEC; 9852 9853 return (!executable && event->attr.mmap_data) || 9854 (executable && (event->attr.mmap || event->attr.mmap2)); 9855 } 9856 9857 static void perf_event_mmap_output(struct perf_event *event, 9858 void *data) 9859 { 9860 struct perf_mmap_event *mmap_event = data; 9861 struct perf_output_handle handle; 9862 struct perf_sample_data sample; 9863 int size = sizeof(mmap_event->event_id) + mmap_event->file_size; 9864 u32 type = PERF_RECORD_MMAP; 9865 u16 misc = PERF_RECORD_MISC_USER; 9866 bool use_build_id = false; 9867 int ret; 9868 9869 if (!perf_event_mmap_match(event, data)) 9870 return; 9871 9872 if (event->attr.mmap2) { 9873 type = PERF_RECORD_MMAP2; 9874 size += sizeof(mmap_event->maj); 9875 size += sizeof(mmap_event->min); 9876 size += sizeof(mmap_event->ino); 9877 size += sizeof(mmap_event->ino_generation); 9878 size += sizeof(mmap_event->prot); 9879 size += sizeof(mmap_event->flags); 9880 use_build_id = event->attr.build_id && mmap_event->build_id_size; 9881 if (use_build_id) 9882 misc |= PERF_RECORD_MISC_MMAP_BUILD_ID; 9883 } 9884 if (!(mmap_event->vma->vm_flags & VM_EXEC)) 9885 misc |= PERF_RECORD_MISC_MMAP_DATA; 9886 9887 perf_event_header__init(&mmap_event->event_id.header, &sample, 9888 type, misc, size, event); 9889 ret = perf_output_begin(&handle, &sample, event, 9890 mmap_event->event_id.header.size); 9891 if (ret) 9892 return; 9893 9894 mmap_event->event_id.pid = perf_event_pid(event, current); 9895 mmap_event->event_id.tid = perf_event_tid(event, current); 9896 9897 perf_output_put(&handle, mmap_event->event_id); 9898 9899 if (event->attr.mmap2) { 9900 if (use_build_id) { 9901 u8 size[4] = { (u8) mmap_event->build_id_size, 0, 0, 0 }; 9902 9903 __output_copy(&handle, size, 4); 9904 __output_copy(&handle, mmap_event->build_id, BUILD_ID_SIZE_MAX); 9905 } else { 9906 perf_output_put(&handle, mmap_event->maj); 9907 perf_output_put(&handle, mmap_event->min); 9908 perf_output_put(&handle, mmap_event->ino); 9909 perf_output_put(&handle, mmap_event->ino_generation); 9910 } 9911 perf_output_put(&handle, mmap_event->prot); 9912 perf_output_put(&handle, mmap_event->flags); 9913 } 9914 9915 __output_copy(&handle, mmap_event->file_name, 9916 mmap_event->file_size); 9917 9918 perf_event__output_id_sample(event, &handle, &sample); 9919 9920 perf_output_end(&handle); 9921 } 9922 9923 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) 9924 { 9925 struct vm_area_struct *vma = mmap_event->vma; 9926 struct file *file = vma->vm_file; 9927 int maj = 0, min = 0; 9928 u64 ino = 0, gen = 0; 9929 u32 prot = 0, flags = 0; 9930 unsigned int size; 9931 char tmp[16]; 9932 char *buf = NULL; 9933 char *name = NULL; 9934 9935 if (vma->vm_flags & VM_READ) 9936 prot |= PROT_READ; 9937 if (vma->vm_flags & VM_WRITE) 9938 prot |= PROT_WRITE; 9939 if (vma->vm_flags & VM_EXEC) 9940 prot |= PROT_EXEC; 9941 9942 if (vma->vm_flags & VM_MAYSHARE) 9943 flags = MAP_SHARED; 9944 else 9945 flags = MAP_PRIVATE; 9946 9947 if (vma->vm_flags & VM_LOCKED) 9948 flags |= MAP_LOCKED; 9949 if (vma_is_hugetlb(vma)) 9950 flags |= MAP_HUGETLB; 9951 9952 if (file) { 9953 const struct inode *inode; 9954 dev_t dev; 9955 9956 buf = kmalloc(PATH_MAX, GFP_KERNEL); 9957 if (!buf) { 9958 name = "//enomem"; 9959 goto cpy_name; 9960 } 9961 /* 9962 * d_path() works from the end of the rb backwards, so we 9963 * need to add enough zero bytes after the string to handle 9964 * the 64bit alignment we do later. 9965 */ 9966 name = d_path(file_user_path(file), buf, PATH_MAX - sizeof(u64)); 9967 if (IS_ERR(name)) { 9968 name = "//toolong"; 9969 goto cpy_name; 9970 } 9971 inode = file_user_inode(vma->vm_file); 9972 dev = inode->i_sb->s_dev; 9973 ino = inode->i_ino; 9974 gen = inode->i_generation; 9975 maj = MAJOR(dev); 9976 min = MINOR(dev); 9977 9978 goto got_name; 9979 } else { 9980 if (vma->vm_ops && vma->vm_ops->name) 9981 name = (char *) vma->vm_ops->name(vma); 9982 if (!name) 9983 name = (char *)arch_vma_name(vma); 9984 if (!name) { 9985 if (vma_is_initial_heap(vma)) 9986 name = "[heap]"; 9987 else if (vma_is_initial_stack(vma)) 9988 name = "[stack]"; 9989 else 9990 name = "//anon"; 9991 } 9992 } 9993 9994 cpy_name: 9995 strscpy(tmp, name); 9996 name = tmp; 9997 got_name: 9998 /* 9999 * Since our buffer works in 8 byte units we need to align our string 10000 * size to a multiple of 8. However, we must guarantee the tail end is 10001 * zero'd out to avoid leaking random bits to userspace. 10002 */ 10003 size = strlen(name)+1; 10004 while (!IS_ALIGNED(size, sizeof(u64))) 10005 name[size++] = '\0'; 10006 10007 mmap_event->file_name = name; 10008 mmap_event->file_size = size; 10009 mmap_event->maj = maj; 10010 mmap_event->min = min; 10011 mmap_event->ino = ino; 10012 mmap_event->ino_generation = gen; 10013 mmap_event->prot = prot; 10014 mmap_event->flags = flags; 10015 10016 if (atomic_read(&nr_build_id_events)) 10017 build_id_parse_nofault(vma, mmap_event->build_id, &mmap_event->build_id_size); 10018 10019 perf_iterate_sb(perf_event_mmap_output, 10020 mmap_event, 10021 NULL); 10022 10023 kfree(buf); 10024 } 10025 10026 /* 10027 * Check whether inode and address range match filter criteria. 10028 */ 10029 static bool perf_addr_filter_match(struct perf_addr_filter *filter, 10030 struct file *file, unsigned long offset, 10031 unsigned long size) 10032 { 10033 /* d_inode(NULL) won't be equal to any mapped user-space file */ 10034 if (!filter->path.dentry) 10035 return false; 10036 10037 if (d_inode(filter->path.dentry) != file_user_inode(file)) 10038 return false; 10039 10040 if (filter->offset > offset + size) 10041 return false; 10042 10043 if (filter->offset + filter->size < offset) 10044 return false; 10045 10046 return true; 10047 } 10048 10049 static bool perf_addr_filter_vma_adjust(struct perf_addr_filter *filter, 10050 struct vm_area_struct *vma, 10051 struct perf_addr_filter_range *fr) 10052 { 10053 unsigned long vma_size = vma->vm_end - vma->vm_start; 10054 unsigned long off = vma_start_pgoff(vma) << PAGE_SHIFT; 10055 struct file *file = vma->vm_file; 10056 10057 if (!perf_addr_filter_match(filter, file, off, vma_size)) 10058 return false; 10059 10060 if (filter->offset < off) { 10061 fr->start = vma->vm_start; 10062 fr->size = min(vma_size, filter->size - (off - filter->offset)); 10063 } else { 10064 fr->start = vma->vm_start + filter->offset - off; 10065 fr->size = min(vma->vm_end - fr->start, filter->size); 10066 } 10067 10068 return true; 10069 } 10070 10071 static void __perf_addr_filters_adjust(struct perf_event *event, void *data) 10072 { 10073 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 10074 struct vm_area_struct *vma = data; 10075 struct perf_addr_filter *filter; 10076 unsigned int restart = 0, count = 0; 10077 unsigned long flags; 10078 10079 if (!has_addr_filter(event)) 10080 return; 10081 10082 if (!vma->vm_file) 10083 return; 10084 10085 raw_spin_lock_irqsave(&ifh->lock, flags); 10086 list_for_each_entry(filter, &ifh->list, entry) { 10087 if (perf_addr_filter_vma_adjust(filter, vma, 10088 &event->addr_filter_ranges[count])) 10089 restart++; 10090 10091 count++; 10092 } 10093 10094 if (restart) 10095 event->addr_filters_gen++; 10096 raw_spin_unlock_irqrestore(&ifh->lock, flags); 10097 10098 if (restart) 10099 perf_event_stop(event, 1); 10100 } 10101 10102 /* 10103 * Adjust all task's events' filters to the new vma 10104 */ 10105 static void perf_addr_filters_adjust(struct vm_area_struct *vma) 10106 { 10107 struct perf_event_context *ctx; 10108 10109 /* 10110 * Data tracing isn't supported yet and as such there is no need 10111 * to keep track of anything that isn't related to executable code: 10112 */ 10113 if (!(vma->vm_flags & VM_EXEC)) 10114 return; 10115 10116 rcu_read_lock(); 10117 ctx = rcu_dereference(current->perf_event_ctxp); 10118 if (ctx) 10119 perf_iterate_ctx(ctx, __perf_addr_filters_adjust, vma, true); 10120 rcu_read_unlock(); 10121 } 10122 10123 void perf_event_mmap(struct vm_area_struct *vma) 10124 { 10125 struct perf_mmap_event mmap_event; 10126 10127 if (!atomic_read(&nr_mmap_events)) 10128 return; 10129 10130 mmap_event = (struct perf_mmap_event){ 10131 .vma = vma, 10132 /* .file_name */ 10133 /* .file_size */ 10134 .event_id = { 10135 /* .header */ 10136 /* .pid */ 10137 /* .tid */ 10138 .start = vma->vm_start, 10139 .len = vma->vm_end - vma->vm_start, 10140 .pgoff = (u64)vma_start_pgoff(vma) << PAGE_SHIFT, 10141 }, 10142 /* .maj (attr_mmap2 only) */ 10143 /* .min (attr_mmap2 only) */ 10144 /* .ino (attr_mmap2 only) */ 10145 /* .ino_generation (attr_mmap2 only) */ 10146 /* .prot (attr_mmap2 only) */ 10147 /* .flags (attr_mmap2 only) */ 10148 }; 10149 10150 perf_addr_filters_adjust(vma); 10151 perf_event_mmap_event(&mmap_event); 10152 } 10153 10154 void perf_event_aux_event(struct perf_event *event, unsigned long head, 10155 unsigned long size, u64 flags) 10156 { 10157 struct perf_output_handle handle; 10158 struct perf_sample_data sample; 10159 struct perf_aux_event { 10160 struct perf_event_header header; 10161 u64 offset; 10162 u64 size; 10163 u64 flags; 10164 } rec = { 10165 .offset = head, 10166 .size = size, 10167 .flags = flags, 10168 }; 10169 int ret; 10170 10171 perf_event_header__init(&rec.header, &sample, 10172 PERF_RECORD_AUX, /* misc= */ 0, sizeof(rec), 10173 event); 10174 ret = perf_output_begin(&handle, &sample, event, rec.header.size); 10175 10176 if (ret) 10177 return; 10178 10179 perf_output_put(&handle, rec); 10180 perf_event__output_id_sample(event, &handle, &sample); 10181 10182 perf_output_end(&handle); 10183 } 10184 10185 /* 10186 * Lost/dropped samples logging 10187 */ 10188 void perf_log_lost_samples(struct perf_event *event, u64 lost) 10189 { 10190 struct perf_output_handle handle; 10191 struct perf_sample_data sample; 10192 int ret; 10193 10194 struct { 10195 struct perf_event_header header; 10196 u64 lost; 10197 } lost_samples_event = { 10198 .lost = lost, 10199 }; 10200 10201 perf_event_header__init(&lost_samples_event.header, &sample, 10202 PERF_RECORD_LOST_SAMPLES, 10203 /* misc= */ 0, 10204 sizeof(lost_samples_event), 10205 event); 10206 10207 ret = perf_output_begin(&handle, &sample, event, 10208 lost_samples_event.header.size); 10209 if (ret) 10210 return; 10211 10212 perf_output_put(&handle, lost_samples_event); 10213 perf_event__output_id_sample(event, &handle, &sample); 10214 perf_output_end(&handle); 10215 } 10216 10217 /* 10218 * context_switch tracking 10219 */ 10220 10221 struct perf_switch_event { 10222 struct task_struct *task; 10223 struct task_struct *next_prev; 10224 10225 struct { 10226 struct perf_event_header header; 10227 u32 next_prev_pid; 10228 u32 next_prev_tid; 10229 } event_id; 10230 bool sched_in; 10231 bool preempt; 10232 }; 10233 10234 static int perf_event_switch_match(struct perf_event *event) 10235 { 10236 return event->attr.context_switch; 10237 } 10238 10239 static void perf_event_switch_output(struct perf_event *event, void *data) 10240 { 10241 struct perf_switch_event *se = data; 10242 struct perf_output_handle handle; 10243 struct perf_sample_data sample; 10244 u32 type; 10245 u16 misc; 10246 u16 size; 10247 int ret; 10248 10249 if (!perf_event_switch_match(event)) 10250 return; 10251 10252 /* Only CPU-wide events are allowed to see next/prev pid/tid */ 10253 if (event->ctx->task) { 10254 type = PERF_RECORD_SWITCH; 10255 size = sizeof(se->event_id.header); 10256 } else { 10257 type = PERF_RECORD_SWITCH_CPU_WIDE; 10258 size = sizeof(se->event_id); 10259 se->event_id.next_prev_pid = 10260 perf_event_pid(event, se->next_prev); 10261 se->event_id.next_prev_tid = 10262 perf_event_tid(event, se->next_prev); 10263 } 10264 misc = se->sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT; 10265 if (se->preempt) 10266 misc |= PERF_RECORD_MISC_SWITCH_OUT_PREEMPT; 10267 10268 perf_event_header__init(&se->event_id.header, &sample, 10269 type, misc, size, event); 10270 10271 ret = perf_output_begin(&handle, &sample, event, se->event_id.header.size); 10272 if (ret) 10273 return; 10274 10275 if (event->ctx->task) 10276 perf_output_put(&handle, se->event_id.header); 10277 else 10278 perf_output_put(&handle, se->event_id); 10279 10280 perf_event__output_id_sample(event, &handle, &sample); 10281 10282 perf_output_end(&handle); 10283 } 10284 10285 static void perf_event_switch(struct task_struct *task, 10286 struct task_struct *next_prev, bool sched_in) 10287 { 10288 struct perf_switch_event switch_event; 10289 10290 /* N.B. caller checks nr_switch_events != 0 */ 10291 10292 switch_event = (struct perf_switch_event){ 10293 .task = task, 10294 .next_prev = next_prev, 10295 /* .event_id */ 10296 .sched_in = sched_in, 10297 .preempt = !sched_in && task_is_runnable(task), 10298 }; 10299 10300 perf_iterate_sb(perf_event_switch_output, &switch_event, NULL); 10301 } 10302 10303 /* 10304 * IRQ throttle logging 10305 */ 10306 10307 static void perf_log_throttle(struct perf_event *event, int enable) 10308 { 10309 struct perf_output_handle handle; 10310 struct perf_sample_data sample; 10311 int ret; 10312 10313 struct { 10314 struct perf_event_header header; 10315 u64 time; 10316 u64 id; 10317 u64 stream_id; 10318 } throttle_event = { 10319 .time = perf_event_clock(event), 10320 .id = primary_event_id(event), 10321 .stream_id = event->id, 10322 }; 10323 10324 perf_event_header__init(&throttle_event.header, &sample, 10325 enable ? PERF_RECORD_UNTHROTTLE 10326 : PERF_RECORD_THROTTLE, 10327 /* misc= */ 0, 10328 sizeof(throttle_event), 10329 event); 10330 10331 ret = perf_output_begin(&handle, &sample, event, 10332 throttle_event.header.size); 10333 if (ret) 10334 return; 10335 10336 perf_output_put(&handle, throttle_event); 10337 perf_event__output_id_sample(event, &handle, &sample); 10338 perf_output_end(&handle); 10339 } 10340 10341 /* 10342 * ksymbol register/unregister tracking 10343 */ 10344 10345 struct perf_ksymbol_event { 10346 const char *name; 10347 int name_len; 10348 struct { 10349 struct perf_event_header header; 10350 u64 addr; 10351 u32 len; 10352 u16 ksym_type; 10353 u16 flags; 10354 } event_id; 10355 }; 10356 10357 static int perf_event_ksymbol_match(struct perf_event *event) 10358 { 10359 return event->attr.ksymbol; 10360 } 10361 10362 static void perf_event_ksymbol_output(struct perf_event *event, void *data) 10363 { 10364 struct perf_ksymbol_event *ksymbol_event = data; 10365 struct perf_output_handle handle; 10366 struct perf_sample_data sample; 10367 int ret; 10368 u16 size = sizeof(ksymbol_event->event_id) + ksymbol_event->name_len; 10369 10370 if (!perf_event_ksymbol_match(event)) 10371 return; 10372 10373 perf_event_header__init(&ksymbol_event->event_id.header, &sample, 10374 PERF_RECORD_KSYMBOL, /* misc= */ 0, size, 10375 event); 10376 ret = perf_output_begin(&handle, &sample, event, 10377 ksymbol_event->event_id.header.size); 10378 if (ret) 10379 return; 10380 10381 perf_output_put(&handle, ksymbol_event->event_id); 10382 __output_copy(&handle, ksymbol_event->name, ksymbol_event->name_len); 10383 perf_event__output_id_sample(event, &handle, &sample); 10384 10385 perf_output_end(&handle); 10386 } 10387 10388 void perf_event_ksymbol(u16 ksym_type, u64 addr, u32 len, bool unregister, 10389 const char *sym) 10390 { 10391 struct perf_ksymbol_event ksymbol_event; 10392 char name[KSYM_NAME_LEN]; 10393 u16 flags = 0; 10394 int name_len; 10395 10396 if (!atomic_read(&nr_ksymbol_events)) 10397 return; 10398 10399 if (ksym_type >= PERF_RECORD_KSYMBOL_TYPE_MAX || 10400 ksym_type == PERF_RECORD_KSYMBOL_TYPE_UNKNOWN) 10401 goto err; 10402 10403 strscpy(name, sym); 10404 name_len = strlen(name) + 1; 10405 while (!IS_ALIGNED(name_len, sizeof(u64))) 10406 name[name_len++] = '\0'; 10407 BUILD_BUG_ON(KSYM_NAME_LEN % sizeof(u64)); 10408 10409 if (unregister) 10410 flags |= PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER; 10411 10412 ksymbol_event = (struct perf_ksymbol_event){ 10413 .name = name, 10414 .name_len = name_len, 10415 .event_id = { 10416 .addr = addr, 10417 .len = len, 10418 .ksym_type = ksym_type, 10419 .flags = flags, 10420 }, 10421 }; 10422 10423 perf_iterate_sb(perf_event_ksymbol_output, &ksymbol_event, NULL); 10424 return; 10425 err: 10426 WARN_ONCE(1, "%s: Invalid KSYMBOL type 0x%x\n", __func__, ksym_type); 10427 } 10428 10429 /* 10430 * bpf program load/unload tracking 10431 */ 10432 10433 struct perf_bpf_event { 10434 struct bpf_prog *prog; 10435 struct { 10436 struct perf_event_header header; 10437 u16 type; 10438 u16 flags; 10439 u32 id; 10440 u8 tag[BPF_TAG_SIZE]; 10441 } event_id; 10442 }; 10443 10444 static int perf_event_bpf_match(struct perf_event *event) 10445 { 10446 return event->attr.bpf_event; 10447 } 10448 10449 static void perf_event_bpf_output(struct perf_event *event, void *data) 10450 { 10451 struct perf_bpf_event *bpf_event = data; 10452 struct perf_output_handle handle; 10453 struct perf_sample_data sample; 10454 int ret; 10455 10456 if (!perf_event_bpf_match(event)) 10457 return; 10458 10459 perf_event_header__init(&bpf_event->event_id.header, &sample, 10460 PERF_RECORD_BPF_EVENT, 10461 /* misc= */ 0, 10462 sizeof(bpf_event->event_id), 10463 event); 10464 ret = perf_output_begin(&handle, &sample, event, 10465 bpf_event->event_id.header.size); 10466 if (ret) 10467 return; 10468 10469 perf_output_put(&handle, bpf_event->event_id); 10470 perf_event__output_id_sample(event, &handle, &sample); 10471 10472 perf_output_end(&handle); 10473 } 10474 10475 static void perf_event_bpf_emit_ksymbols(struct bpf_prog *prog, 10476 enum perf_bpf_event_type type) 10477 { 10478 bool unregister = type == PERF_BPF_EVENT_PROG_UNLOAD; 10479 int i; 10480 10481 perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_BPF, 10482 (u64)(unsigned long)prog->bpf_func, 10483 prog->jited_len, unregister, 10484 prog->aux->ksym.name); 10485 10486 for (i = 1; i < prog->aux->func_cnt; i++) { 10487 struct bpf_prog *subprog = prog->aux->func[i]; 10488 10489 perf_event_ksymbol( 10490 PERF_RECORD_KSYMBOL_TYPE_BPF, 10491 (u64)(unsigned long)subprog->bpf_func, 10492 subprog->jited_len, unregister, 10493 subprog->aux->ksym.name); 10494 } 10495 } 10496 10497 void perf_event_bpf_event(struct bpf_prog *prog, 10498 enum perf_bpf_event_type type, 10499 u16 flags) 10500 { 10501 struct perf_bpf_event bpf_event; 10502 10503 switch (type) { 10504 case PERF_BPF_EVENT_PROG_LOAD: 10505 case PERF_BPF_EVENT_PROG_UNLOAD: 10506 if (atomic_read(&nr_ksymbol_events)) 10507 perf_event_bpf_emit_ksymbols(prog, type); 10508 break; 10509 default: 10510 return; 10511 } 10512 10513 if (!atomic_read(&nr_bpf_events)) 10514 return; 10515 10516 bpf_event = (struct perf_bpf_event){ 10517 .prog = prog, 10518 .event_id = { 10519 .type = type, 10520 .flags = flags, 10521 .id = prog->aux->id, 10522 }, 10523 }; 10524 10525 BUILD_BUG_ON(BPF_TAG_SIZE % sizeof(u64)); 10526 10527 memcpy(bpf_event.event_id.tag, prog->tag, BPF_TAG_SIZE); 10528 perf_iterate_sb(perf_event_bpf_output, &bpf_event, NULL); 10529 } 10530 10531 struct perf_callchain_deferred_event { 10532 struct unwind_stacktrace *trace; 10533 struct { 10534 struct perf_event_header header; 10535 u64 cookie; 10536 } event; 10537 }; 10538 10539 static void perf_callchain_deferred_output(struct perf_event *event, void *data) 10540 { 10541 struct perf_callchain_deferred_event *deferred_event = data; 10542 struct perf_output_handle handle; 10543 struct perf_sample_data sample; 10544 u64 nr = deferred_event->trace->nr; 10545 u16 size, id_size, max_payload; 10546 int ret; 10547 10548 if (!event->attr.defer_output) 10549 return; 10550 10551 id_size = event->attr.sample_id_all ? event->id_header_size : 0; 10552 max_payload = round_down(U16_MAX - sizeof(deferred_event->event) - 10553 sizeof(u64) - id_size, sizeof(u64)); 10554 nr = min_t(u64, nr, max_payload / sizeof(u64)); 10555 size = sizeof(deferred_event->event) + sizeof(u64) + nr * sizeof(u64); 10556 10557 /* XXX do we really need sample_id_all for this ??? */ 10558 perf_event_header__init(&deferred_event->event.header, &sample, 10559 PERF_RECORD_CALLCHAIN_DEFERRED, 10560 PERF_RECORD_MISC_USER, 10561 size, 10562 event); 10563 10564 ret = perf_output_begin(&handle, &sample, event, 10565 deferred_event->event.header.size); 10566 if (ret) 10567 return; 10568 10569 perf_output_put(&handle, deferred_event->event); 10570 perf_output_put(&handle, nr); 10571 for (u64 i = 0; i < nr; i++) { 10572 u64 entry = deferred_event->trace->entries[i]; 10573 10574 perf_output_put(&handle, entry); 10575 } 10576 perf_event__output_id_sample(event, &handle, &sample); 10577 10578 perf_output_end(&handle); 10579 } 10580 10581 static void perf_unwind_deferred_callback(struct unwind_work *work, 10582 struct unwind_stacktrace *trace, u64 cookie) 10583 { 10584 struct perf_callchain_deferred_event deferred_event = { 10585 .trace = trace, 10586 .event = { 10587 .cookie = cookie, 10588 }, 10589 }; 10590 10591 perf_iterate_sb(perf_callchain_deferred_output, &deferred_event, NULL); 10592 } 10593 10594 struct perf_text_poke_event { 10595 const void *old_bytes; 10596 const void *new_bytes; 10597 u16 tot; 10598 u16 pad; 10599 u16 old_len; 10600 u16 new_len; 10601 10602 struct { 10603 struct perf_event_header header; 10604 10605 u64 addr; 10606 } event_id; 10607 }; 10608 10609 static int perf_event_text_poke_match(struct perf_event *event) 10610 { 10611 return event->attr.text_poke; 10612 } 10613 10614 static void perf_event_text_poke_output(struct perf_event *event, void *data) 10615 { 10616 struct perf_text_poke_event *text_poke_event = data; 10617 struct perf_output_handle handle; 10618 struct perf_sample_data sample; 10619 u16 size = sizeof(text_poke_event->event_id) + text_poke_event->tot + text_poke_event->pad; 10620 u64 padding = 0; 10621 int ret; 10622 10623 if (!perf_event_text_poke_match(event)) 10624 return; 10625 10626 perf_event_header__init(&text_poke_event->event_id.header, &sample, 10627 PERF_RECORD_TEXT_POKE, 10628 PERF_RECORD_MISC_KERNEL, 10629 size, 10630 event); 10631 10632 ret = perf_output_begin(&handle, &sample, event, 10633 text_poke_event->event_id.header.size); 10634 if (ret) 10635 return; 10636 10637 perf_output_put(&handle, text_poke_event->event_id); 10638 perf_output_put(&handle, text_poke_event->old_len); 10639 perf_output_put(&handle, text_poke_event->new_len); 10640 10641 __output_copy(&handle, text_poke_event->old_bytes, text_poke_event->old_len); 10642 __output_copy(&handle, text_poke_event->new_bytes, text_poke_event->new_len); 10643 10644 if (text_poke_event->pad) 10645 __output_copy(&handle, &padding, text_poke_event->pad); 10646 10647 perf_event__output_id_sample(event, &handle, &sample); 10648 10649 perf_output_end(&handle); 10650 } 10651 10652 void perf_event_text_poke(const void *addr, const void *old_bytes, 10653 size_t old_len, const void *new_bytes, size_t new_len) 10654 { 10655 struct perf_text_poke_event text_poke_event; 10656 size_t tot, pad; 10657 10658 if (!atomic_read(&nr_text_poke_events)) 10659 return; 10660 10661 tot = sizeof(text_poke_event.old_len) + old_len; 10662 tot += sizeof(text_poke_event.new_len) + new_len; 10663 pad = ALIGN(tot, sizeof(u64)) - tot; 10664 10665 text_poke_event = (struct perf_text_poke_event){ 10666 .old_bytes = old_bytes, 10667 .new_bytes = new_bytes, 10668 .tot = tot, 10669 .pad = pad, 10670 .old_len = old_len, 10671 .new_len = new_len, 10672 .event_id = { 10673 .addr = (unsigned long)addr, 10674 }, 10675 }; 10676 10677 perf_iterate_sb(perf_event_text_poke_output, &text_poke_event, NULL); 10678 } 10679 10680 void perf_event_itrace_started(struct perf_event *event) 10681 { 10682 WRITE_ONCE(event->attach_state, event->attach_state | PERF_ATTACH_ITRACE); 10683 } 10684 EXPORT_SYMBOL_GPL(perf_event_itrace_started); 10685 10686 static void perf_log_itrace_start(struct perf_event *event) 10687 { 10688 struct perf_output_handle handle; 10689 struct perf_sample_data sample; 10690 struct perf_aux_event { 10691 struct perf_event_header header; 10692 u32 pid; 10693 u32 tid; 10694 } rec; 10695 int ret; 10696 10697 if (event->parent) 10698 event = event->parent; 10699 10700 if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) || 10701 event->attach_state & PERF_ATTACH_ITRACE) 10702 return; 10703 10704 rec.pid = perf_event_pid(event, current); 10705 rec.tid = perf_event_tid(event, current); 10706 10707 perf_event_header__init(&rec.header, &sample, 10708 PERF_RECORD_ITRACE_START, /* misc= */ 0, sizeof(rec), 10709 event); 10710 ret = perf_output_begin(&handle, &sample, event, rec.header.size); 10711 10712 if (ret) 10713 return; 10714 10715 perf_output_put(&handle, rec); 10716 perf_event__output_id_sample(event, &handle, &sample); 10717 10718 perf_output_end(&handle); 10719 } 10720 10721 void perf_report_aux_output_id(struct perf_event *event, u64 hw_id) 10722 { 10723 struct perf_output_handle handle; 10724 struct perf_sample_data sample; 10725 struct perf_aux_event { 10726 struct perf_event_header header; 10727 u64 hw_id; 10728 } rec; 10729 int ret; 10730 10731 if (event->parent) 10732 event = event->parent; 10733 10734 rec.hw_id = hw_id; 10735 perf_event_header__init(&rec.header, &sample, 10736 PERF_RECORD_AUX_OUTPUT_HW_ID, /* misc= */ 0, 10737 sizeof(rec), event); 10738 ret = perf_output_begin(&handle, &sample, event, rec.header.size); 10739 10740 if (ret) 10741 return; 10742 10743 perf_output_put(&handle, rec); 10744 perf_event__output_id_sample(event, &handle, &sample); 10745 10746 perf_output_end(&handle); 10747 } 10748 EXPORT_SYMBOL_GPL(perf_report_aux_output_id); 10749 10750 static int 10751 __perf_event_account_interrupt(struct perf_event *event, int throttle) 10752 { 10753 struct hw_perf_event *hwc = &event->hw; 10754 int ret = 0; 10755 u64 seq; 10756 10757 seq = __this_cpu_read(perf_throttled_seq); 10758 if (seq != hwc->interrupts_seq) { 10759 hwc->interrupts_seq = seq; 10760 hwc->interrupts = 1; 10761 } else { 10762 hwc->interrupts++; 10763 } 10764 10765 if (unlikely(throttle && hwc->interrupts >= max_samples_per_tick)) { 10766 __this_cpu_inc(perf_throttled_count); 10767 tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); 10768 perf_event_throttle_group(event); 10769 ret = 1; 10770 } 10771 10772 if (event->attr.freq) { 10773 u64 now = perf_clock(); 10774 s64 delta = now - hwc->freq_time_stamp; 10775 10776 hwc->freq_time_stamp = now; 10777 10778 if (delta > 0 && delta < 2*TICK_NSEC) 10779 perf_adjust_period(event, delta, hwc->last_period, true); 10780 } 10781 10782 return ret; 10783 } 10784 10785 int perf_event_account_interrupt(struct perf_event *event) 10786 { 10787 return __perf_event_account_interrupt(event, 1); 10788 } 10789 10790 static inline bool sample_is_allowed(struct perf_event *event, struct pt_regs *regs) 10791 { 10792 /* 10793 * Due to interrupt latency (AKA "skid"), we may enter the 10794 * kernel before taking an overflow, even if the PMU is only 10795 * counting user events. 10796 */ 10797 if (event->attr.exclude_kernel && !user_mode(regs)) 10798 return false; 10799 10800 return true; 10801 } 10802 10803 #ifdef CONFIG_BPF_SYSCALL 10804 static int bpf_overflow_handler(struct perf_event *event, 10805 struct perf_sample_data *data, 10806 struct pt_regs *regs) 10807 { 10808 struct bpf_perf_event_data_kern ctx = { 10809 .data = data, 10810 .event = event, 10811 }; 10812 struct bpf_prog *prog; 10813 int ret = 0; 10814 10815 ctx.regs = perf_arch_bpf_user_pt_regs(regs); 10816 if (unlikely(__this_cpu_inc_return(bpf_prog_active) != 1)) 10817 goto out; 10818 rcu_read_lock(); 10819 prog = READ_ONCE(event->prog); 10820 if (prog) { 10821 perf_prepare_sample(data, event, regs); 10822 ret = bpf_prog_run(prog, &ctx); 10823 } 10824 rcu_read_unlock(); 10825 out: 10826 __this_cpu_dec(bpf_prog_active); 10827 10828 return ret; 10829 } 10830 10831 static inline int perf_event_set_bpf_handler(struct perf_event *event, 10832 struct bpf_prog *prog, 10833 u64 bpf_cookie) 10834 { 10835 if (event->overflow_handler_context) 10836 /* hw breakpoint or kernel counter */ 10837 return -EINVAL; 10838 10839 if (event->prog) 10840 return -EEXIST; 10841 10842 if (prog->type != BPF_PROG_TYPE_PERF_EVENT) 10843 return -EINVAL; 10844 10845 if (event->attr.precise_ip && 10846 prog->call_get_stack && 10847 (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) || 10848 event->attr.exclude_callchain_kernel || 10849 event->attr.exclude_callchain_user)) { 10850 /* 10851 * On perf_event with precise_ip, calling bpf_get_stack() 10852 * may trigger unwinder warnings and occasional crashes. 10853 * bpf_get_[stack|stackid] works around this issue by using 10854 * callchain attached to perf_sample_data. If the 10855 * perf_event does not full (kernel and user) callchain 10856 * attached to perf_sample_data, do not allow attaching BPF 10857 * program that calls bpf_get_[stack|stackid]. 10858 */ 10859 return -EPROTO; 10860 } 10861 10862 event->prog = prog; 10863 event->bpf_cookie = bpf_cookie; 10864 return 0; 10865 } 10866 10867 static inline void perf_event_free_bpf_handler(struct perf_event *event) 10868 { 10869 struct bpf_prog *prog = event->prog; 10870 10871 if (!prog) 10872 return; 10873 10874 event->prog = NULL; 10875 bpf_prog_put(prog); 10876 } 10877 #else 10878 static inline int bpf_overflow_handler(struct perf_event *event, 10879 struct perf_sample_data *data, 10880 struct pt_regs *regs) 10881 { 10882 return 1; 10883 } 10884 10885 static inline int perf_event_set_bpf_handler(struct perf_event *event, 10886 struct bpf_prog *prog, 10887 u64 bpf_cookie) 10888 { 10889 return -EOPNOTSUPP; 10890 } 10891 10892 static inline void perf_event_free_bpf_handler(struct perf_event *event) 10893 { 10894 } 10895 #endif 10896 10897 /* 10898 * Generic event overflow handling, sampling. 10899 */ 10900 10901 static int __perf_event_overflow(struct perf_event *event, 10902 int throttle, struct perf_sample_data *data, 10903 struct pt_regs *regs) 10904 { 10905 int events = atomic_read(&event->event_limit); 10906 int ret = 0; 10907 10908 /* 10909 * Non-sampling counters might still use the PMI to fold short 10910 * hardware counters, ignore those. 10911 */ 10912 if (unlikely(!is_sampling_event(event))) 10913 return 0; 10914 10915 ret = __perf_event_account_interrupt(event, throttle); 10916 10917 if (event->attr.aux_pause) 10918 perf_event_aux_pause(event->aux_event, true); 10919 10920 if (event->prog && event->prog->type == BPF_PROG_TYPE_PERF_EVENT && 10921 !bpf_overflow_handler(event, data, regs)) 10922 goto out; 10923 10924 /* 10925 * XXX event_limit might not quite work as expected on inherited 10926 * events 10927 */ 10928 10929 event->pending_kill = POLL_IN; 10930 if (events && atomic_dec_and_test(&event->event_limit)) { 10931 ret = 1; 10932 event->pending_kill = POLL_HUP; 10933 perf_event_disable_inatomic(event); 10934 event->pmu->stop(event, 0); 10935 } 10936 10937 if (event->attr.sigtrap) { 10938 /* 10939 * The desired behaviour of sigtrap vs invalid samples is a bit 10940 * tricky; on the one hand, one should not loose the SIGTRAP if 10941 * it is the first event, on the other hand, we should also not 10942 * trigger the WARN or override the data address. 10943 */ 10944 bool valid_sample = sample_is_allowed(event, regs); 10945 unsigned int pending_id = 1; 10946 enum task_work_notify_mode notify_mode; 10947 10948 if (regs) 10949 pending_id = hash32_ptr((void *)instruction_pointer(regs)) ?: 1; 10950 10951 notify_mode = in_nmi() ? TWA_NMI_CURRENT : TWA_RESUME; 10952 10953 if (!event->pending_work && 10954 !task_work_add(current, &event->pending_task, notify_mode)) { 10955 event->pending_work = pending_id; 10956 local_inc(&event->ctx->nr_no_switch_fast); 10957 WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount)); 10958 10959 event->pending_addr = 0; 10960 if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR)) 10961 event->pending_addr = data->addr; 10962 10963 } else if (event->attr.exclude_kernel && valid_sample) { 10964 /* 10965 * Should not be able to return to user space without 10966 * consuming pending_work; with exceptions: 10967 * 10968 * 1. Where !exclude_kernel, events can overflow again 10969 * in the kernel without returning to user space. 10970 * 10971 * 2. Events that can overflow again before the IRQ- 10972 * work without user space progress (e.g. hrtimer). 10973 * To approximate progress (with false negatives), 10974 * check 32-bit hash of the current IP. 10975 */ 10976 WARN_ON_ONCE(event->pending_work != pending_id); 10977 } 10978 } 10979 10980 READ_ONCE(event->overflow_handler)(event, data, regs); 10981 10982 if (*perf_event_fasync(event) && event->pending_kill) { 10983 event->pending_wakeup = 1; 10984 irq_work_queue(&event->pending_irq); 10985 } 10986 out: 10987 if (event->attr.aux_resume) 10988 perf_event_aux_pause(event->aux_event, false); 10989 10990 return ret; 10991 } 10992 10993 int perf_event_overflow(struct perf_event *event, 10994 struct perf_sample_data *data, 10995 struct pt_regs *regs) 10996 { 10997 /* 10998 * Entry point from hardware PMI, interrupts should be disabled here. 10999 * This serializes us against perf_event_remove_from_context() in 11000 * things like perf_event_release_kernel(). 11001 */ 11002 lockdep_assert_irqs_disabled(); 11003 11004 return __perf_event_overflow(event, 1, data, regs); 11005 } 11006 11007 /* 11008 * Generic software event infrastructure 11009 */ 11010 11011 struct swevent_htable { 11012 struct swevent_hlist *swevent_hlist; 11013 struct mutex hlist_mutex; 11014 int hlist_refcount; 11015 }; 11016 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable); 11017 11018 /* 11019 * We directly increment event->count and keep a second value in 11020 * event->hw.period_left to count intervals. This period event 11021 * is kept in the range [-sample_period, 0] so that we can use the 11022 * sign as trigger. 11023 */ 11024 11025 u64 perf_swevent_set_period(struct perf_event *event) 11026 { 11027 struct hw_perf_event *hwc = &event->hw; 11028 u64 period = hwc->last_period; 11029 u64 nr, offset; 11030 s64 old, val; 11031 11032 hwc->last_period = hwc->sample_period; 11033 11034 old = local64_read(&hwc->period_left); 11035 do { 11036 val = old; 11037 if (val < 0) 11038 return 0; 11039 11040 nr = div64_u64(period + val, period); 11041 offset = nr * period; 11042 val -= offset; 11043 } while (!local64_try_cmpxchg(&hwc->period_left, &old, val)); 11044 11045 return nr; 11046 } 11047 11048 static void perf_swevent_overflow(struct perf_event *event, u64 overflow, 11049 struct perf_sample_data *data, 11050 struct pt_regs *regs) 11051 { 11052 struct hw_perf_event *hwc = &event->hw; 11053 int throttle = 0; 11054 11055 if (!overflow) 11056 overflow = perf_swevent_set_period(event); 11057 11058 if (hwc->interrupts == MAX_INTERRUPTS) 11059 return; 11060 11061 for (; overflow; overflow--) { 11062 if (__perf_event_overflow(event, throttle, 11063 data, regs)) { 11064 /* 11065 * We inhibit the overflow from happening when 11066 * hwc->interrupts == MAX_INTERRUPTS. 11067 */ 11068 break; 11069 } 11070 throttle = 1; 11071 } 11072 } 11073 11074 static void perf_swevent_event(struct perf_event *event, u64 nr, 11075 struct perf_sample_data *data, 11076 struct pt_regs *regs) 11077 { 11078 struct hw_perf_event *hwc = &event->hw; 11079 11080 /* 11081 * This is: 11082 * - software preempt 11083 * - tracepoint preempt 11084 * - tp_target_task irq (ctx->lock) 11085 * - uprobes preempt/irq 11086 * - kprobes preempt/irq 11087 * - hw_breakpoint irq 11088 * 11089 * Any of these are sufficient to hold off RCU and thus ensure @event 11090 * exists. 11091 */ 11092 lockdep_assert_preemption_disabled(); 11093 local64_add(nr, &event->count); 11094 11095 if (!regs) 11096 return; 11097 11098 if (!is_sampling_event(event)) 11099 return; 11100 11101 /* 11102 * Serialize against event_function_call() IPIs like normal overflow 11103 * event handling. Specifically, must not allow 11104 * perf_event_release_kernel() -> perf_remove_from_context() to make 11105 * progress and 'release' the event from under us. 11106 */ 11107 guard(irqsave)(); 11108 if (event->state != PERF_EVENT_STATE_ACTIVE) 11109 return; 11110 11111 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { 11112 data->period = nr; 11113 return perf_swevent_overflow(event, 1, data, regs); 11114 } else 11115 data->period = event->hw.last_period; 11116 11117 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) 11118 return perf_swevent_overflow(event, 1, data, regs); 11119 11120 if (local64_add_negative(nr, &hwc->period_left)) 11121 return; 11122 11123 perf_swevent_overflow(event, 0, data, regs); 11124 } 11125 11126 int perf_exclude_event(struct perf_event *event, struct pt_regs *regs) 11127 { 11128 if (event->hw.state & PERF_HES_STOPPED) 11129 return 1; 11130 11131 if (regs) { 11132 if (event->attr.exclude_user && user_mode(regs)) 11133 return 1; 11134 11135 if (event->attr.exclude_kernel && !user_mode(regs)) 11136 return 1; 11137 } 11138 11139 return 0; 11140 } 11141 11142 static int perf_swevent_match(struct perf_event *event, 11143 enum perf_type_id type, 11144 u32 event_id, 11145 struct perf_sample_data *data, 11146 struct pt_regs *regs) 11147 { 11148 if (event->attr.type != type) 11149 return 0; 11150 11151 if (event->attr.config != event_id) 11152 return 0; 11153 11154 if (perf_exclude_event(event, regs)) 11155 return 0; 11156 11157 return 1; 11158 } 11159 11160 static inline u64 swevent_hash(u64 type, u32 event_id) 11161 { 11162 u64 val = event_id | (type << 32); 11163 11164 return hash_64(val, SWEVENT_HLIST_BITS); 11165 } 11166 11167 static inline struct hlist_head * 11168 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id) 11169 { 11170 u64 hash = swevent_hash(type, event_id); 11171 11172 return &hlist->heads[hash]; 11173 } 11174 11175 /* For the read side: events when they trigger */ 11176 static inline struct hlist_head * 11177 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id) 11178 { 11179 struct swevent_hlist *hlist; 11180 11181 hlist = rcu_dereference(swhash->swevent_hlist); 11182 if (!hlist) 11183 return NULL; 11184 11185 return __find_swevent_head(hlist, type, event_id); 11186 } 11187 11188 /* For the event head insertion and removal in the hlist */ 11189 static inline struct hlist_head * 11190 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) 11191 { 11192 struct swevent_hlist *hlist; 11193 u32 event_id = event->attr.config; 11194 u64 type = event->attr.type; 11195 11196 /* 11197 * Event scheduling is always serialized against hlist allocation 11198 * and release. Which makes the protected version suitable here. 11199 * The context lock guarantees that. 11200 */ 11201 hlist = rcu_dereference_protected(swhash->swevent_hlist, 11202 lockdep_is_held(&event->ctx->lock)); 11203 if (!hlist) 11204 return NULL; 11205 11206 return __find_swevent_head(hlist, type, event_id); 11207 } 11208 11209 static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 11210 u64 nr, 11211 struct perf_sample_data *data, 11212 struct pt_regs *regs) 11213 { 11214 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 11215 struct perf_event *event; 11216 struct hlist_head *head; 11217 11218 rcu_read_lock(); 11219 head = find_swevent_head_rcu(swhash, type, event_id); 11220 if (!head) 11221 goto end; 11222 11223 hlist_for_each_entry_rcu(event, head, hlist_entry) { 11224 if (perf_swevent_match(event, type, event_id, data, regs)) 11225 perf_swevent_event(event, nr, data, regs); 11226 } 11227 end: 11228 rcu_read_unlock(); 11229 } 11230 11231 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]); 11232 11233 int perf_swevent_get_recursion_context(void) 11234 { 11235 return get_recursion_context(current->perf_recursion); 11236 } 11237 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); 11238 11239 void perf_swevent_put_recursion_context(int rctx) 11240 { 11241 put_recursion_context(current->perf_recursion, rctx); 11242 } 11243 11244 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 11245 { 11246 struct perf_sample_data data; 11247 11248 if (WARN_ON_ONCE(!regs)) 11249 return; 11250 11251 perf_sample_data_init(&data, addr, 0); 11252 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); 11253 } 11254 11255 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 11256 { 11257 int rctx; 11258 11259 preempt_disable_notrace(); 11260 rctx = perf_swevent_get_recursion_context(); 11261 if (unlikely(rctx < 0)) 11262 goto fail; 11263 11264 ___perf_sw_event(event_id, nr, regs, addr); 11265 11266 perf_swevent_put_recursion_context(rctx); 11267 fail: 11268 preempt_enable_notrace(); 11269 } 11270 11271 static void perf_swevent_read(struct perf_event *event) 11272 { 11273 } 11274 11275 static int perf_swevent_add(struct perf_event *event, int flags) 11276 { 11277 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 11278 struct hw_perf_event *hwc = &event->hw; 11279 struct hlist_head *head; 11280 11281 if (is_sampling_event(event)) { 11282 hwc->last_period = hwc->sample_period; 11283 perf_swevent_set_period(event); 11284 } 11285 11286 hwc->state = !(flags & PERF_EF_START); 11287 11288 head = find_swevent_head(swhash, event); 11289 if (WARN_ON_ONCE(!head)) 11290 return -EINVAL; 11291 11292 hlist_add_head_rcu(&event->hlist_entry, head); 11293 perf_event_update_userpage(event); 11294 11295 return 0; 11296 } 11297 11298 static void perf_swevent_del(struct perf_event *event, int flags) 11299 { 11300 hlist_del_rcu(&event->hlist_entry); 11301 } 11302 11303 static void perf_swevent_start(struct perf_event *event, int flags) 11304 { 11305 event->hw.state = 0; 11306 } 11307 11308 static void perf_swevent_stop(struct perf_event *event, int flags) 11309 { 11310 event->hw.state = PERF_HES_STOPPED; 11311 } 11312 11313 /* Deref the hlist from the update side */ 11314 static inline struct swevent_hlist * 11315 swevent_hlist_deref(struct swevent_htable *swhash) 11316 { 11317 return rcu_dereference_protected(swhash->swevent_hlist, 11318 lockdep_is_held(&swhash->hlist_mutex)); 11319 } 11320 11321 static void swevent_hlist_release(struct swevent_htable *swhash) 11322 { 11323 struct swevent_hlist *hlist = swevent_hlist_deref(swhash); 11324 11325 if (!hlist) 11326 return; 11327 11328 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); 11329 kfree_rcu(hlist, rcu_head); 11330 } 11331 11332 static void swevent_hlist_put_cpu(int cpu) 11333 { 11334 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 11335 11336 mutex_lock(&swhash->hlist_mutex); 11337 11338 if (!--swhash->hlist_refcount) 11339 swevent_hlist_release(swhash); 11340 11341 mutex_unlock(&swhash->hlist_mutex); 11342 } 11343 11344 static void swevent_hlist_put(void) 11345 { 11346 int cpu; 11347 11348 for_each_possible_cpu(cpu) 11349 swevent_hlist_put_cpu(cpu); 11350 } 11351 11352 static int swevent_hlist_get_cpu(int cpu) 11353 { 11354 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 11355 int err = 0; 11356 11357 mutex_lock(&swhash->hlist_mutex); 11358 if (!swevent_hlist_deref(swhash) && 11359 cpumask_test_cpu(cpu, perf_online_mask)) { 11360 struct swevent_hlist *hlist; 11361 11362 hlist = kzalloc_obj(*hlist); 11363 if (!hlist) { 11364 err = -ENOMEM; 11365 goto exit; 11366 } 11367 rcu_assign_pointer(swhash->swevent_hlist, hlist); 11368 } 11369 swhash->hlist_refcount++; 11370 exit: 11371 mutex_unlock(&swhash->hlist_mutex); 11372 11373 return err; 11374 } 11375 11376 static int swevent_hlist_get(void) 11377 { 11378 int err, cpu, failed_cpu; 11379 11380 mutex_lock(&pmus_lock); 11381 for_each_possible_cpu(cpu) { 11382 err = swevent_hlist_get_cpu(cpu); 11383 if (err) { 11384 failed_cpu = cpu; 11385 goto fail; 11386 } 11387 } 11388 mutex_unlock(&pmus_lock); 11389 return 0; 11390 fail: 11391 for_each_possible_cpu(cpu) { 11392 if (cpu == failed_cpu) 11393 break; 11394 swevent_hlist_put_cpu(cpu); 11395 } 11396 mutex_unlock(&pmus_lock); 11397 return err; 11398 } 11399 11400 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 11401 11402 static void sw_perf_event_destroy(struct perf_event *event) 11403 { 11404 u64 event_id = event->attr.config; 11405 11406 WARN_ON(event->parent); 11407 11408 static_key_slow_dec(&perf_swevent_enabled[event_id]); 11409 swevent_hlist_put(); 11410 } 11411 11412 static struct pmu perf_cpu_clock; /* fwd declaration */ 11413 static struct pmu perf_task_clock; 11414 11415 static int perf_swevent_init(struct perf_event *event) 11416 { 11417 u64 event_id = event->attr.config; 11418 11419 if (event->attr.type != PERF_TYPE_SOFTWARE) 11420 return -ENOENT; 11421 11422 /* 11423 * no branch sampling for software events 11424 */ 11425 if (has_branch_stack(event)) 11426 return -EOPNOTSUPP; 11427 11428 switch (event_id) { 11429 case PERF_COUNT_SW_CPU_CLOCK: 11430 event->attr.type = perf_cpu_clock.type; 11431 return -ENOENT; 11432 case PERF_COUNT_SW_TASK_CLOCK: 11433 event->attr.type = perf_task_clock.type; 11434 return -ENOENT; 11435 11436 default: 11437 break; 11438 } 11439 11440 if (event_id >= PERF_COUNT_SW_MAX) 11441 return -ENOENT; 11442 11443 if (!event->parent) { 11444 int err; 11445 11446 err = swevent_hlist_get(); 11447 if (err) 11448 return err; 11449 11450 static_key_slow_inc(&perf_swevent_enabled[event_id]); 11451 event->destroy = sw_perf_event_destroy; 11452 } 11453 11454 return 0; 11455 } 11456 11457 static struct pmu perf_swevent = { 11458 .task_ctx_nr = perf_sw_context, 11459 11460 .capabilities = PERF_PMU_CAP_NO_NMI, 11461 11462 .event_init = perf_swevent_init, 11463 .add = perf_swevent_add, 11464 .del = perf_swevent_del, 11465 .start = perf_swevent_start, 11466 .stop = perf_swevent_stop, 11467 .read = perf_swevent_read, 11468 }; 11469 11470 #ifdef CONFIG_EVENT_TRACING 11471 11472 static void tp_perf_event_destroy(struct perf_event *event) 11473 { 11474 perf_trace_destroy(event); 11475 } 11476 11477 static int perf_tp_event_init(struct perf_event *event) 11478 { 11479 int err; 11480 11481 if (event->attr.type != PERF_TYPE_TRACEPOINT) 11482 return -ENOENT; 11483 11484 /* 11485 * no branch sampling for tracepoint events 11486 */ 11487 if (has_branch_stack(event)) 11488 return -EOPNOTSUPP; 11489 11490 err = perf_trace_init(event); 11491 if (err) 11492 return err; 11493 11494 event->destroy = tp_perf_event_destroy; 11495 11496 return 0; 11497 } 11498 11499 static struct pmu perf_tracepoint = { 11500 .task_ctx_nr = perf_sw_context, 11501 11502 .event_init = perf_tp_event_init, 11503 .add = perf_trace_add, 11504 .del = perf_trace_del, 11505 .start = perf_swevent_start, 11506 .stop = perf_swevent_stop, 11507 .read = perf_swevent_read, 11508 }; 11509 11510 static int perf_tp_filter_match(struct perf_event *event, 11511 struct perf_raw_record *raw) 11512 { 11513 void *record = raw->frag.data; 11514 11515 /* only top level events have filters set */ 11516 if (event->parent) 11517 event = event->parent; 11518 11519 if (likely(!event->filter) || filter_match_preds(event->filter, record)) 11520 return 1; 11521 return 0; 11522 } 11523 11524 static int perf_tp_event_match(struct perf_event *event, 11525 struct perf_raw_record *raw, 11526 struct pt_regs *regs) 11527 { 11528 if (event->hw.state & PERF_HES_STOPPED) 11529 return 0; 11530 /* 11531 * If exclude_kernel, only trace user-space tracepoints (uprobes) 11532 */ 11533 if (event->attr.exclude_kernel && !user_mode(regs)) 11534 return 0; 11535 11536 if (!perf_tp_filter_match(event, raw)) 11537 return 0; 11538 11539 return 1; 11540 } 11541 11542 void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx, 11543 struct trace_event_call *call, u64 count, 11544 struct pt_regs *regs, struct hlist_head *head, 11545 struct task_struct *task) 11546 { 11547 if (bpf_prog_array_valid(call)) { 11548 *(struct pt_regs **)raw_data = regs; 11549 if (!trace_call_bpf(call, raw_data) || hlist_empty(head)) { 11550 perf_swevent_put_recursion_context(rctx); 11551 return; 11552 } 11553 } 11554 perf_tp_event(call->event.type, count, raw_data, size, regs, head, 11555 rctx, task); 11556 } 11557 EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit); 11558 11559 static void __perf_tp_event_target_task(u64 count, void *record, 11560 struct pt_regs *regs, 11561 struct perf_sample_data *data, 11562 struct perf_raw_record *raw, 11563 struct perf_event *event) 11564 { 11565 struct trace_entry *entry = record; 11566 11567 if (event->attr.config != entry->type) 11568 return; 11569 /* Cannot deliver synchronous signal to other task. */ 11570 if (event->attr.sigtrap) 11571 return; 11572 if (perf_tp_event_match(event, raw, regs)) { 11573 perf_sample_data_init(data, 0, 0); 11574 perf_sample_save_raw_data(data, event, raw); 11575 perf_swevent_event(event, count, data, regs); 11576 } 11577 } 11578 11579 static void perf_tp_event_target_task(u64 count, void *record, 11580 struct pt_regs *regs, 11581 struct perf_sample_data *data, 11582 struct perf_raw_record *raw, 11583 struct perf_event_context *ctx) 11584 { 11585 unsigned int cpu = smp_processor_id(); 11586 struct pmu *pmu = &perf_tracepoint; 11587 struct perf_event *event, *sibling; 11588 11589 perf_event_groups_for_cpu_pmu(event, &ctx->pinned_groups, cpu, pmu) { 11590 __perf_tp_event_target_task(count, record, regs, data, raw, event); 11591 for_each_sibling_event(sibling, event) 11592 __perf_tp_event_target_task(count, record, regs, data, raw, sibling); 11593 } 11594 11595 perf_event_groups_for_cpu_pmu(event, &ctx->flexible_groups, cpu, pmu) { 11596 __perf_tp_event_target_task(count, record, regs, data, raw, event); 11597 for_each_sibling_event(sibling, event) 11598 __perf_tp_event_target_task(count, record, regs, data, raw, sibling); 11599 } 11600 } 11601 11602 void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, 11603 struct pt_regs *regs, struct hlist_head *head, int rctx, 11604 struct task_struct *task) 11605 { 11606 struct perf_sample_data data; 11607 struct perf_event *event; 11608 11609 /* 11610 * Per being a tracepoint, this runs with preemption disabled. 11611 */ 11612 lockdep_assert_preemption_disabled(); 11613 11614 struct perf_raw_record raw = { 11615 .frag = { 11616 .size = entry_size, 11617 .data = record, 11618 }, 11619 }; 11620 11621 perf_trace_buf_update(record, event_type); 11622 11623 hlist_for_each_entry_rcu(event, head, hlist_entry) { 11624 if (perf_tp_event_match(event, &raw, regs)) { 11625 /* 11626 * Here use the same on-stack perf_sample_data, 11627 * some members in data are event-specific and 11628 * need to be re-computed for different sweveents. 11629 * Re-initialize data->sample_flags safely to avoid 11630 * the problem that next event skips preparing data 11631 * because data->sample_flags is set. 11632 */ 11633 perf_sample_data_init(&data, 0, 0); 11634 perf_sample_save_raw_data(&data, event, &raw); 11635 perf_swevent_event(event, count, &data, regs); 11636 } 11637 } 11638 11639 /* 11640 * If we got specified a target task, also iterate its context and 11641 * deliver this event there too. 11642 */ 11643 if (task && task != current) { 11644 struct perf_event_context *ctx; 11645 11646 rcu_read_lock(); 11647 ctx = rcu_dereference(task->perf_event_ctxp); 11648 if (!ctx) 11649 goto unlock; 11650 11651 raw_spin_lock(&ctx->lock); 11652 perf_tp_event_target_task(count, record, regs, &data, &raw, ctx); 11653 raw_spin_unlock(&ctx->lock); 11654 unlock: 11655 rcu_read_unlock(); 11656 } 11657 11658 perf_swevent_put_recursion_context(rctx); 11659 } 11660 EXPORT_SYMBOL_GPL(perf_tp_event); 11661 11662 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS) 11663 /* 11664 * Flags in config, used by dynamic PMU kprobe and uprobe 11665 * The flags should match following PMU_FORMAT_ATTR(). 11666 * 11667 * PERF_PROBE_CONFIG_IS_RETPROBE if set, create kretprobe/uretprobe 11668 * if not set, create kprobe/uprobe 11669 * 11670 * The following values specify a reference counter (or semaphore in the 11671 * terminology of tools like dtrace, systemtap, etc.) Userspace Statically 11672 * Defined Tracepoints (USDT). Currently, we use 40 bit for the offset. 11673 * 11674 * PERF_UPROBE_REF_CTR_OFFSET_BITS # of bits in config as th offset 11675 * PERF_UPROBE_REF_CTR_OFFSET_SHIFT # of bits to shift left 11676 */ 11677 enum perf_probe_config { 11678 PERF_PROBE_CONFIG_IS_RETPROBE = 1U << 0, /* [k,u]retprobe */ 11679 PERF_UPROBE_REF_CTR_OFFSET_BITS = 32, 11680 PERF_UPROBE_REF_CTR_OFFSET_SHIFT = 64 - PERF_UPROBE_REF_CTR_OFFSET_BITS, 11681 }; 11682 11683 PMU_FORMAT_ATTR(retprobe, "config:0"); 11684 #endif 11685 11686 #ifdef CONFIG_KPROBE_EVENTS 11687 static struct attribute *kprobe_attrs[] = { 11688 &format_attr_retprobe.attr, 11689 NULL, 11690 }; 11691 11692 static struct attribute_group kprobe_format_group = { 11693 .name = "format", 11694 .attrs = kprobe_attrs, 11695 }; 11696 11697 static const struct attribute_group *kprobe_attr_groups[] = { 11698 &kprobe_format_group, 11699 NULL, 11700 }; 11701 11702 static int perf_kprobe_event_init(struct perf_event *event); 11703 static struct pmu perf_kprobe = { 11704 .task_ctx_nr = perf_sw_context, 11705 .event_init = perf_kprobe_event_init, 11706 .add = perf_trace_add, 11707 .del = perf_trace_del, 11708 .start = perf_swevent_start, 11709 .stop = perf_swevent_stop, 11710 .read = perf_swevent_read, 11711 .attr_groups = kprobe_attr_groups, 11712 }; 11713 11714 static int perf_kprobe_event_init(struct perf_event *event) 11715 { 11716 int err; 11717 bool is_retprobe; 11718 11719 if (event->attr.type != perf_kprobe.type) 11720 return -ENOENT; 11721 11722 if (!perfmon_capable()) 11723 return -EACCES; 11724 11725 /* 11726 * no branch sampling for probe events 11727 */ 11728 if (has_branch_stack(event)) 11729 return -EOPNOTSUPP; 11730 11731 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; 11732 err = perf_kprobe_init(event, is_retprobe); 11733 if (err) 11734 return err; 11735 11736 event->destroy = perf_kprobe_destroy; 11737 11738 return 0; 11739 } 11740 #endif /* CONFIG_KPROBE_EVENTS */ 11741 11742 #ifdef CONFIG_UPROBE_EVENTS 11743 PMU_FORMAT_ATTR(ref_ctr_offset, "config:32-63"); 11744 11745 static struct attribute *uprobe_attrs[] = { 11746 &format_attr_retprobe.attr, 11747 &format_attr_ref_ctr_offset.attr, 11748 NULL, 11749 }; 11750 11751 static struct attribute_group uprobe_format_group = { 11752 .name = "format", 11753 .attrs = uprobe_attrs, 11754 }; 11755 11756 static const struct attribute_group *uprobe_attr_groups[] = { 11757 &uprobe_format_group, 11758 NULL, 11759 }; 11760 11761 static int perf_uprobe_event_init(struct perf_event *event); 11762 static struct pmu perf_uprobe = { 11763 .task_ctx_nr = perf_sw_context, 11764 .event_init = perf_uprobe_event_init, 11765 .add = perf_trace_add, 11766 .del = perf_trace_del, 11767 .start = perf_swevent_start, 11768 .stop = perf_swevent_stop, 11769 .read = perf_swevent_read, 11770 .attr_groups = uprobe_attr_groups, 11771 }; 11772 11773 static int perf_uprobe_event_init(struct perf_event *event) 11774 { 11775 int err; 11776 unsigned long ref_ctr_offset; 11777 bool is_retprobe; 11778 11779 if (event->attr.type != perf_uprobe.type) 11780 return -ENOENT; 11781 11782 if (!capable(CAP_SYS_ADMIN)) 11783 return -EACCES; 11784 11785 /* 11786 * no branch sampling for probe events 11787 */ 11788 if (has_branch_stack(event)) 11789 return -EOPNOTSUPP; 11790 11791 is_retprobe = event->attr.config & PERF_PROBE_CONFIG_IS_RETPROBE; 11792 ref_ctr_offset = event->attr.config >> PERF_UPROBE_REF_CTR_OFFSET_SHIFT; 11793 err = perf_uprobe_init(event, ref_ctr_offset, is_retprobe); 11794 if (err) 11795 return err; 11796 11797 event->destroy = perf_uprobe_destroy; 11798 11799 return 0; 11800 } 11801 #endif /* CONFIG_UPROBE_EVENTS */ 11802 11803 static inline void perf_tp_register(void) 11804 { 11805 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT); 11806 #ifdef CONFIG_KPROBE_EVENTS 11807 perf_pmu_register(&perf_kprobe, "kprobe", -1); 11808 #endif 11809 #ifdef CONFIG_UPROBE_EVENTS 11810 perf_pmu_register(&perf_uprobe, "uprobe", -1); 11811 #endif 11812 } 11813 11814 static void perf_event_free_filter(struct perf_event *event) 11815 { 11816 ftrace_profile_free_filter(event); 11817 } 11818 11819 /* 11820 * returns true if the event is a tracepoint, or a kprobe/upprobe created 11821 * with perf_event_open() 11822 */ 11823 static inline bool perf_event_is_tracing(struct perf_event *event) 11824 { 11825 if (event->pmu == &perf_tracepoint) 11826 return true; 11827 #ifdef CONFIG_KPROBE_EVENTS 11828 if (event->pmu == &perf_kprobe) 11829 return true; 11830 #endif 11831 #ifdef CONFIG_UPROBE_EVENTS 11832 if (event->pmu == &perf_uprobe) 11833 return true; 11834 #endif 11835 return false; 11836 } 11837 11838 static int __perf_event_set_bpf_prog(struct perf_event *event, 11839 struct bpf_prog *prog, 11840 u64 bpf_cookie) 11841 { 11842 bool is_kprobe, is_uprobe, is_tracepoint, is_syscall_tp; 11843 11844 if (event->state <= PERF_EVENT_STATE_REVOKED) 11845 return -ENODEV; 11846 11847 if (!perf_event_is_tracing(event)) 11848 return perf_event_set_bpf_handler(event, prog, bpf_cookie); 11849 11850 is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_KPROBE; 11851 is_uprobe = event->tp_event->flags & TRACE_EVENT_FL_UPROBE; 11852 is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT; 11853 is_syscall_tp = is_syscall_trace_event(event->tp_event); 11854 if (!is_kprobe && !is_uprobe && !is_tracepoint && !is_syscall_tp) 11855 /* bpf programs can only be attached to u/kprobe or tracepoint */ 11856 return -EINVAL; 11857 11858 if (((is_kprobe || is_uprobe) && prog->type != BPF_PROG_TYPE_KPROBE) || 11859 (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT) || 11860 (is_syscall_tp && prog->type != BPF_PROG_TYPE_TRACEPOINT)) 11861 return -EINVAL; 11862 11863 if (prog->type == BPF_PROG_TYPE_KPROBE && prog->sleepable && !is_uprobe) 11864 /* only uprobe programs are allowed to be sleepable */ 11865 return -EINVAL; 11866 11867 if (prog->type == BPF_PROG_TYPE_TRACEPOINT && prog->sleepable) { 11868 /* 11869 * Sleepable tracepoint programs can only attach to faultable 11870 * tracepoints. Currently only syscall tracepoints are faultable. 11871 */ 11872 if (!is_syscall_tp) 11873 return -EINVAL; 11874 } 11875 11876 /* Kprobe override only works for kprobes, not uprobes. */ 11877 if (prog->kprobe_override && !is_kprobe) 11878 return -EINVAL; 11879 11880 /* Writing to context allowed only for uprobes. */ 11881 if (prog->aux->kprobe_write_ctx && !is_uprobe) 11882 return -EINVAL; 11883 11884 if (is_tracepoint || is_syscall_tp) { 11885 int off = trace_event_get_offsets(event->tp_event); 11886 11887 if (prog->aux->max_ctx_offset > off) 11888 return -EACCES; 11889 } 11890 11891 return perf_event_attach_bpf_prog(event, prog, bpf_cookie); 11892 } 11893 11894 int perf_event_set_bpf_prog(struct perf_event *event, 11895 struct bpf_prog *prog, 11896 u64 bpf_cookie) 11897 { 11898 struct perf_event_context *ctx; 11899 int ret; 11900 11901 ctx = perf_event_ctx_lock(event); 11902 ret = __perf_event_set_bpf_prog(event, prog, bpf_cookie); 11903 perf_event_ctx_unlock(event, ctx); 11904 11905 return ret; 11906 } 11907 11908 void perf_event_free_bpf_prog(struct perf_event *event) 11909 { 11910 if (!event->prog) 11911 return; 11912 11913 if (!perf_event_is_tracing(event)) { 11914 perf_event_free_bpf_handler(event); 11915 return; 11916 } 11917 perf_event_detach_bpf_prog(event); 11918 } 11919 11920 #else 11921 11922 static inline void perf_tp_register(void) 11923 { 11924 } 11925 11926 static void perf_event_free_filter(struct perf_event *event) 11927 { 11928 } 11929 11930 static int __perf_event_set_bpf_prog(struct perf_event *event, 11931 struct bpf_prog *prog, 11932 u64 bpf_cookie) 11933 { 11934 return -ENOENT; 11935 } 11936 11937 int perf_event_set_bpf_prog(struct perf_event *event, 11938 struct bpf_prog *prog, 11939 u64 bpf_cookie) 11940 { 11941 return -ENOENT; 11942 } 11943 11944 void perf_event_free_bpf_prog(struct perf_event *event) 11945 { 11946 } 11947 #endif /* CONFIG_EVENT_TRACING */ 11948 11949 #ifdef CONFIG_HAVE_HW_BREAKPOINT 11950 void perf_bp_event(struct perf_event *bp, void *data) 11951 { 11952 struct perf_sample_data sample; 11953 struct pt_regs *regs = data; 11954 11955 /* 11956 * Exception context, will have interrupts disabled. 11957 */ 11958 lockdep_assert_irqs_disabled(); 11959 11960 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); 11961 11962 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 11963 perf_swevent_event(bp, 1, &sample, regs); 11964 } 11965 #endif 11966 11967 /* 11968 * Allocate a new address filter 11969 */ 11970 static struct perf_addr_filter * 11971 perf_addr_filter_new(struct perf_event *event, struct list_head *filters) 11972 { 11973 int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu); 11974 struct perf_addr_filter *filter; 11975 11976 filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node); 11977 if (!filter) 11978 return NULL; 11979 11980 INIT_LIST_HEAD(&filter->entry); 11981 list_add_tail(&filter->entry, filters); 11982 11983 return filter; 11984 } 11985 11986 static void free_filters_list(struct list_head *filters) 11987 { 11988 struct perf_addr_filter *filter, *iter; 11989 11990 list_for_each_entry_safe(filter, iter, filters, entry) { 11991 path_put(&filter->path); 11992 list_del(&filter->entry); 11993 kfree(filter); 11994 } 11995 } 11996 11997 /* 11998 * Free existing address filters and optionally install new ones 11999 */ 12000 static void perf_addr_filters_splice(struct perf_event *event, 12001 struct list_head *head) 12002 { 12003 unsigned long flags; 12004 LIST_HEAD(list); 12005 12006 if (!has_addr_filter(event)) 12007 return; 12008 12009 /* don't bother with children, they don't have their own filters */ 12010 if (event->parent) 12011 return; 12012 12013 raw_spin_lock_irqsave(&event->addr_filters.lock, flags); 12014 12015 list_splice_init(&event->addr_filters.list, &list); 12016 if (head) 12017 list_splice(head, &event->addr_filters.list); 12018 12019 raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags); 12020 12021 free_filters_list(&list); 12022 } 12023 12024 static void perf_free_addr_filters(struct perf_event *event) 12025 { 12026 /* 12027 * Used during free paths, there is no concurrency. 12028 */ 12029 if (list_empty(&event->addr_filters.list)) 12030 return; 12031 12032 perf_addr_filters_splice(event, NULL); 12033 } 12034 12035 /* 12036 * Scan through mm's vmas and see if one of them matches the 12037 * @filter; if so, adjust filter's address range. 12038 * Called with mm::mmap_lock down for reading. 12039 */ 12040 static void perf_addr_filter_apply(struct perf_addr_filter *filter, 12041 struct mm_struct *mm, 12042 struct perf_addr_filter_range *fr) 12043 { 12044 struct vm_area_struct *vma; 12045 VMA_ITERATOR(vmi, mm, 0); 12046 12047 for_each_vma(vmi, vma) { 12048 if (!vma->vm_file) 12049 continue; 12050 12051 if (perf_addr_filter_vma_adjust(filter, vma, fr)) 12052 return; 12053 } 12054 } 12055 12056 /* 12057 * Update event's address range filters based on the 12058 * task's existing mappings, if any. 12059 */ 12060 static void perf_event_addr_filters_apply(struct perf_event *event) 12061 { 12062 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 12063 struct task_struct *task = READ_ONCE(event->ctx->task); 12064 struct perf_addr_filter *filter; 12065 struct mm_struct *mm = NULL; 12066 unsigned int count = 0; 12067 unsigned long flags; 12068 12069 /* 12070 * We may observe TASK_TOMBSTONE, which means that the event tear-down 12071 * will stop on the parent's child_mutex that our caller is also holding 12072 */ 12073 if (task == TASK_TOMBSTONE) 12074 return; 12075 12076 if (ifh->nr_file_filters) { 12077 mm = get_task_mm(task); 12078 if (!mm) 12079 goto restart; 12080 12081 mmap_read_lock(mm); 12082 } 12083 12084 raw_spin_lock_irqsave(&ifh->lock, flags); 12085 list_for_each_entry(filter, &ifh->list, entry) { 12086 if (filter->path.dentry) { 12087 /* 12088 * Adjust base offset if the filter is associated to a 12089 * binary that needs to be mapped: 12090 */ 12091 event->addr_filter_ranges[count].start = 0; 12092 event->addr_filter_ranges[count].size = 0; 12093 12094 perf_addr_filter_apply(filter, mm, &event->addr_filter_ranges[count]); 12095 } else { 12096 event->addr_filter_ranges[count].start = filter->offset; 12097 event->addr_filter_ranges[count].size = filter->size; 12098 } 12099 12100 count++; 12101 } 12102 12103 event->addr_filters_gen++; 12104 raw_spin_unlock_irqrestore(&ifh->lock, flags); 12105 12106 if (ifh->nr_file_filters) { 12107 mmap_read_unlock(mm); 12108 12109 mmput(mm); 12110 } 12111 12112 restart: 12113 perf_event_stop(event, 1); 12114 } 12115 12116 /* 12117 * Address range filtering: limiting the data to certain 12118 * instruction address ranges. Filters are ioctl()ed to us from 12119 * userspace as ascii strings. 12120 * 12121 * Filter string format: 12122 * 12123 * ACTION RANGE_SPEC 12124 * where ACTION is one of the 12125 * * "filter": limit the trace to this region 12126 * * "start": start tracing from this address 12127 * * "stop": stop tracing at this address/region; 12128 * RANGE_SPEC is 12129 * * for kernel addresses: <start address>[/<size>] 12130 * * for object files: <start address>[/<size>]@</path/to/object/file> 12131 * 12132 * if <size> is not specified or is zero, the range is treated as a single 12133 * address; not valid for ACTION=="filter". 12134 */ 12135 enum { 12136 IF_ACT_NONE = -1, 12137 IF_ACT_FILTER, 12138 IF_ACT_START, 12139 IF_ACT_STOP, 12140 IF_SRC_FILE, 12141 IF_SRC_KERNEL, 12142 IF_SRC_FILEADDR, 12143 IF_SRC_KERNELADDR, 12144 }; 12145 12146 enum { 12147 IF_STATE_ACTION = 0, 12148 IF_STATE_SOURCE, 12149 IF_STATE_END, 12150 }; 12151 12152 static const match_table_t if_tokens = { 12153 { IF_ACT_FILTER, "filter" }, 12154 { IF_ACT_START, "start" }, 12155 { IF_ACT_STOP, "stop" }, 12156 { IF_SRC_FILE, "%u/%u@%s" }, 12157 { IF_SRC_KERNEL, "%u/%u" }, 12158 { IF_SRC_FILEADDR, "%u@%s" }, 12159 { IF_SRC_KERNELADDR, "%u" }, 12160 { IF_ACT_NONE, NULL }, 12161 }; 12162 12163 /* 12164 * Address filter string parser 12165 */ 12166 static int 12167 perf_event_parse_addr_filter(struct perf_event *event, char *fstr, 12168 struct list_head *filters) 12169 { 12170 struct perf_addr_filter *filter = NULL; 12171 char *start, *orig, *filename = NULL; 12172 substring_t args[MAX_OPT_ARGS]; 12173 int state = IF_STATE_ACTION, token; 12174 unsigned int kernel = 0; 12175 int ret = -EINVAL; 12176 12177 orig = fstr = kstrdup(fstr, GFP_KERNEL); 12178 if (!fstr) 12179 return -ENOMEM; 12180 12181 while ((start = strsep(&fstr, " ,\n")) != NULL) { 12182 static const enum perf_addr_filter_action_t actions[] = { 12183 [IF_ACT_FILTER] = PERF_ADDR_FILTER_ACTION_FILTER, 12184 [IF_ACT_START] = PERF_ADDR_FILTER_ACTION_START, 12185 [IF_ACT_STOP] = PERF_ADDR_FILTER_ACTION_STOP, 12186 }; 12187 ret = -EINVAL; 12188 12189 if (!*start) 12190 continue; 12191 12192 /* filter definition begins */ 12193 if (state == IF_STATE_ACTION) { 12194 filter = perf_addr_filter_new(event, filters); 12195 if (!filter) 12196 goto fail; 12197 } 12198 12199 token = match_token(start, if_tokens, args); 12200 switch (token) { 12201 case IF_ACT_FILTER: 12202 case IF_ACT_START: 12203 case IF_ACT_STOP: 12204 if (state != IF_STATE_ACTION) 12205 goto fail; 12206 12207 filter->action = actions[token]; 12208 state = IF_STATE_SOURCE; 12209 break; 12210 12211 case IF_SRC_KERNELADDR: 12212 case IF_SRC_KERNEL: 12213 kernel = 1; 12214 fallthrough; 12215 12216 case IF_SRC_FILEADDR: 12217 case IF_SRC_FILE: 12218 if (state != IF_STATE_SOURCE) 12219 goto fail; 12220 12221 *args[0].to = 0; 12222 ret = kstrtoul(args[0].from, 0, &filter->offset); 12223 if (ret) 12224 goto fail; 12225 12226 if (token == IF_SRC_KERNEL || token == IF_SRC_FILE) { 12227 *args[1].to = 0; 12228 ret = kstrtoul(args[1].from, 0, &filter->size); 12229 if (ret) 12230 goto fail; 12231 } 12232 12233 if (token == IF_SRC_FILE || token == IF_SRC_FILEADDR) { 12234 int fpos = token == IF_SRC_FILE ? 2 : 1; 12235 12236 kfree(filename); 12237 filename = match_strdup(&args[fpos]); 12238 if (!filename) { 12239 ret = -ENOMEM; 12240 goto fail; 12241 } 12242 } 12243 12244 state = IF_STATE_END; 12245 break; 12246 12247 default: 12248 goto fail; 12249 } 12250 12251 /* 12252 * Filter definition is fully parsed, validate and install it. 12253 * Make sure that it doesn't contradict itself or the event's 12254 * attribute. 12255 */ 12256 if (state == IF_STATE_END) { 12257 ret = -EINVAL; 12258 12259 /* 12260 * ACTION "filter" must have a non-zero length region 12261 * specified. 12262 */ 12263 if (filter->action == PERF_ADDR_FILTER_ACTION_FILTER && 12264 !filter->size) 12265 goto fail; 12266 12267 if (!kernel) { 12268 if (!filename) 12269 goto fail; 12270 12271 /* 12272 * For now, we only support file-based filters 12273 * in per-task events; doing so for CPU-wide 12274 * events requires additional context switching 12275 * trickery, since same object code will be 12276 * mapped at different virtual addresses in 12277 * different processes. 12278 */ 12279 ret = -EOPNOTSUPP; 12280 if (!event->ctx->task) 12281 goto fail; 12282 12283 /* look up the path and grab its inode */ 12284 ret = kern_path(filename, LOOKUP_FOLLOW, 12285 &filter->path); 12286 if (ret) 12287 goto fail; 12288 12289 ret = -EINVAL; 12290 if (!filter->path.dentry || 12291 !S_ISREG(d_inode(filter->path.dentry) 12292 ->i_mode)) 12293 goto fail; 12294 12295 event->addr_filters.nr_file_filters++; 12296 } 12297 12298 /* ready to consume more filters */ 12299 kfree(filename); 12300 filename = NULL; 12301 state = IF_STATE_ACTION; 12302 filter = NULL; 12303 kernel = 0; 12304 } 12305 } 12306 12307 if (state != IF_STATE_ACTION) 12308 goto fail; 12309 12310 kfree(filename); 12311 kfree(orig); 12312 12313 return 0; 12314 12315 fail: 12316 kfree(filename); 12317 free_filters_list(filters); 12318 kfree(orig); 12319 12320 return ret; 12321 } 12322 12323 static int 12324 perf_event_set_addr_filter(struct perf_event *event, char *filter_str) 12325 { 12326 LIST_HEAD(filters); 12327 int ret; 12328 12329 /* 12330 * Since this is called in perf_ioctl() path, we're already holding 12331 * ctx::mutex. 12332 */ 12333 lockdep_assert_held(&event->ctx->mutex); 12334 12335 if (WARN_ON_ONCE(event->parent)) 12336 return -EINVAL; 12337 12338 ret = perf_event_parse_addr_filter(event, filter_str, &filters); 12339 if (ret) 12340 goto fail_clear_files; 12341 12342 ret = event->pmu->addr_filters_validate(&filters); 12343 if (ret) 12344 goto fail_free_filters; 12345 12346 /* remove existing filters, if any */ 12347 perf_addr_filters_splice(event, &filters); 12348 12349 /* install new filters */ 12350 perf_event_for_each_child(event, perf_event_addr_filters_apply); 12351 12352 return ret; 12353 12354 fail_free_filters: 12355 free_filters_list(&filters); 12356 12357 fail_clear_files: 12358 event->addr_filters.nr_file_filters = 0; 12359 12360 return ret; 12361 } 12362 12363 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 12364 { 12365 int ret = -EINVAL; 12366 char *filter_str; 12367 12368 filter_str = strndup_user(arg, PAGE_SIZE); 12369 if (IS_ERR(filter_str)) 12370 return PTR_ERR(filter_str); 12371 12372 #ifdef CONFIG_EVENT_TRACING 12373 if (perf_event_is_tracing(event)) { 12374 struct perf_event_context *ctx = event->ctx; 12375 12376 /* 12377 * Beware, here be dragons!! 12378 * 12379 * the tracepoint muck will deadlock against ctx->mutex, but 12380 * the tracepoint stuff does not actually need it. So 12381 * temporarily drop ctx->mutex. As per perf_event_ctx_lock() we 12382 * already have a reference on ctx. 12383 * 12384 * This can result in event getting moved to a different ctx, 12385 * but that does not affect the tracepoint state. 12386 */ 12387 mutex_unlock(&ctx->mutex); 12388 ret = ftrace_profile_set_filter(event, event->attr.config, filter_str); 12389 mutex_lock(&ctx->mutex); 12390 } else 12391 #endif 12392 if (has_addr_filter(event)) 12393 ret = perf_event_set_addr_filter(event, filter_str); 12394 12395 kfree(filter_str); 12396 return ret; 12397 } 12398 12399 /* 12400 * hrtimer based swevent callback 12401 */ 12402 12403 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) 12404 { 12405 enum hrtimer_restart ret = HRTIMER_RESTART; 12406 struct perf_sample_data data; 12407 struct pt_regs *regs; 12408 struct perf_event *event; 12409 u64 period; 12410 12411 event = container_of(hrtimer, struct perf_event, hw.hrtimer); 12412 12413 if (event->state != PERF_EVENT_STATE_ACTIVE || 12414 event->hw.state & PERF_HES_STOPPED) 12415 return HRTIMER_NORESTART; 12416 12417 event->pmu->read(event); 12418 12419 perf_sample_data_init(&data, 0, event->hw.last_period); 12420 regs = get_irq_regs(); 12421 12422 if (regs && !perf_exclude_event(event, regs)) { 12423 if (!(event->attr.exclude_idle && is_idle_task(current))) 12424 if (perf_event_overflow(event, &data, regs)) 12425 ret = HRTIMER_NORESTART; 12426 } 12427 12428 period = max_t(u64, 10000, event->hw.sample_period); 12429 hrtimer_forward_now(hrtimer, ns_to_ktime(period)); 12430 12431 return ret; 12432 } 12433 12434 static void perf_swevent_start_hrtimer(struct perf_event *event) 12435 { 12436 struct hw_perf_event *hwc = &event->hw; 12437 s64 period; 12438 12439 if (!is_sampling_event(event)) 12440 return; 12441 12442 period = local64_read(&hwc->period_left); 12443 if (period) { 12444 if (period < 0) 12445 period = 10000; 12446 12447 local64_set(&hwc->period_left, 0); 12448 } else { 12449 period = max_t(u64, 10000, hwc->sample_period); 12450 } 12451 hrtimer_start(&hwc->hrtimer, ns_to_ktime(period), 12452 HRTIMER_MODE_REL_PINNED_HARD); 12453 } 12454 12455 static void perf_swevent_cancel_hrtimer(struct perf_event *event) 12456 { 12457 struct hw_perf_event *hwc = &event->hw; 12458 12459 /* 12460 * Careful: this function can be triggered in the hrtimer handler, 12461 * for cpu-clock events, so hrtimer_cancel() would cause a 12462 * deadlock. 12463 * 12464 * So use hrtimer_try_to_cancel() to try to stop the hrtimer, 12465 * and the cpu-clock handler also sets the PERF_HES_STOPPED flag, 12466 * which guarantees that perf_swevent_hrtimer() will stop the 12467 * hrtimer once it sees the PERF_HES_STOPPED flag. 12468 */ 12469 if (is_sampling_event(event) && (hwc->interrupts != MAX_INTERRUPTS)) { 12470 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); 12471 local64_set(&hwc->period_left, ktime_to_ns(remaining)); 12472 12473 hrtimer_try_to_cancel(&hwc->hrtimer); 12474 } 12475 } 12476 12477 static void perf_swevent_destroy_hrtimer(struct perf_event *event) 12478 { 12479 hrtimer_cancel(&event->hw.hrtimer); 12480 } 12481 12482 static void perf_swevent_init_hrtimer(struct perf_event *event) 12483 { 12484 struct hw_perf_event *hwc = &event->hw; 12485 12486 if (!is_sampling_event(event)) 12487 return; 12488 12489 hrtimer_setup(&hwc->hrtimer, perf_swevent_hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD); 12490 event->destroy = perf_swevent_destroy_hrtimer; 12491 12492 /* 12493 * Since hrtimers have a fixed rate, we can do a static freq->period 12494 * mapping and avoid the whole period adjust feedback stuff. 12495 */ 12496 if (event->attr.freq) { 12497 long freq = event->attr.sample_freq; 12498 12499 event->attr.sample_period = NSEC_PER_SEC / freq; 12500 hwc->sample_period = event->attr.sample_period; 12501 local64_set(&hwc->period_left, hwc->sample_period); 12502 hwc->last_period = hwc->sample_period; 12503 event->attr.freq = 0; 12504 } 12505 } 12506 12507 /* 12508 * Software event: cpu wall time clock 12509 */ 12510 12511 static void cpu_clock_event_update(struct perf_event *event) 12512 { 12513 s64 prev; 12514 u64 now; 12515 12516 now = local_clock(); 12517 prev = local64_xchg(&event->hw.prev_count, now); 12518 local64_add(now - prev, &event->count); 12519 } 12520 12521 static void cpu_clock_event_start(struct perf_event *event, int flags) 12522 { 12523 event->hw.state = 0; 12524 local64_set(&event->hw.prev_count, local_clock()); 12525 perf_swevent_start_hrtimer(event); 12526 } 12527 12528 static void cpu_clock_event_stop(struct perf_event *event, int flags) 12529 { 12530 event->hw.state = PERF_HES_STOPPED; 12531 perf_swevent_cancel_hrtimer(event); 12532 if (flags & PERF_EF_UPDATE) 12533 cpu_clock_event_update(event); 12534 } 12535 12536 static int cpu_clock_event_add(struct perf_event *event, int flags) 12537 { 12538 if (flags & PERF_EF_START) 12539 cpu_clock_event_start(event, flags); 12540 perf_event_update_userpage(event); 12541 12542 return 0; 12543 } 12544 12545 static void cpu_clock_event_del(struct perf_event *event, int flags) 12546 { 12547 cpu_clock_event_stop(event, PERF_EF_UPDATE); 12548 } 12549 12550 static void cpu_clock_event_read(struct perf_event *event) 12551 { 12552 cpu_clock_event_update(event); 12553 } 12554 12555 static int cpu_clock_event_init(struct perf_event *event) 12556 { 12557 if (event->attr.type != perf_cpu_clock.type) 12558 return -ENOENT; 12559 12560 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) 12561 return -ENOENT; 12562 12563 /* 12564 * no branch sampling for software events 12565 */ 12566 if (has_branch_stack(event)) 12567 return -EOPNOTSUPP; 12568 12569 perf_swevent_init_hrtimer(event); 12570 12571 return 0; 12572 } 12573 12574 static struct pmu perf_cpu_clock = { 12575 .task_ctx_nr = perf_sw_context, 12576 12577 .capabilities = PERF_PMU_CAP_NO_NMI, 12578 .dev = PMU_NULL_DEV, 12579 12580 .event_init = cpu_clock_event_init, 12581 .add = cpu_clock_event_add, 12582 .del = cpu_clock_event_del, 12583 .start = cpu_clock_event_start, 12584 .stop = cpu_clock_event_stop, 12585 .read = cpu_clock_event_read, 12586 }; 12587 12588 /* 12589 * Software event: task time clock 12590 */ 12591 12592 static void task_clock_event_update(struct perf_event *event, u64 now) 12593 { 12594 u64 prev; 12595 s64 delta; 12596 12597 prev = local64_xchg(&event->hw.prev_count, now); 12598 delta = now - prev; 12599 local64_add(delta, &event->count); 12600 } 12601 12602 static void task_clock_event_start(struct perf_event *event, int flags) 12603 { 12604 event->hw.state = 0; 12605 local64_set(&event->hw.prev_count, event->ctx->time.time); 12606 perf_swevent_start_hrtimer(event); 12607 } 12608 12609 static void task_clock_event_stop(struct perf_event *event, int flags) 12610 { 12611 event->hw.state = PERF_HES_STOPPED; 12612 perf_swevent_cancel_hrtimer(event); 12613 if (flags & PERF_EF_UPDATE) 12614 task_clock_event_update(event, event->ctx->time.time); 12615 } 12616 12617 static int task_clock_event_add(struct perf_event *event, int flags) 12618 { 12619 if (flags & PERF_EF_START) 12620 task_clock_event_start(event, flags); 12621 perf_event_update_userpage(event); 12622 12623 return 0; 12624 } 12625 12626 static void task_clock_event_del(struct perf_event *event, int flags) 12627 { 12628 task_clock_event_stop(event, PERF_EF_UPDATE); 12629 } 12630 12631 static void task_clock_event_read(struct perf_event *event) 12632 { 12633 u64 now = perf_clock(); 12634 u64 delta = now - event->ctx->time.stamp; 12635 u64 time = event->ctx->time.time + delta; 12636 12637 task_clock_event_update(event, time); 12638 } 12639 12640 static int task_clock_event_init(struct perf_event *event) 12641 { 12642 if (event->attr.type != perf_task_clock.type) 12643 return -ENOENT; 12644 12645 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) 12646 return -ENOENT; 12647 12648 /* 12649 * no branch sampling for software events 12650 */ 12651 if (has_branch_stack(event)) 12652 return -EOPNOTSUPP; 12653 12654 perf_swevent_init_hrtimer(event); 12655 12656 return 0; 12657 } 12658 12659 static struct pmu perf_task_clock = { 12660 .task_ctx_nr = perf_sw_context, 12661 12662 .capabilities = PERF_PMU_CAP_NO_NMI, 12663 .dev = PMU_NULL_DEV, 12664 12665 .event_init = task_clock_event_init, 12666 .add = task_clock_event_add, 12667 .del = task_clock_event_del, 12668 .start = task_clock_event_start, 12669 .stop = task_clock_event_stop, 12670 .read = task_clock_event_read, 12671 }; 12672 12673 static void perf_pmu_nop_void(struct pmu *pmu) 12674 { 12675 } 12676 12677 static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags) 12678 { 12679 } 12680 12681 static int perf_pmu_nop_int(struct pmu *pmu) 12682 { 12683 return 0; 12684 } 12685 12686 static int perf_event_nop_int(struct perf_event *event, u64 value) 12687 { 12688 return 0; 12689 } 12690 12691 static DEFINE_PER_CPU(unsigned int, nop_txn_flags); 12692 12693 static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags) 12694 { 12695 __this_cpu_write(nop_txn_flags, flags); 12696 12697 if (flags & ~PERF_PMU_TXN_ADD) 12698 return; 12699 12700 perf_pmu_disable(pmu); 12701 } 12702 12703 static int perf_pmu_commit_txn(struct pmu *pmu) 12704 { 12705 unsigned int flags = __this_cpu_read(nop_txn_flags); 12706 12707 __this_cpu_write(nop_txn_flags, 0); 12708 12709 if (flags & ~PERF_PMU_TXN_ADD) 12710 return 0; 12711 12712 perf_pmu_enable(pmu); 12713 return 0; 12714 } 12715 12716 static void perf_pmu_cancel_txn(struct pmu *pmu) 12717 { 12718 unsigned int flags = __this_cpu_read(nop_txn_flags); 12719 12720 __this_cpu_write(nop_txn_flags, 0); 12721 12722 if (flags & ~PERF_PMU_TXN_ADD) 12723 return; 12724 12725 perf_pmu_enable(pmu); 12726 } 12727 12728 static int perf_event_idx_default(struct perf_event *event) 12729 { 12730 return 0; 12731 } 12732 12733 /* 12734 * Let userspace know that this PMU supports address range filtering: 12735 */ 12736 static ssize_t nr_addr_filters_show(struct device *dev, 12737 struct device_attribute *attr, 12738 char *page) 12739 { 12740 struct pmu *pmu = dev_get_drvdata(dev); 12741 12742 return sysfs_emit(page, "%d\n", pmu->nr_addr_filters); 12743 } 12744 DEVICE_ATTR_RO(nr_addr_filters); 12745 12746 static struct idr pmu_idr; 12747 12748 static ssize_t 12749 type_show(struct device *dev, struct device_attribute *attr, char *page) 12750 { 12751 struct pmu *pmu = dev_get_drvdata(dev); 12752 12753 return sysfs_emit(page, "%d\n", pmu->type); 12754 } 12755 static DEVICE_ATTR_RO(type); 12756 12757 static ssize_t 12758 perf_event_mux_interval_ms_show(struct device *dev, 12759 struct device_attribute *attr, 12760 char *page) 12761 { 12762 struct pmu *pmu = dev_get_drvdata(dev); 12763 12764 return sysfs_emit(page, "%d\n", pmu->hrtimer_interval_ms); 12765 } 12766 12767 static DEFINE_MUTEX(mux_interval_mutex); 12768 12769 static ssize_t 12770 perf_event_mux_interval_ms_store(struct device *dev, 12771 struct device_attribute *attr, 12772 const char *buf, size_t count) 12773 { 12774 struct pmu *pmu = dev_get_drvdata(dev); 12775 int timer, cpu, ret; 12776 12777 ret = kstrtoint(buf, 0, &timer); 12778 if (ret) 12779 return ret; 12780 12781 if (timer < 1) 12782 return -EINVAL; 12783 12784 /* same value, noting to do */ 12785 if (timer == pmu->hrtimer_interval_ms) 12786 return count; 12787 12788 mutex_lock(&mux_interval_mutex); 12789 pmu->hrtimer_interval_ms = timer; 12790 12791 /* update all cpuctx for this PMU */ 12792 cpus_read_lock(); 12793 for_each_online_cpu(cpu) 12794 cpu_function_call(cpu, perf_mux_hrtimer_interval_ipi, pmu); 12795 cpus_read_unlock(); 12796 mutex_unlock(&mux_interval_mutex); 12797 12798 return count; 12799 } 12800 static DEVICE_ATTR_RW(perf_event_mux_interval_ms); 12801 12802 static inline const struct cpumask *perf_scope_cpu_topology_cpumask(unsigned int scope, int cpu) 12803 { 12804 switch (scope) { 12805 case PERF_PMU_SCOPE_CORE: 12806 return topology_sibling_cpumask(cpu); 12807 case PERF_PMU_SCOPE_DIE: 12808 return topology_die_cpumask(cpu); 12809 case PERF_PMU_SCOPE_CLUSTER: 12810 return topology_cluster_cpumask(cpu); 12811 case PERF_PMU_SCOPE_PKG: 12812 return topology_core_cpumask(cpu); 12813 case PERF_PMU_SCOPE_SYS_WIDE: 12814 return cpu_online_mask; 12815 } 12816 12817 return NULL; 12818 } 12819 12820 static inline struct cpumask *perf_scope_cpumask(unsigned int scope) 12821 { 12822 switch (scope) { 12823 case PERF_PMU_SCOPE_CORE: 12824 return perf_online_core_mask; 12825 case PERF_PMU_SCOPE_DIE: 12826 return perf_online_die_mask; 12827 case PERF_PMU_SCOPE_CLUSTER: 12828 return perf_online_cluster_mask; 12829 case PERF_PMU_SCOPE_PKG: 12830 return perf_online_pkg_mask; 12831 case PERF_PMU_SCOPE_SYS_WIDE: 12832 return perf_online_sys_mask; 12833 } 12834 12835 return NULL; 12836 } 12837 12838 static ssize_t cpumask_show(struct device *dev, struct device_attribute *attr, 12839 char *buf) 12840 { 12841 struct pmu *pmu = dev_get_drvdata(dev); 12842 struct cpumask *mask = perf_scope_cpumask(pmu->scope); 12843 12844 if (mask) 12845 return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(mask)); 12846 return 0; 12847 } 12848 12849 static DEVICE_ATTR_RO(cpumask); 12850 12851 static struct attribute *pmu_dev_attrs[] = { 12852 &dev_attr_type.attr, 12853 &dev_attr_perf_event_mux_interval_ms.attr, 12854 &dev_attr_nr_addr_filters.attr, 12855 &dev_attr_cpumask.attr, 12856 NULL, 12857 }; 12858 12859 static umode_t pmu_dev_is_visible(struct kobject *kobj, struct attribute *a, int n) 12860 { 12861 struct device *dev = kobj_to_dev(kobj); 12862 struct pmu *pmu = dev_get_drvdata(dev); 12863 12864 if (n == 2 && !pmu->nr_addr_filters) 12865 return 0; 12866 12867 /* cpumask */ 12868 if (n == 3 && pmu->scope == PERF_PMU_SCOPE_NONE) 12869 return 0; 12870 12871 return a->mode; 12872 } 12873 12874 static struct attribute_group pmu_dev_attr_group = { 12875 .is_visible = pmu_dev_is_visible, 12876 .attrs = pmu_dev_attrs, 12877 }; 12878 12879 static const struct attribute_group *pmu_dev_groups[] = { 12880 &pmu_dev_attr_group, 12881 NULL, 12882 }; 12883 12884 static int pmu_bus_running; 12885 static const struct bus_type pmu_bus = { 12886 .name = "event_source", 12887 .dev_groups = pmu_dev_groups, 12888 }; 12889 12890 static void pmu_dev_release(struct device *dev) 12891 { 12892 kfree(dev); 12893 } 12894 12895 static int pmu_dev_alloc(struct pmu *pmu) 12896 { 12897 int ret = -ENOMEM; 12898 12899 pmu->dev = kzalloc_obj(struct device); 12900 if (!pmu->dev) 12901 goto out; 12902 12903 pmu->dev->groups = pmu->attr_groups; 12904 device_initialize(pmu->dev); 12905 12906 dev_set_drvdata(pmu->dev, pmu); 12907 pmu->dev->bus = &pmu_bus; 12908 pmu->dev->parent = pmu->parent; 12909 pmu->dev->release = pmu_dev_release; 12910 12911 ret = dev_set_name(pmu->dev, "%s", pmu->name); 12912 if (ret) 12913 goto free_dev; 12914 12915 ret = device_add(pmu->dev); 12916 if (ret) 12917 goto free_dev; 12918 12919 if (pmu->attr_update) { 12920 ret = sysfs_update_groups(&pmu->dev->kobj, pmu->attr_update); 12921 if (ret) 12922 goto del_dev; 12923 } 12924 12925 out: 12926 return ret; 12927 12928 del_dev: 12929 device_del(pmu->dev); 12930 12931 free_dev: 12932 put_device(pmu->dev); 12933 pmu->dev = NULL; 12934 goto out; 12935 } 12936 12937 static struct lock_class_key cpuctx_mutex; 12938 static struct lock_class_key cpuctx_lock; 12939 12940 static bool idr_cmpxchg(struct idr *idr, unsigned long id, void *old, void *new) 12941 { 12942 void *tmp, *val = idr_find(idr, id); 12943 12944 if (val != old) 12945 return false; 12946 12947 tmp = idr_replace(idr, new, id); 12948 if (IS_ERR(tmp)) 12949 return false; 12950 12951 WARN_ON_ONCE(tmp != val); 12952 return true; 12953 } 12954 12955 static void perf_pmu_free(struct pmu *pmu) 12956 { 12957 if (pmu_bus_running && pmu->dev && pmu->dev != PMU_NULL_DEV) { 12958 if (pmu->nr_addr_filters) 12959 device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); 12960 device_del(pmu->dev); 12961 put_device(pmu->dev); 12962 } 12963 12964 if (pmu->cpu_pmu_context) { 12965 int cpu; 12966 12967 for_each_possible_cpu(cpu) { 12968 struct perf_cpu_pmu_context *cpc; 12969 12970 cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu); 12971 if (!cpc) 12972 continue; 12973 if (cpc->epc.embedded) { 12974 /* refcount managed */ 12975 put_pmu_ctx(&cpc->epc); 12976 continue; 12977 } 12978 kfree(cpc); 12979 } 12980 free_percpu(pmu->cpu_pmu_context); 12981 } 12982 } 12983 12984 DEFINE_FREE(pmu_unregister, struct pmu *, if (_T) perf_pmu_free(_T)) 12985 12986 int perf_pmu_register(struct pmu *_pmu, const char *name, int type) 12987 { 12988 int cpu, max = PERF_TYPE_MAX; 12989 12990 struct pmu *pmu __free(pmu_unregister) = _pmu; 12991 guard(mutex)(&pmus_lock); 12992 12993 if (WARN_ONCE(!name, "Can not register anonymous pmu.\n")) 12994 return -EINVAL; 12995 12996 if (WARN_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE, 12997 "Can not register a pmu with an invalid scope.\n")) 12998 return -EINVAL; 12999 13000 pmu->name = name; 13001 13002 if (type >= 0) 13003 max = type; 13004 13005 CLASS(idr_alloc, pmu_type)(&pmu_idr, NULL, max, 0, GFP_KERNEL); 13006 if (pmu_type.id < 0) 13007 return pmu_type.id; 13008 13009 WARN_ON(type >= 0 && pmu_type.id != type); 13010 13011 pmu->type = pmu_type.id; 13012 atomic_set(&pmu->exclusive_cnt, 0); 13013 13014 if (pmu_bus_running && !pmu->dev) { 13015 int ret = pmu_dev_alloc(pmu); 13016 if (ret) 13017 return ret; 13018 } 13019 13020 pmu->cpu_pmu_context = alloc_percpu(struct perf_cpu_pmu_context *); 13021 if (!pmu->cpu_pmu_context) 13022 return -ENOMEM; 13023 13024 for_each_possible_cpu(cpu) { 13025 struct perf_cpu_pmu_context *cpc = 13026 kmalloc_node(sizeof(struct perf_cpu_pmu_context), 13027 GFP_KERNEL | __GFP_ZERO, 13028 cpu_to_node(cpu)); 13029 13030 if (!cpc) 13031 return -ENOMEM; 13032 13033 *per_cpu_ptr(pmu->cpu_pmu_context, cpu) = cpc; 13034 __perf_init_event_pmu_context(&cpc->epc, pmu); 13035 __perf_mux_hrtimer_init(cpc, cpu); 13036 } 13037 13038 if (!pmu->start_txn) { 13039 if (pmu->pmu_enable) { 13040 /* 13041 * If we have pmu_enable/pmu_disable calls, install 13042 * transaction stubs that use that to try and batch 13043 * hardware accesses. 13044 */ 13045 pmu->start_txn = perf_pmu_start_txn; 13046 pmu->commit_txn = perf_pmu_commit_txn; 13047 pmu->cancel_txn = perf_pmu_cancel_txn; 13048 } else { 13049 pmu->start_txn = perf_pmu_nop_txn; 13050 pmu->commit_txn = perf_pmu_nop_int; 13051 pmu->cancel_txn = perf_pmu_nop_void; 13052 } 13053 } 13054 13055 if (!pmu->pmu_enable) { 13056 pmu->pmu_enable = perf_pmu_nop_void; 13057 pmu->pmu_disable = perf_pmu_nop_void; 13058 } 13059 13060 if (!pmu->check_period) 13061 pmu->check_period = perf_event_nop_int; 13062 13063 if (!pmu->event_idx) 13064 pmu->event_idx = perf_event_idx_default; 13065 13066 INIT_LIST_HEAD(&pmu->events); 13067 spin_lock_init(&pmu->events_lock); 13068 13069 /* 13070 * Now that the PMU is complete, make it visible to perf_try_init_event(). 13071 */ 13072 if (!idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu)) 13073 return -EINVAL; 13074 list_add_rcu(&pmu->entry, &pmus); 13075 13076 take_idr_id(pmu_type); 13077 _pmu = no_free_ptr(pmu); // let it rip 13078 return 0; 13079 } 13080 EXPORT_SYMBOL_GPL(perf_pmu_register); 13081 13082 static void __pmu_detach_event(struct pmu *pmu, struct perf_event *event, 13083 struct perf_event_context *ctx) 13084 { 13085 /* 13086 * De-schedule the event and mark it REVOKED. 13087 */ 13088 perf_event_exit_event(event, ctx, ctx->task, DETACH_REVOKE); 13089 13090 /* 13091 * All _free_event() bits that rely on event->pmu: 13092 * 13093 * Notably, perf_mmap() relies on the ordering here. 13094 */ 13095 scoped_guard (mutex, &event->mmap_mutex) { 13096 WARN_ON_ONCE(pmu->event_unmapped); 13097 /* 13098 * Mostly an empty lock sequence, such that perf_mmap(), which 13099 * relies on mmap_mutex, is sure to observe the state change. 13100 */ 13101 } 13102 13103 perf_event_free_bpf_prog(event); 13104 perf_free_addr_filters(event); 13105 13106 if (event->destroy) { 13107 event->destroy(event); 13108 event->destroy = NULL; 13109 } 13110 13111 if (event->pmu_ctx) { 13112 put_pmu_ctx(event->pmu_ctx); 13113 event->pmu_ctx = NULL; 13114 } 13115 13116 exclusive_event_destroy(event); 13117 module_put(pmu->module); 13118 13119 mediated_pmu_unaccount_event(event); 13120 event->pmu = NULL; /* force fault instead of UAF */ 13121 } 13122 13123 static void pmu_detach_event(struct pmu *pmu, struct perf_event *event) 13124 { 13125 struct perf_event_context *ctx; 13126 13127 ctx = perf_event_ctx_lock(event); 13128 __pmu_detach_event(pmu, event, ctx); 13129 perf_event_ctx_unlock(event, ctx); 13130 13131 scoped_guard (spinlock, &pmu->events_lock) 13132 list_del(&event->pmu_list); 13133 } 13134 13135 static struct perf_event *pmu_get_event(struct pmu *pmu) 13136 { 13137 struct perf_event *event; 13138 13139 guard(spinlock)(&pmu->events_lock); 13140 list_for_each_entry(event, &pmu->events, pmu_list) { 13141 if (atomic_long_inc_not_zero(&event->refcount)) 13142 return event; 13143 } 13144 13145 return NULL; 13146 } 13147 13148 static bool pmu_empty(struct pmu *pmu) 13149 { 13150 guard(spinlock)(&pmu->events_lock); 13151 return list_empty(&pmu->events); 13152 } 13153 13154 static void pmu_detach_events(struct pmu *pmu) 13155 { 13156 struct perf_event *event; 13157 13158 for (;;) { 13159 event = pmu_get_event(pmu); 13160 if (!event) 13161 break; 13162 13163 pmu_detach_event(pmu, event); 13164 put_event(event); 13165 } 13166 13167 /* 13168 * wait for pending _free_event()s 13169 */ 13170 wait_var_event(pmu, pmu_empty(pmu)); 13171 } 13172 13173 int perf_pmu_unregister(struct pmu *pmu) 13174 { 13175 scoped_guard (mutex, &pmus_lock) { 13176 if (!idr_cmpxchg(&pmu_idr, pmu->type, pmu, NULL)) 13177 return -EINVAL; 13178 13179 list_del_rcu(&pmu->entry); 13180 } 13181 13182 /* 13183 * We dereference the pmu list under both SRCU and regular RCU, so 13184 * synchronize against both of those. 13185 * 13186 * Notably, the entirety of event creation, from perf_init_event() 13187 * (which will now fail, because of the above) until 13188 * perf_install_in_context() should be under SRCU such that 13189 * this synchronizes against event creation. This avoids trying to 13190 * detach events that are not fully formed. 13191 */ 13192 synchronize_srcu(&pmus_srcu); 13193 synchronize_rcu(); 13194 13195 if (pmu->event_unmapped && !pmu_empty(pmu)) { 13196 /* 13197 * Can't force remove events when pmu::event_unmapped() 13198 * is used in perf_mmap_close(). 13199 */ 13200 guard(mutex)(&pmus_lock); 13201 idr_cmpxchg(&pmu_idr, pmu->type, NULL, pmu); 13202 list_add_rcu(&pmu->entry, &pmus); 13203 return -EBUSY; 13204 } 13205 13206 scoped_guard (mutex, &pmus_lock) 13207 idr_remove(&pmu_idr, pmu->type); 13208 13209 /* 13210 * PMU is removed from the pmus list, so no new events will 13211 * be created, now take care of the existing ones. 13212 */ 13213 pmu_detach_events(pmu); 13214 13215 /* 13216 * PMU is unused, make it go away. 13217 */ 13218 perf_pmu_free(pmu); 13219 return 0; 13220 } 13221 EXPORT_SYMBOL_GPL(perf_pmu_unregister); 13222 13223 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) 13224 { 13225 struct perf_event_context *ctx = NULL; 13226 int ret; 13227 13228 if (!try_module_get(pmu->module)) 13229 return -ENODEV; 13230 13231 /* 13232 * A number of pmu->event_init() methods iterate the sibling_list to, 13233 * for example, validate if the group fits on the PMU. Therefore, 13234 * if this is a sibling event, acquire the ctx->mutex to protect 13235 * the sibling_list. 13236 */ 13237 if (event->group_leader != event && pmu->task_ctx_nr != perf_sw_context) { 13238 /* 13239 * This ctx->mutex can nest when we're called through 13240 * inheritance. See the perf_event_ctx_lock_nested() comment. 13241 */ 13242 ctx = perf_event_ctx_lock_nested(event->group_leader, 13243 SINGLE_DEPTH_NESTING); 13244 BUG_ON(!ctx); 13245 } 13246 13247 event->pmu = pmu; 13248 ret = pmu->event_init(event); 13249 13250 if (ctx) 13251 perf_event_ctx_unlock(event->group_leader, ctx); 13252 13253 if (ret) 13254 goto err_pmu; 13255 13256 if (!(pmu->capabilities & PERF_PMU_CAP_SIMD_REGS) && 13257 event_has_simd_regs(event)) { 13258 ret = -EOPNOTSUPP; 13259 goto err_destroy; 13260 } 13261 13262 if (!(pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS) && 13263 event_has_extended_regs(event)) { 13264 ret = -EOPNOTSUPP; 13265 goto err_destroy; 13266 } 13267 13268 if (pmu->capabilities & PERF_PMU_CAP_NO_EXCLUDE && 13269 event_has_any_exclude_flag(event)) { 13270 ret = -EINVAL; 13271 goto err_destroy; 13272 } 13273 13274 if (pmu->scope != PERF_PMU_SCOPE_NONE && event->cpu >= 0) { 13275 const struct cpumask *cpumask; 13276 struct cpumask *pmu_cpumask; 13277 int cpu; 13278 13279 cpumask = perf_scope_cpu_topology_cpumask(pmu->scope, event->cpu); 13280 pmu_cpumask = perf_scope_cpumask(pmu->scope); 13281 13282 ret = -ENODEV; 13283 if (!pmu_cpumask || !cpumask) 13284 goto err_destroy; 13285 13286 cpu = cpumask_any_and(pmu_cpumask, cpumask); 13287 if (cpu >= nr_cpu_ids) 13288 goto err_destroy; 13289 13290 event->event_caps |= PERF_EV_CAP_READ_SCOPE; 13291 } 13292 13293 return 0; 13294 13295 err_destroy: 13296 if (event->destroy) { 13297 event->destroy(event); 13298 event->destroy = NULL; 13299 } 13300 13301 err_pmu: 13302 event->pmu = NULL; 13303 module_put(pmu->module); 13304 return ret; 13305 } 13306 13307 static struct pmu *perf_init_event(struct perf_event *event) 13308 { 13309 bool extended_type = false; 13310 struct pmu *pmu; 13311 int type, ret; 13312 13313 guard(srcu)(&pmus_srcu); /* pmu idr/list access */ 13314 13315 /* 13316 * Save original type before calling pmu->event_init() since certain 13317 * pmus overwrites event->attr.type to forward event to another pmu. 13318 */ 13319 event->orig_type = event->attr.type; 13320 13321 /* Try parent's PMU first: */ 13322 if (event->parent && event->parent->pmu) { 13323 pmu = event->parent->pmu; 13324 ret = perf_try_init_event(pmu, event); 13325 if (!ret) 13326 return pmu; 13327 } 13328 13329 /* 13330 * PERF_TYPE_HARDWARE and PERF_TYPE_HW_CACHE 13331 * are often aliases for PERF_TYPE_RAW. 13332 */ 13333 type = event->attr.type; 13334 if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_HW_CACHE) { 13335 type = event->attr.config >> PERF_PMU_TYPE_SHIFT; 13336 if (!type) { 13337 type = PERF_TYPE_RAW; 13338 } else { 13339 extended_type = true; 13340 event->attr.config &= PERF_HW_EVENT_MASK; 13341 } 13342 } 13343 13344 again: 13345 scoped_guard (rcu) 13346 pmu = idr_find(&pmu_idr, type); 13347 if (pmu) { 13348 if (event->attr.type != type && type != PERF_TYPE_RAW && 13349 !(pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE)) 13350 return ERR_PTR(-ENOENT); 13351 13352 ret = perf_try_init_event(pmu, event); 13353 if (ret == -ENOENT && event->attr.type != type && !extended_type) { 13354 type = event->attr.type; 13355 goto again; 13356 } 13357 13358 if (ret) 13359 return ERR_PTR(ret); 13360 13361 return pmu; 13362 } 13363 13364 list_for_each_entry_rcu(pmu, &pmus, entry, lockdep_is_held(&pmus_srcu)) { 13365 ret = perf_try_init_event(pmu, event); 13366 if (!ret) 13367 return pmu; 13368 13369 if (ret != -ENOENT) 13370 return ERR_PTR(ret); 13371 } 13372 13373 return ERR_PTR(-ENOENT); 13374 } 13375 13376 static void attach_sb_event(struct perf_event *event) 13377 { 13378 struct pmu_event_list *pel = per_cpu_ptr(&pmu_sb_events, event->cpu); 13379 13380 raw_spin_lock(&pel->lock); 13381 list_add_rcu(&event->sb_list, &pel->list); 13382 raw_spin_unlock(&pel->lock); 13383 } 13384 13385 /* 13386 * We keep a list of all !task (and therefore per-cpu) events 13387 * that need to receive side-band records. 13388 * 13389 * This avoids having to scan all the various PMU per-cpu contexts 13390 * looking for them. 13391 */ 13392 static void account_pmu_sb_event(struct perf_event *event) 13393 { 13394 if (is_sb_event(event)) 13395 attach_sb_event(event); 13396 } 13397 13398 /* Freq events need the tick to stay alive (see perf_event_task_tick). */ 13399 static void account_freq_event_nohz(void) 13400 { 13401 #ifdef CONFIG_NO_HZ_FULL 13402 /* Lock so we don't race with concurrent unaccount */ 13403 spin_lock(&nr_freq_lock); 13404 if (atomic_inc_return(&nr_freq_events) == 1) 13405 tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS); 13406 spin_unlock(&nr_freq_lock); 13407 #endif 13408 } 13409 13410 static void account_freq_event(void) 13411 { 13412 if (tick_nohz_full_enabled()) 13413 account_freq_event_nohz(); 13414 else 13415 atomic_inc(&nr_freq_events); 13416 } 13417 13418 13419 static void account_event(struct perf_event *event) 13420 { 13421 bool inc = false; 13422 13423 if (event->parent) 13424 return; 13425 13426 if (event->attach_state & (PERF_ATTACH_TASK | PERF_ATTACH_SCHED_CB)) 13427 inc = true; 13428 if (event->attr.mmap || event->attr.mmap_data) 13429 atomic_inc(&nr_mmap_events); 13430 if (event->attr.build_id) 13431 atomic_inc(&nr_build_id_events); 13432 if (event->attr.comm) 13433 atomic_inc(&nr_comm_events); 13434 if (event->attr.namespaces) 13435 atomic_inc(&nr_namespaces_events); 13436 if (event->attr.cgroup) 13437 atomic_inc(&nr_cgroup_events); 13438 if (event->attr.task) 13439 atomic_inc(&nr_task_events); 13440 if (event->attr.freq) 13441 account_freq_event(); 13442 if (event->attr.context_switch) { 13443 atomic_inc(&nr_switch_events); 13444 inc = true; 13445 } 13446 if (has_branch_stack(event)) 13447 inc = true; 13448 if (is_cgroup_event(event)) 13449 inc = true; 13450 if (event->attr.ksymbol) 13451 atomic_inc(&nr_ksymbol_events); 13452 if (event->attr.bpf_event) 13453 atomic_inc(&nr_bpf_events); 13454 if (event->attr.text_poke) 13455 atomic_inc(&nr_text_poke_events); 13456 13457 if (inc) { 13458 /* 13459 * We need the mutex here because static_branch_enable() 13460 * must complete *before* the perf_sched_count increment 13461 * becomes visible. 13462 */ 13463 if (atomic_inc_not_zero(&perf_sched_count)) 13464 goto enabled; 13465 13466 mutex_lock(&perf_sched_mutex); 13467 if (!atomic_read(&perf_sched_count)) { 13468 static_branch_enable(&perf_sched_events); 13469 /* 13470 * Guarantee that all CPUs observe they key change and 13471 * call the perf scheduling hooks before proceeding to 13472 * install events that need them. 13473 */ 13474 synchronize_rcu(); 13475 } 13476 /* 13477 * Now that we have waited for the sync_sched(), allow further 13478 * increments to by-pass the mutex. 13479 */ 13480 atomic_inc(&perf_sched_count); 13481 mutex_unlock(&perf_sched_mutex); 13482 } 13483 enabled: 13484 13485 account_pmu_sb_event(event); 13486 } 13487 13488 /* 13489 * Allocate and initialize an event structure 13490 */ 13491 static struct perf_event * 13492 perf_event_alloc(struct perf_event_attr *attr, int cpu, 13493 struct task_struct *task, 13494 struct perf_event *group_leader, 13495 struct perf_event *parent_event, 13496 perf_overflow_handler_t overflow_handler, 13497 void *context, int cgroup_fd) 13498 { 13499 struct pmu *pmu; 13500 struct hw_perf_event *hwc; 13501 long err = -EINVAL; 13502 int node; 13503 13504 if ((unsigned)cpu >= nr_cpu_ids) { 13505 if (!task || cpu != -1) 13506 return ERR_PTR(-EINVAL); 13507 } 13508 if (attr->sigtrap && !task) { 13509 /* Requires a task: avoid signalling random tasks. */ 13510 return ERR_PTR(-EINVAL); 13511 } 13512 13513 node = (cpu >= 0) ? cpu_to_node(cpu) : -1; 13514 struct perf_event *event __free(__free_event) = 13515 kmem_cache_alloc_node(perf_event_cache, GFP_KERNEL | __GFP_ZERO, node); 13516 if (!event) 13517 return ERR_PTR(-ENOMEM); 13518 13519 /* 13520 * Single events are their own group leaders, with an 13521 * empty sibling list: 13522 */ 13523 if (!group_leader) 13524 group_leader = event; 13525 13526 mutex_init(&event->child_mutex); 13527 INIT_LIST_HEAD(&event->child_list); 13528 13529 INIT_LIST_HEAD(&event->event_entry); 13530 INIT_LIST_HEAD(&event->sibling_list); 13531 INIT_LIST_HEAD(&event->active_list); 13532 init_event_group(event); 13533 INIT_LIST_HEAD(&event->rb_entry); 13534 INIT_LIST_HEAD(&event->active_entry); 13535 INIT_LIST_HEAD(&event->addr_filters.list); 13536 INIT_HLIST_NODE(&event->hlist_entry); 13537 INIT_LIST_HEAD(&event->pmu_list); 13538 13539 13540 init_waitqueue_head(&event->waitq); 13541 init_irq_work(&event->pending_irq, perf_pending_irq); 13542 event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable); 13543 init_task_work(&event->pending_task, perf_pending_task); 13544 13545 mutex_init(&event->mmap_mutex); 13546 raw_spin_lock_init(&event->addr_filters.lock); 13547 13548 atomic_long_set(&event->refcount, 1); 13549 event->cpu = cpu; 13550 event->attr = *attr; 13551 event->group_leader = group_leader; 13552 event->pmu = NULL; 13553 event->oncpu = -1; 13554 13555 event->parent = parent_event; 13556 13557 event->ns = get_pid_ns(task_active_pid_ns(current)); 13558 event->id = atomic64_inc_return(&perf_event_id); 13559 13560 event->state = PERF_EVENT_STATE_INACTIVE; 13561 13562 if (parent_event) 13563 event->event_caps = parent_event->event_caps; 13564 13565 if (task) { 13566 event->attach_state = PERF_ATTACH_TASK; 13567 /* 13568 * XXX pmu::event_init needs to know what task to account to 13569 * and we cannot use the ctx information because we need the 13570 * pmu before we get a ctx. 13571 */ 13572 event->hw.target = get_task_struct(task); 13573 } 13574 13575 event->clock = &local_clock; 13576 if (parent_event) 13577 event->clock = parent_event->clock; 13578 13579 if (!overflow_handler && parent_event) { 13580 overflow_handler = parent_event->overflow_handler; 13581 context = parent_event->overflow_handler_context; 13582 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_EVENT_TRACING) 13583 if (parent_event->prog) { 13584 struct bpf_prog *prog = parent_event->prog; 13585 13586 bpf_prog_inc(prog); 13587 event->prog = prog; 13588 } 13589 #endif 13590 } 13591 13592 if (overflow_handler) { 13593 event->overflow_handler = overflow_handler; 13594 event->overflow_handler_context = context; 13595 } else if (is_write_backward(event)){ 13596 event->overflow_handler = perf_event_output_backward; 13597 event->overflow_handler_context = NULL; 13598 } else { 13599 event->overflow_handler = perf_event_output_forward; 13600 event->overflow_handler_context = NULL; 13601 } 13602 13603 perf_event__state_init(event); 13604 13605 pmu = NULL; 13606 13607 hwc = &event->hw; 13608 hwc->sample_period = attr->sample_period; 13609 if (is_event_in_freq_mode(event)) 13610 hwc->sample_period = 1; 13611 hwc->last_period = hwc->sample_period; 13612 13613 local64_set(&hwc->period_left, hwc->sample_period); 13614 13615 /* 13616 * We do not support PERF_SAMPLE_READ on inherited events unless 13617 * PERF_SAMPLE_TID is also selected, which allows inherited events to 13618 * collect per-thread samples. 13619 * See perf_output_read(). 13620 */ 13621 if (has_inherit_and_sample_read(attr) && !(attr->sample_type & PERF_SAMPLE_TID)) 13622 return ERR_PTR(-EINVAL); 13623 13624 if (!has_branch_stack(event)) 13625 event->attr.branch_sample_type = 0; 13626 13627 pmu = perf_init_event(event); 13628 if (IS_ERR(pmu)) 13629 return (void*)pmu; 13630 13631 /* 13632 * The PERF_ATTACH_TASK_DATA is set in the event_init()->hw_config(). 13633 * The attach should be right after the perf_init_event(). 13634 * Otherwise, the __free_event() would mistakenly detach the non-exist 13635 * perf_ctx_data because of the other errors between them. 13636 */ 13637 if (event->attach_state & PERF_ATTACH_TASK_DATA) { 13638 err = attach_perf_ctx_data(event); 13639 if (err) 13640 return ERR_PTR(err); 13641 } 13642 13643 /* 13644 * Disallow uncore-task events. Similarly, disallow uncore-cgroup 13645 * events (they don't make sense as the cgroup will be different 13646 * on other CPUs in the uncore mask). 13647 */ 13648 if (pmu->task_ctx_nr == perf_invalid_context && (task || cgroup_fd != -1)) 13649 return ERR_PTR(-EINVAL); 13650 13651 if (event->attr.aux_output && 13652 (!(pmu->capabilities & PERF_PMU_CAP_AUX_OUTPUT) || 13653 event->attr.aux_pause || event->attr.aux_resume)) 13654 return ERR_PTR(-EOPNOTSUPP); 13655 13656 if (event->attr.aux_pause && event->attr.aux_resume) 13657 return ERR_PTR(-EINVAL); 13658 13659 if (event->attr.aux_start_paused) { 13660 if (!(pmu->capabilities & PERF_PMU_CAP_AUX_PAUSE)) 13661 return ERR_PTR(-EOPNOTSUPP); 13662 event->hw.aux_paused = 1; 13663 } 13664 13665 if (cgroup_fd != -1) { 13666 err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader); 13667 if (err) 13668 return ERR_PTR(err); 13669 } 13670 13671 err = exclusive_event_init(event); 13672 if (err) 13673 return ERR_PTR(err); 13674 13675 if (has_addr_filter(event)) { 13676 event->addr_filter_ranges = kzalloc_objs(struct perf_addr_filter_range, 13677 pmu->nr_addr_filters); 13678 if (!event->addr_filter_ranges) 13679 return ERR_PTR(-ENOMEM); 13680 13681 /* 13682 * Clone the parent's vma offsets: they are valid until exec() 13683 * even if the mm is not shared with the parent. 13684 */ 13685 if (event->parent) { 13686 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 13687 13688 raw_spin_lock_irq(&ifh->lock); 13689 memcpy(event->addr_filter_ranges, 13690 event->parent->addr_filter_ranges, 13691 pmu->nr_addr_filters * sizeof(struct perf_addr_filter_range)); 13692 raw_spin_unlock_irq(&ifh->lock); 13693 } 13694 13695 /* force hw sync on the address filters */ 13696 event->addr_filters_gen = 1; 13697 } 13698 13699 if (!event->parent) { 13700 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { 13701 err = get_callchain_buffers(attr->sample_max_stack); 13702 if (err) 13703 return ERR_PTR(err); 13704 event->attach_state |= PERF_ATTACH_CALLCHAIN; 13705 } 13706 } 13707 13708 err = security_perf_event_alloc(event); 13709 if (err) 13710 return ERR_PTR(err); 13711 13712 err = mediated_pmu_account_event(event); 13713 if (err) 13714 return ERR_PTR(err); 13715 13716 /* symmetric to unaccount_event() in _free_event() */ 13717 account_event(event); 13718 13719 /* 13720 * Event creation should be under SRCU, see perf_pmu_unregister(). 13721 */ 13722 lockdep_assert_held(&pmus_srcu); 13723 scoped_guard (spinlock, &pmu->events_lock) 13724 list_add(&event->pmu_list, &pmu->events); 13725 13726 return_ptr(event); 13727 } 13728 13729 static int perf_copy_attr(struct perf_event_attr __user *uattr, 13730 struct perf_event_attr *attr) 13731 { 13732 u32 size; 13733 int ret; 13734 13735 /* Zero the full structure, so that a short copy will be nice. */ 13736 memset(attr, 0, sizeof(*attr)); 13737 13738 ret = get_user(size, &uattr->size); 13739 if (ret) 13740 return ret; 13741 13742 /* ABI compatibility quirk: */ 13743 if (!size) 13744 size = PERF_ATTR_SIZE_VER0; 13745 if (size < PERF_ATTR_SIZE_VER0 || size > PAGE_SIZE) 13746 goto err_size; 13747 13748 ret = copy_struct_from_user(attr, sizeof(*attr), uattr, size); 13749 if (ret) { 13750 if (ret == -E2BIG) 13751 goto err_size; 13752 return ret; 13753 } 13754 13755 attr->size = size; 13756 13757 if (attr->__reserved_1 || attr->__reserved_2 || 13758 attr->__reserved_3 || attr->__reserved_4) 13759 return -EINVAL; 13760 13761 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) 13762 return -EINVAL; 13763 13764 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) 13765 return -EINVAL; 13766 13767 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { 13768 u64 mask = attr->branch_sample_type; 13769 13770 /* only using defined bits */ 13771 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) 13772 return -EINVAL; 13773 13774 /* at least one branch bit must be set */ 13775 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL)) 13776 return -EINVAL; 13777 13778 /* propagate priv level, when not set for branch */ 13779 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) { 13780 13781 /* exclude_kernel checked on syscall entry */ 13782 if (!attr->exclude_kernel) 13783 mask |= PERF_SAMPLE_BRANCH_KERNEL; 13784 13785 if (!attr->exclude_user) 13786 mask |= PERF_SAMPLE_BRANCH_USER; 13787 13788 if (!attr->exclude_hv) 13789 mask |= PERF_SAMPLE_BRANCH_HV; 13790 /* 13791 * adjust user setting (for HW filter setup) 13792 */ 13793 attr->branch_sample_type = mask; 13794 } 13795 /* privileged levels capture (kernel, hv): check permissions */ 13796 if (mask & PERF_SAMPLE_BRANCH_PERM_PLM) { 13797 ret = perf_allow_kernel(); 13798 if (ret) 13799 return ret; 13800 } 13801 } 13802 13803 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { 13804 ret = perf_reg_validate(attr->sample_regs_user, 13805 attr->sample_simd_regs_enabled); 13806 if (ret) 13807 return ret; 13808 if (attr->sample_simd_regs_enabled) { 13809 ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords, 13810 attr->sample_simd_vec_reg_user, 13811 attr->sample_simd_pred_reg_qwords, 13812 attr->sample_simd_pred_reg_user); 13813 if (ret) 13814 return ret; 13815 } 13816 } 13817 13818 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { 13819 if (!arch_perf_have_user_stack_dump()) 13820 return -ENOSYS; 13821 13822 /* 13823 * We have __u32 type for the size, but so far 13824 * we can only use __u16 as maximum due to the 13825 * __u16 sample size limit. 13826 */ 13827 if (attr->sample_stack_user >= USHRT_MAX) 13828 return -EINVAL; 13829 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) 13830 return -EINVAL; 13831 } 13832 13833 if (!attr->sample_max_stack) 13834 attr->sample_max_stack = sysctl_perf_event_max_stack; 13835 13836 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) { 13837 ret = perf_reg_validate(attr->sample_regs_intr, 13838 attr->sample_simd_regs_enabled); 13839 if (ret) 13840 return ret; 13841 if (attr->sample_simd_regs_enabled) { 13842 ret = perf_simd_reg_validate(attr->sample_simd_vec_reg_qwords, 13843 attr->sample_simd_vec_reg_intr, 13844 attr->sample_simd_pred_reg_qwords, 13845 attr->sample_simd_pred_reg_intr); 13846 if (ret) 13847 return ret; 13848 } 13849 } 13850 13851 #ifndef CONFIG_CGROUP_PERF 13852 if (attr->sample_type & PERF_SAMPLE_CGROUP) 13853 return -EINVAL; 13854 #endif 13855 if ((attr->sample_type & PERF_SAMPLE_WEIGHT) && 13856 (attr->sample_type & PERF_SAMPLE_WEIGHT_STRUCT)) 13857 return -EINVAL; 13858 13859 if (!attr->inherit && attr->inherit_thread) 13860 return -EINVAL; 13861 13862 if (attr->remove_on_exec && attr->enable_on_exec) 13863 return -EINVAL; 13864 13865 if (attr->sigtrap && !attr->remove_on_exec) 13866 return -EINVAL; 13867 13868 out: 13869 return ret; 13870 13871 err_size: 13872 put_user(sizeof(*attr), &uattr->size); 13873 ret = -E2BIG; 13874 goto out; 13875 } 13876 13877 static void mutex_lock_double(struct mutex *a, struct mutex *b) 13878 { 13879 if (b < a) 13880 swap(a, b); 13881 13882 mutex_lock(a); 13883 mutex_lock_nested(b, SINGLE_DEPTH_NESTING); 13884 } 13885 13886 static int 13887 perf_event_set_output(struct perf_event *event, struct perf_event *output_event) 13888 { 13889 struct perf_buffer *rb = NULL; 13890 int ret = -EINVAL; 13891 13892 if (!output_event) { 13893 mutex_lock(&event->mmap_mutex); 13894 goto set; 13895 } 13896 13897 /* don't allow circular references */ 13898 if (event == output_event) 13899 goto out; 13900 13901 /* 13902 * Don't allow cross-cpu buffers 13903 */ 13904 if (output_event->cpu != event->cpu) 13905 goto out; 13906 13907 /* 13908 * If its not a per-cpu rb, it must be the same task. 13909 */ 13910 if (output_event->cpu == -1 && output_event->hw.target != event->hw.target) 13911 goto out; 13912 13913 /* 13914 * Mixing clocks in the same buffer is trouble you don't need. 13915 */ 13916 if (output_event->clock != event->clock) 13917 goto out; 13918 13919 /* 13920 * Either writing ring buffer from beginning or from end. 13921 * Mixing is not allowed. 13922 */ 13923 if (is_write_backward(output_event) != is_write_backward(event)) 13924 goto out; 13925 13926 /* 13927 * If both events generate aux data, they must be on the same PMU 13928 */ 13929 if (has_aux(event) && has_aux(output_event) && 13930 event->pmu != output_event->pmu) 13931 goto out; 13932 13933 /* 13934 * Hold both mmap_mutex to serialize against perf_mmap_close(). Since 13935 * output_event is already on rb->event_list, and the list iteration 13936 * restarts after every removal, it is guaranteed this new event is 13937 * observed *OR* if output_event is already removed, it's guaranteed we 13938 * observe !rb->mmap_count. 13939 */ 13940 mutex_lock_double(&event->mmap_mutex, &output_event->mmap_mutex); 13941 set: 13942 /* Can't redirect output if we've got an active mmap() */ 13943 if (refcount_read(&event->mmap_count)) 13944 goto unlock; 13945 13946 if (output_event) { 13947 if (output_event->state <= PERF_EVENT_STATE_REVOKED) 13948 goto unlock; 13949 13950 /* get the rb we want to redirect to */ 13951 rb = ring_buffer_get(output_event); 13952 if (!rb) 13953 goto unlock; 13954 13955 /* did we race against perf_mmap_close() */ 13956 if (!refcount_read(&rb->mmap_count)) { 13957 ring_buffer_put(rb); 13958 goto unlock; 13959 } 13960 } 13961 13962 ring_buffer_attach(event, rb); 13963 13964 ret = 0; 13965 unlock: 13966 mutex_unlock(&event->mmap_mutex); 13967 if (output_event) 13968 mutex_unlock(&output_event->mmap_mutex); 13969 13970 out: 13971 return ret; 13972 } 13973 13974 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id) 13975 { 13976 bool nmi_safe = false; 13977 13978 switch (clk_id) { 13979 case CLOCK_MONOTONIC: 13980 event->clock = &ktime_get_mono_fast_ns; 13981 nmi_safe = true; 13982 break; 13983 13984 case CLOCK_MONOTONIC_RAW: 13985 event->clock = &ktime_get_raw_fast_ns; 13986 nmi_safe = true; 13987 break; 13988 13989 case CLOCK_REALTIME: 13990 event->clock = &ktime_get_real_ns; 13991 break; 13992 13993 case CLOCK_BOOTTIME: 13994 event->clock = &ktime_get_boottime_ns; 13995 break; 13996 13997 case CLOCK_TAI: 13998 event->clock = &ktime_get_clocktai_ns; 13999 break; 14000 14001 default: 14002 return -EINVAL; 14003 } 14004 14005 if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI)) 14006 return -EINVAL; 14007 14008 return 0; 14009 } 14010 14011 static bool 14012 perf_check_permission(struct perf_event_attr *attr, struct task_struct *task) 14013 { 14014 unsigned int ptrace_mode = PTRACE_MODE_READ_REALCREDS; 14015 bool is_capable = perfmon_capable(); 14016 14017 if (attr->sigtrap) { 14018 /* 14019 * perf_event_attr::sigtrap sends signals to the other task. 14020 * Require the current task to also have CAP_KILL. 14021 */ 14022 rcu_read_lock(); 14023 is_capable &= ns_capable(__task_cred(task)->user_ns, CAP_KILL); 14024 rcu_read_unlock(); 14025 14026 /* 14027 * If the required capabilities aren't available, checks for 14028 * ptrace permissions: upgrade to ATTACH, since sending signals 14029 * can effectively change the target task. 14030 */ 14031 ptrace_mode = PTRACE_MODE_ATTACH_REALCREDS; 14032 } 14033 14034 /* 14035 * Preserve ptrace permission check for backwards compatibility. The 14036 * ptrace check also includes checks that the current task and other 14037 * task have matching uids, and is therefore not done here explicitly. 14038 */ 14039 return is_capable || ptrace_may_access(task, ptrace_mode); 14040 } 14041 14042 /** 14043 * sys_perf_event_open - open a performance event, associate it to a task/cpu 14044 * 14045 * @attr_uptr: event_id type attributes for monitoring/sampling 14046 * @pid: target pid 14047 * @cpu: target cpu 14048 * @group_fd: group leader event fd 14049 * @flags: perf event open flags 14050 */ 14051 SYSCALL_DEFINE5(perf_event_open, 14052 struct perf_event_attr __user *, attr_uptr, 14053 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags) 14054 { 14055 struct perf_event *group_leader = NULL, *output_event = NULL; 14056 struct perf_event_pmu_context *pmu_ctx; 14057 struct perf_event *event, *sibling; 14058 struct perf_event_attr attr; 14059 struct perf_event_context *ctx; 14060 struct file *event_file = NULL; 14061 struct task_struct *task = NULL; 14062 struct pmu *pmu; 14063 int event_fd; 14064 int move_group = 0; 14065 int err; 14066 int f_flags = O_RDWR; 14067 int cgroup_fd = -1; 14068 14069 /* for future expandability... */ 14070 if (flags & ~PERF_FLAG_ALL) 14071 return -EINVAL; 14072 14073 err = perf_copy_attr(attr_uptr, &attr); 14074 if (err) 14075 return err; 14076 14077 /* Do we allow access to perf_event_open(2) ? */ 14078 err = security_perf_event_open(PERF_SECURITY_OPEN); 14079 if (err) 14080 return err; 14081 14082 if (!attr.exclude_kernel || attr.text_poke || 14083 ((attr.sample_type & PERF_SAMPLE_CALLCHAIN) && 14084 !attr.exclude_callchain_kernel)) { 14085 err = perf_allow_kernel(); 14086 if (err) 14087 return err; 14088 } 14089 14090 if (attr.namespaces) { 14091 if (!perfmon_capable()) 14092 return -EACCES; 14093 } 14094 14095 if (attr.freq) { 14096 if (attr.sample_freq > sysctl_perf_event_sample_rate) 14097 return -EINVAL; 14098 } else { 14099 if (attr.sample_period & (1ULL << 63)) 14100 return -EINVAL; 14101 } 14102 14103 /* Only privileged users can get physical addresses */ 14104 if ((attr.sample_type & PERF_SAMPLE_PHYS_ADDR)) { 14105 err = perf_allow_kernel(); 14106 if (err) 14107 return err; 14108 } 14109 14110 /* REGS_INTR can leak data, lockdown must prevent this */ 14111 if (attr.sample_type & PERF_SAMPLE_REGS_INTR) { 14112 err = security_locked_down(LOCKDOWN_PERF); 14113 if (err) 14114 return err; 14115 } 14116 14117 /* 14118 * In cgroup mode, the pid argument is used to pass the fd 14119 * opened to the cgroup directory in cgroupfs. The cpu argument 14120 * designates the cpu on which to monitor threads from that 14121 * cgroup. 14122 */ 14123 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) 14124 return -EINVAL; 14125 14126 if (flags & PERF_FLAG_FD_CLOEXEC) 14127 f_flags |= O_CLOEXEC; 14128 14129 event_fd = get_unused_fd_flags(f_flags); 14130 if (event_fd < 0) 14131 return event_fd; 14132 14133 /* 14134 * Event creation should be under SRCU, see perf_pmu_unregister(). 14135 */ 14136 guard(srcu)(&pmus_srcu); 14137 14138 CLASS(fd, group)(group_fd); // group_fd == -1 => empty 14139 if (group_fd != -1) { 14140 if (!is_perf_file(group)) { 14141 err = -EBADF; 14142 goto err_fd; 14143 } 14144 group_leader = fd_file(group)->private_data; 14145 if (group_leader->state <= PERF_EVENT_STATE_EXIT) { 14146 err = -ENODEV; 14147 goto err_fd; 14148 } 14149 if (flags & PERF_FLAG_FD_OUTPUT) 14150 output_event = group_leader; 14151 if (flags & PERF_FLAG_FD_NO_GROUP) 14152 group_leader = NULL; 14153 } 14154 14155 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { 14156 task = find_lively_task_by_vpid(pid); 14157 if (IS_ERR(task)) { 14158 err = PTR_ERR(task); 14159 goto err_fd; 14160 } 14161 } 14162 14163 if (task && group_leader && 14164 group_leader->attr.inherit != attr.inherit) { 14165 err = -EINVAL; 14166 goto err_task; 14167 } 14168 14169 if (flags & PERF_FLAG_PID_CGROUP) 14170 cgroup_fd = pid; 14171 14172 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, 14173 NULL, NULL, cgroup_fd); 14174 if (IS_ERR(event)) { 14175 err = PTR_ERR(event); 14176 goto err_task; 14177 } 14178 14179 if (is_sampling_event(event)) { 14180 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { 14181 err = -EOPNOTSUPP; 14182 goto err_alloc; 14183 } 14184 } 14185 14186 /* 14187 * Special case software events and allow them to be part of 14188 * any hardware group. 14189 */ 14190 pmu = event->pmu; 14191 14192 if (attr.use_clockid) { 14193 err = perf_event_set_clock(event, attr.clockid); 14194 if (err) 14195 goto err_alloc; 14196 } 14197 14198 if (pmu->task_ctx_nr == perf_sw_context) 14199 event->event_caps |= PERF_EV_CAP_SOFTWARE; 14200 14201 if (task) { 14202 err = down_read_interruptible(&task->signal->exec_update_lock); 14203 if (err) 14204 goto err_alloc; 14205 14206 /* 14207 * We must hold exec_update_lock across this and any potential 14208 * perf_install_in_context() call for this new event to 14209 * serialize against exec() altering our credentials (and the 14210 * perf_event_exit_task() that could imply). 14211 */ 14212 err = -EACCES; 14213 if (!perf_check_permission(&attr, task)) 14214 goto err_cred; 14215 } 14216 14217 /* 14218 * Get the target context (task or percpu): 14219 */ 14220 ctx = find_get_context(task, event); 14221 if (IS_ERR(ctx)) { 14222 err = PTR_ERR(ctx); 14223 goto err_cred; 14224 } 14225 14226 mutex_lock(&ctx->mutex); 14227 14228 if (ctx->task == TASK_TOMBSTONE) { 14229 err = -ESRCH; 14230 goto err_locked; 14231 } 14232 14233 if (!task) { 14234 /* 14235 * Check if the @cpu we're creating an event for is online. 14236 * 14237 * We use the perf_cpu_context::ctx::mutex to serialize against 14238 * the hotplug notifiers. See perf_event_{init,exit}_cpu(). 14239 */ 14240 struct perf_cpu_context *cpuctx = per_cpu_ptr(&perf_cpu_context, event->cpu); 14241 14242 if (!cpuctx->online) { 14243 err = -ENODEV; 14244 goto err_locked; 14245 } 14246 } 14247 14248 if (group_leader) { 14249 err = -EINVAL; 14250 14251 /* 14252 * Do not allow a recursive hierarchy (this new sibling 14253 * becoming part of another group-sibling): 14254 */ 14255 if (group_leader->group_leader != group_leader) 14256 goto err_locked; 14257 14258 /* All events in a group should have the same clock */ 14259 if (group_leader->clock != event->clock) 14260 goto err_locked; 14261 14262 /* 14263 * Make sure we're both events for the same CPU; 14264 * grouping events for different CPUs is broken; since 14265 * you can never concurrently schedule them anyhow. 14266 */ 14267 if (group_leader->cpu != event->cpu) 14268 goto err_locked; 14269 14270 /* 14271 * Make sure we're both on the same context; either task or cpu. 14272 */ 14273 if (group_leader->ctx != ctx) 14274 goto err_locked; 14275 14276 /* Recheck under ctx::mutex to serialize against remove-on-exec. */ 14277 if (group_leader->state <= PERF_EVENT_STATE_EXIT) { 14278 err = -ENODEV; 14279 goto err_locked; 14280 } 14281 14282 /* 14283 * Only a group leader can be exclusive or pinned 14284 */ 14285 if (attr.exclusive || attr.pinned) 14286 goto err_locked; 14287 14288 if (is_software_event(event) && 14289 !in_software_context(group_leader)) { 14290 /* 14291 * If the event is a sw event, but the group_leader 14292 * is on hw context. 14293 * 14294 * Allow the addition of software events to hw 14295 * groups, this is safe because software events 14296 * never fail to schedule. 14297 * 14298 * Note the comment that goes with struct 14299 * perf_event_pmu_context. 14300 */ 14301 pmu = group_leader->pmu_ctx->pmu; 14302 } else if (!is_software_event(event)) { 14303 if (is_software_event(group_leader) && 14304 (group_leader->group_caps & PERF_EV_CAP_SOFTWARE)) { 14305 /* 14306 * In case the group is a pure software group, and we 14307 * try to add a hardware event, move the whole group to 14308 * the hardware context. 14309 */ 14310 move_group = 1; 14311 } 14312 14313 /* Don't allow group of multiple hw events from different pmus */ 14314 if (!in_software_context(group_leader) && 14315 group_leader->pmu_ctx->pmu != pmu) 14316 goto err_locked; 14317 } 14318 } 14319 14320 /* 14321 * Now that we're certain of the pmu; find the pmu_ctx. 14322 */ 14323 pmu_ctx = find_get_pmu_context(pmu, ctx, event); 14324 if (IS_ERR(pmu_ctx)) { 14325 err = PTR_ERR(pmu_ctx); 14326 goto err_locked; 14327 } 14328 event->pmu_ctx = pmu_ctx; 14329 14330 if (output_event) { 14331 err = perf_event_set_output(event, output_event); 14332 if (err) 14333 goto err_context; 14334 } 14335 14336 if (!perf_event_validate_size(event)) { 14337 err = -E2BIG; 14338 goto err_context; 14339 } 14340 14341 if (perf_need_aux_event(event) && !perf_get_aux_event(event, group_leader)) { 14342 err = -EINVAL; 14343 goto err_context; 14344 } 14345 14346 /* 14347 * Must be under the same ctx::mutex as perf_install_in_context(), 14348 * because we need to serialize with concurrent event creation. 14349 */ 14350 if (!exclusive_event_installable(event, ctx)) { 14351 err = -EBUSY; 14352 goto err_context; 14353 } 14354 14355 WARN_ON_ONCE(ctx->parent_ctx); 14356 14357 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, f_flags); 14358 if (IS_ERR(event_file)) { 14359 err = PTR_ERR(event_file); 14360 event_file = NULL; 14361 goto err_context; 14362 } 14363 14364 /* 14365 * This is the point on no return; we cannot fail hereafter. This is 14366 * where we start modifying current state. 14367 */ 14368 14369 if (move_group) { 14370 perf_remove_from_context(group_leader, 0); 14371 put_pmu_ctx(group_leader->pmu_ctx); 14372 14373 for_each_sibling_event(sibling, group_leader) { 14374 perf_remove_from_context(sibling, 0); 14375 put_pmu_ctx(sibling->pmu_ctx); 14376 } 14377 14378 /* 14379 * Install the group siblings before the group leader. 14380 * 14381 * Because a group leader will try and install the entire group 14382 * (through the sibling list, which is still in-tact), we can 14383 * end up with siblings installed in the wrong context. 14384 * 14385 * By installing siblings first we NO-OP because they're not 14386 * reachable through the group lists. 14387 */ 14388 for_each_sibling_event(sibling, group_leader) { 14389 sibling->pmu_ctx = pmu_ctx; 14390 get_pmu_ctx(pmu_ctx); 14391 perf_event__state_init(sibling); 14392 perf_install_in_context(ctx, sibling, sibling->cpu); 14393 } 14394 14395 /* 14396 * Removing from the context ends up with disabled 14397 * event. What we want here is event in the initial 14398 * startup state, ready to be add into new context. 14399 */ 14400 group_leader->pmu_ctx = pmu_ctx; 14401 get_pmu_ctx(pmu_ctx); 14402 perf_event__state_init(group_leader); 14403 perf_install_in_context(ctx, group_leader, group_leader->cpu); 14404 } 14405 14406 /* 14407 * Precalculate sample_data sizes; do while holding ctx::mutex such 14408 * that we're serialized against further additions and before 14409 * perf_install_in_context() which is the point the event is active and 14410 * can use these values. 14411 */ 14412 perf_event__header_size(event); 14413 perf_event__id_header_size(event); 14414 14415 event->owner = current; 14416 14417 perf_install_in_context(ctx, event, event->cpu); 14418 perf_unpin_context(ctx); 14419 14420 mutex_unlock(&ctx->mutex); 14421 14422 if (task) { 14423 up_read(&task->signal->exec_update_lock); 14424 put_task_struct(task); 14425 } 14426 14427 mutex_lock(¤t->perf_event_mutex); 14428 list_add_tail(&event->owner_entry, ¤t->perf_event_list); 14429 mutex_unlock(¤t->perf_event_mutex); 14430 14431 /* 14432 * File reference in group guarantees that group_leader has been 14433 * kept alive until we place the new event on the sibling_list. 14434 * This ensures destruction of the group leader will find 14435 * the pointer to itself in perf_group_detach(). 14436 */ 14437 fd_install(event_fd, event_file); 14438 return event_fd; 14439 14440 err_context: 14441 put_pmu_ctx(event->pmu_ctx); 14442 event->pmu_ctx = NULL; /* _free_event() */ 14443 err_locked: 14444 mutex_unlock(&ctx->mutex); 14445 perf_unpin_context(ctx); 14446 put_ctx(ctx); 14447 err_cred: 14448 if (task) 14449 up_read(&task->signal->exec_update_lock); 14450 err_alloc: 14451 put_event(event); 14452 err_task: 14453 if (task) 14454 put_task_struct(task); 14455 err_fd: 14456 put_unused_fd(event_fd); 14457 return err; 14458 } 14459 14460 /** 14461 * perf_event_create_kernel_counter 14462 * 14463 * @attr: attributes of the counter to create 14464 * @cpu: cpu in which the counter is bound 14465 * @task: task to profile (NULL for percpu) 14466 * @overflow_handler: callback to trigger when we hit the event 14467 * @context: context data could be used in overflow_handler callback 14468 */ 14469 struct perf_event * 14470 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 14471 struct task_struct *task, 14472 perf_overflow_handler_t overflow_handler, 14473 void *context) 14474 { 14475 struct perf_event_pmu_context *pmu_ctx; 14476 struct perf_event_context *ctx; 14477 struct perf_event *event; 14478 struct pmu *pmu; 14479 int err; 14480 14481 /* 14482 * Grouping is not supported for kernel events, neither is 'AUX', 14483 * make sure the caller's intentions are adjusted. 14484 */ 14485 if (attr->aux_output || attr->aux_action) 14486 return ERR_PTR(-EINVAL); 14487 14488 /* 14489 * Event creation should be under SRCU, see perf_pmu_unregister(). 14490 */ 14491 guard(srcu)(&pmus_srcu); 14492 14493 event = perf_event_alloc(attr, cpu, task, NULL, NULL, 14494 overflow_handler, context, -1); 14495 if (IS_ERR(event)) { 14496 err = PTR_ERR(event); 14497 goto err; 14498 } 14499 14500 /* Mark owner so we could distinguish it from user events. */ 14501 event->owner = TASK_TOMBSTONE; 14502 pmu = event->pmu; 14503 14504 if (pmu->task_ctx_nr == perf_sw_context) 14505 event->event_caps |= PERF_EV_CAP_SOFTWARE; 14506 14507 /* 14508 * Get the target context (task or percpu): 14509 */ 14510 ctx = find_get_context(task, event); 14511 if (IS_ERR(ctx)) { 14512 err = PTR_ERR(ctx); 14513 goto err_alloc; 14514 } 14515 14516 WARN_ON_ONCE(ctx->parent_ctx); 14517 mutex_lock(&ctx->mutex); 14518 if (ctx->task == TASK_TOMBSTONE) { 14519 err = -ESRCH; 14520 goto err_unlock; 14521 } 14522 14523 pmu_ctx = find_get_pmu_context(pmu, ctx, event); 14524 if (IS_ERR(pmu_ctx)) { 14525 err = PTR_ERR(pmu_ctx); 14526 goto err_unlock; 14527 } 14528 event->pmu_ctx = pmu_ctx; 14529 14530 if (!task) { 14531 /* 14532 * Check if the @cpu we're creating an event for is online. 14533 * 14534 * We use the perf_cpu_context::ctx::mutex to serialize against 14535 * the hotplug notifiers. See perf_event_{init,exit}_cpu(). 14536 */ 14537 struct perf_cpu_context *cpuctx = 14538 container_of(ctx, struct perf_cpu_context, ctx); 14539 if (!cpuctx->online) { 14540 err = -ENODEV; 14541 goto err_pmu_ctx; 14542 } 14543 } 14544 14545 if (!exclusive_event_installable(event, ctx)) { 14546 err = -EBUSY; 14547 goto err_pmu_ctx; 14548 } 14549 14550 perf_install_in_context(ctx, event, event->cpu); 14551 perf_unpin_context(ctx); 14552 mutex_unlock(&ctx->mutex); 14553 14554 return event; 14555 14556 err_pmu_ctx: 14557 put_pmu_ctx(pmu_ctx); 14558 event->pmu_ctx = NULL; /* _free_event() */ 14559 err_unlock: 14560 mutex_unlock(&ctx->mutex); 14561 perf_unpin_context(ctx); 14562 put_ctx(ctx); 14563 err_alloc: 14564 put_event(event); 14565 err: 14566 return ERR_PTR(err); 14567 } 14568 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); 14569 14570 static void __perf_pmu_remove(struct perf_event_context *ctx, struct pmu *pmu, 14571 struct list_head *events) 14572 { 14573 struct perf_event *event, *tmp; 14574 14575 list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) { 14576 if (event->pmu_ctx->pmu != pmu) 14577 continue; 14578 14579 perf_remove_from_context(event, 0); 14580 put_pmu_ctx(event->pmu_ctx); 14581 list_add(&event->migrate_entry, events); 14582 } 14583 } 14584 14585 static void __perf_pmu_install_event(struct pmu *pmu, 14586 struct perf_event_context *ctx, 14587 int cpu, struct perf_event *event) 14588 { 14589 struct perf_event_pmu_context *epc; 14590 struct perf_event_context *old_ctx = event->ctx; 14591 14592 get_ctx(ctx); /* normally find_get_context() */ 14593 14594 event->cpu = cpu; 14595 epc = find_get_pmu_context(pmu, ctx, event); 14596 event->pmu_ctx = epc; 14597 14598 if (event->state >= PERF_EVENT_STATE_OFF) 14599 event->state = PERF_EVENT_STATE_INACTIVE; 14600 perf_install_in_context(ctx, event, cpu); 14601 14602 /* 14603 * Now that event->ctx is updated and visible, put the old ctx. 14604 */ 14605 put_ctx(old_ctx); 14606 } 14607 14608 static void __perf_pmu_install(struct perf_event_context *ctx, 14609 int cpu, struct pmu *pmu, struct list_head *events) 14610 { 14611 struct perf_event *event, *tmp; 14612 14613 /* 14614 * Re-instate events in 2 passes. 14615 * 14616 * Skip over group leaders and only install siblings on this first 14617 * pass, siblings will not get enabled without a leader, however a 14618 * leader will enable its siblings, even if those are still on the old 14619 * context. 14620 */ 14621 list_for_each_entry_safe(event, tmp, events, migrate_entry) { 14622 if (event->group_leader == event) 14623 continue; 14624 14625 list_del(&event->migrate_entry); 14626 __perf_pmu_install_event(pmu, ctx, cpu, event); 14627 } 14628 14629 /* 14630 * Once all the siblings are setup properly, install the group leaders 14631 * to make it go. 14632 */ 14633 list_for_each_entry_safe(event, tmp, events, migrate_entry) { 14634 list_del(&event->migrate_entry); 14635 __perf_pmu_install_event(pmu, ctx, cpu, event); 14636 } 14637 } 14638 14639 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) 14640 { 14641 struct perf_event_context *src_ctx, *dst_ctx; 14642 LIST_HEAD(events); 14643 14644 /* 14645 * Since per-cpu context is persistent, no need to grab an extra 14646 * reference. 14647 */ 14648 src_ctx = &per_cpu_ptr(&perf_cpu_context, src_cpu)->ctx; 14649 dst_ctx = &per_cpu_ptr(&perf_cpu_context, dst_cpu)->ctx; 14650 14651 /* 14652 * See perf_event_ctx_lock() for comments on the details 14653 * of swizzling perf_event::ctx. 14654 */ 14655 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); 14656 14657 __perf_pmu_remove(src_ctx, pmu, &events); 14658 14659 if (!list_empty(&events)) { 14660 /* 14661 * Wait for the events to quiesce before re-instating them. 14662 */ 14663 synchronize_rcu(); 14664 14665 __perf_pmu_install(dst_ctx, dst_cpu, pmu, &events); 14666 } 14667 14668 mutex_unlock(&dst_ctx->mutex); 14669 mutex_unlock(&src_ctx->mutex); 14670 } 14671 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); 14672 14673 static void sync_child_event(struct perf_event *child_event, 14674 struct task_struct *task) 14675 { 14676 struct perf_event *parent_event = child_event->parent; 14677 u64 child_val; 14678 14679 if (child_event->attr.inherit_stat) { 14680 if (task && task != TASK_TOMBSTONE) 14681 perf_event_read_event(child_event, task); 14682 } 14683 14684 child_val = perf_event_count(child_event, false); 14685 14686 /* 14687 * Add back the child's count to the parent's count: 14688 */ 14689 atomic64_add(child_val, &parent_event->child_count); 14690 atomic64_add(child_event->total_time_enabled, 14691 &parent_event->child_total_time_enabled); 14692 atomic64_add(child_event->total_time_running, 14693 &parent_event->child_total_time_running); 14694 } 14695 14696 static void 14697 perf_event_exit_event(struct perf_event *event, 14698 struct perf_event_context *ctx, 14699 struct task_struct *task, 14700 unsigned long detach_flags) 14701 { 14702 struct perf_event *parent_event = event->parent; 14703 unsigned int attach_state; 14704 14705 detach_flags |= DETACH_EXIT; 14706 14707 if (parent_event) { 14708 /* 14709 * Do not destroy the 'original' grouping; because of the 14710 * context switch optimization the original events could've 14711 * ended up in a random child task. 14712 * 14713 * If we were to destroy the original group, all group related 14714 * operations would cease to function properly after this 14715 * random child dies. 14716 * 14717 * Do destroy all inherited groups, we don't care about those 14718 * and being thorough is better. 14719 */ 14720 detach_flags |= DETACH_GROUP | DETACH_CHILD; 14721 mutex_lock(&parent_event->child_mutex); 14722 /* PERF_ATTACH_ITRACE might be set concurrently */ 14723 attach_state = READ_ONCE(event->attach_state); 14724 14725 if (attach_state & PERF_ATTACH_CHILD) 14726 sync_child_event(event, task); 14727 } 14728 14729 if (detach_flags & DETACH_REVOKE) 14730 detach_flags |= DETACH_GROUP; 14731 14732 perf_remove_from_context(event, detach_flags); 14733 /* 14734 * Child events can be freed. 14735 */ 14736 if (parent_event) { 14737 mutex_unlock(&parent_event->child_mutex); 14738 14739 /* 14740 * Match the refcount initialization. Make sure it doesn't happen 14741 * twice if pmu_detach_event() calls it on an already exited task. 14742 */ 14743 if (attach_state & PERF_ATTACH_CHILD) { 14744 /* 14745 * Kick perf_poll() for is_event_hup(); 14746 */ 14747 perf_event_wakeup(parent_event); 14748 /* 14749 * pmu_detach_event() will have an extra refcount. 14750 * perf_pending_task() might have one too. 14751 */ 14752 put_event(event); 14753 } 14754 14755 return; 14756 } 14757 14758 /* 14759 * Parent events are governed by their filedesc, retain them. 14760 */ 14761 perf_event_wakeup(event); 14762 } 14763 14764 static void perf_event_exit_task_context(struct task_struct *task, bool exit) 14765 { 14766 struct perf_event_context *ctx, *clone_ctx = NULL; 14767 struct perf_event *child_event, *next; 14768 14769 ctx = perf_pin_task_context(task); 14770 if (!ctx) 14771 return; 14772 14773 /* 14774 * In order to reduce the amount of tricky in ctx tear-down, we hold 14775 * ctx::mutex over the entire thing. This serializes against almost 14776 * everything that wants to access the ctx. 14777 * 14778 * The exception is sys_perf_event_open() / 14779 * perf_event_create_kernel_count() which does find_get_context() 14780 * without ctx::mutex (it cannot because of the move_group double mutex 14781 * lock thing). See the comments in perf_install_in_context(). 14782 */ 14783 mutex_lock(&ctx->mutex); 14784 14785 /* 14786 * In a single ctx::lock section, de-schedule the events and detach the 14787 * context from the task such that we cannot ever get it scheduled back 14788 * in. 14789 */ 14790 raw_spin_lock_irq(&ctx->lock); 14791 if (exit) 14792 task_ctx_sched_out(ctx, NULL, EVENT_ALL); 14793 14794 /* 14795 * Now that the context is inactive, destroy the task <-> ctx relation 14796 * and mark the context dead. 14797 */ 14798 RCU_INIT_POINTER(task->perf_event_ctxp, NULL); 14799 put_ctx(ctx); /* cannot be last */ 14800 WRITE_ONCE(ctx->task, TASK_TOMBSTONE); 14801 put_task_struct(task); /* cannot be last */ 14802 14803 clone_ctx = unclone_ctx(ctx); 14804 raw_spin_unlock_irq(&ctx->lock); 14805 14806 if (clone_ctx) 14807 put_ctx(clone_ctx); 14808 14809 /* 14810 * Report the task dead after unscheduling the events so that we 14811 * won't get any samples after PERF_RECORD_EXIT. We can however still 14812 * get a few PERF_RECORD_READ events. 14813 */ 14814 if (exit) 14815 perf_event_task(task, ctx, 0); 14816 14817 list_for_each_entry_safe(child_event, next, &ctx->event_list, event_entry) 14818 perf_event_exit_event(child_event, ctx, exit ? task : NULL, 0); 14819 14820 mutex_unlock(&ctx->mutex); 14821 14822 if (!exit) { 14823 /* 14824 * perf_event_release_kernel() could still have a reference on 14825 * this context. In that case we must wait for these events to 14826 * have been freed (in particular all their references to this 14827 * task must've been dropped). 14828 * 14829 * Without this copy_process() will unconditionally free this 14830 * task (irrespective of its reference count) and 14831 * _free_event()'s put_task_struct(event->hw.target) will be a 14832 * use-after-free. 14833 * 14834 * Wait for all events to drop their context reference. 14835 */ 14836 wait_var_event(&ctx->refcount, 14837 refcount_read(&ctx->refcount) == 1); 14838 } 14839 put_ctx(ctx); 14840 } 14841 14842 /* 14843 * When a task exits, feed back event values to parent events. 14844 * 14845 * Can be called with exec_update_lock held when called from 14846 * setup_new_exec(). 14847 */ 14848 void perf_event_exit_task(struct task_struct *task) 14849 { 14850 struct perf_event *event, *tmp; 14851 14852 WARN_ON_ONCE(task != current); 14853 14854 mutex_lock(&task->perf_event_mutex); 14855 list_for_each_entry_safe(event, tmp, &task->perf_event_list, 14856 owner_entry) { 14857 list_del_init(&event->owner_entry); 14858 14859 /* 14860 * Ensure the list deletion is visible before we clear 14861 * the owner, closes a race against perf_release() where 14862 * we need to serialize on the owner->perf_event_mutex. 14863 */ 14864 smp_store_release(&event->owner, NULL); 14865 } 14866 mutex_unlock(&task->perf_event_mutex); 14867 14868 perf_event_exit_task_context(task, true); 14869 14870 /* 14871 * The perf_event_exit_task_context calls perf_event_task 14872 * with task's task_ctx, which generates EXIT events for 14873 * task contexts and sets task->perf_event_ctxp[] to NULL. 14874 * At this point we need to send EXIT events to cpu contexts. 14875 */ 14876 perf_event_task(task, NULL, 0); 14877 14878 /* 14879 * Detach the perf_ctx_data for the system-wide event. 14880 * 14881 * Done without holding global_ctx_data_rwsem; typically 14882 * attach_global_ctx_data() will skip over this task, but otherwise 14883 * attach_task_ctx_data() will observe PF_EXITING. 14884 */ 14885 detach_task_ctx_data(task); 14886 } 14887 14888 /* 14889 * Free a context as created by inheritance by perf_event_init_task() below, 14890 * used by fork() in case of fail. 14891 * 14892 * Even though the task has never lived, the context and events have been 14893 * exposed through the child_list, so we must take care tearing it all down. 14894 */ 14895 void perf_event_free_task(struct task_struct *task) 14896 { 14897 perf_event_exit_task_context(task, false); 14898 } 14899 14900 void perf_event_delayed_put(struct task_struct *task) 14901 { 14902 WARN_ON_ONCE(task->perf_event_ctxp); 14903 } 14904 14905 struct file *perf_event_get(unsigned int fd) 14906 { 14907 struct file *file = fget(fd); 14908 if (!file) 14909 return ERR_PTR(-EBADF); 14910 14911 if (file->f_op != &perf_fops) { 14912 fput(file); 14913 return ERR_PTR(-EBADF); 14914 } 14915 14916 return file; 14917 } 14918 14919 const struct perf_event *perf_get_event(struct file *file) 14920 { 14921 if (file->f_op != &perf_fops) 14922 return ERR_PTR(-EINVAL); 14923 14924 return file->private_data; 14925 } 14926 14927 const struct perf_event_attr *perf_event_attrs(struct perf_event *event) 14928 { 14929 if (!event) 14930 return ERR_PTR(-EINVAL); 14931 14932 return &event->attr; 14933 } 14934 14935 int perf_allow_kernel(void) 14936 { 14937 if (sysctl_perf_event_paranoid > 1 && !perfmon_capable()) 14938 return -EACCES; 14939 14940 return security_perf_event_open(PERF_SECURITY_KERNEL); 14941 } 14942 EXPORT_SYMBOL_GPL(perf_allow_kernel); 14943 14944 int perf_allow_cpu(void) 14945 { 14946 if (sysctl_perf_event_paranoid > 0 && !perfmon_capable()) 14947 return -EACCES; 14948 14949 return security_perf_event_open(PERF_SECURITY_CPU); 14950 } 14951 EXPORT_SYMBOL_GPL(perf_allow_cpu); 14952 14953 int perf_allow_tracepoint(void) 14954 { 14955 if (sysctl_perf_event_paranoid > -1 && !perfmon_capable()) 14956 return -EPERM; 14957 14958 return security_perf_event_open(PERF_SECURITY_TRACEPOINT); 14959 } 14960 EXPORT_SYMBOL_GPL(perf_allow_tracepoint); 14961 14962 /* 14963 * Inherit an event from parent task to child task. 14964 * 14965 * Returns: 14966 * - valid pointer on success 14967 * - NULL for orphaned events 14968 * - IS_ERR() on error 14969 */ 14970 static struct perf_event * 14971 inherit_event(struct perf_event *parent_event, 14972 struct task_struct *parent, 14973 struct perf_event_context *parent_ctx, 14974 struct task_struct *child, 14975 struct perf_event *group_leader, 14976 struct perf_event_context *child_ctx) 14977 { 14978 enum perf_event_state parent_state = parent_event->state; 14979 struct perf_event_pmu_context *pmu_ctx; 14980 struct perf_event *child_event; 14981 unsigned long flags; 14982 14983 /* 14984 * Instead of creating recursive hierarchies of events, 14985 * we link inherited events back to the original parent, 14986 * which has a filp for sure, which we use as the reference 14987 * count: 14988 */ 14989 if (parent_event->parent) 14990 parent_event = parent_event->parent; 14991 14992 if (parent_event->state <= PERF_EVENT_STATE_REVOKED) 14993 return NULL; 14994 14995 /* 14996 * Event creation should be under SRCU, see perf_pmu_unregister(). 14997 */ 14998 guard(srcu)(&pmus_srcu); 14999 15000 child_event = perf_event_alloc(&parent_event->attr, 15001 parent_event->cpu, 15002 child, 15003 group_leader, parent_event, 15004 NULL, NULL, -1); 15005 if (IS_ERR(child_event)) 15006 return child_event; 15007 15008 get_ctx(child_ctx); 15009 child_event->ctx = child_ctx; 15010 15011 pmu_ctx = find_get_pmu_context(parent_event->pmu_ctx->pmu, child_ctx, child_event); 15012 if (IS_ERR(pmu_ctx)) { 15013 free_event(child_event); 15014 return ERR_CAST(pmu_ctx); 15015 } 15016 child_event->pmu_ctx = pmu_ctx; 15017 15018 /* 15019 * is_orphaned_event() and list_add_tail(&parent_event->child_list) 15020 * must be under the same lock in order to serialize against 15021 * perf_event_release_kernel(), such that either we must observe 15022 * is_orphaned_event() or they will observe us on the child_list. 15023 */ 15024 mutex_lock(&parent_event->child_mutex); 15025 if (is_orphaned_event(parent_event) || 15026 !atomic_long_inc_not_zero(&parent_event->refcount)) { 15027 mutex_unlock(&parent_event->child_mutex); 15028 free_event(child_event); 15029 return NULL; 15030 } 15031 15032 /* 15033 * Make the child state follow the state of the parent event, 15034 * not its attr.disabled bit. We hold the parent's mutex, 15035 * so we won't race with perf_event_{en, dis}able_family. 15036 */ 15037 if (parent_state >= PERF_EVENT_STATE_INACTIVE) 15038 child_event->state = PERF_EVENT_STATE_INACTIVE; 15039 else 15040 child_event->state = PERF_EVENT_STATE_OFF; 15041 15042 if (parent_event->attr.freq) { 15043 u64 sample_period = parent_event->hw.sample_period; 15044 struct hw_perf_event *hwc = &child_event->hw; 15045 15046 hwc->sample_period = sample_period; 15047 hwc->last_period = sample_period; 15048 15049 local64_set(&hwc->period_left, sample_period); 15050 } 15051 15052 child_event->overflow_handler = parent_event->overflow_handler; 15053 child_event->overflow_handler_context 15054 = parent_event->overflow_handler_context; 15055 15056 /* 15057 * Precalculate sample_data sizes 15058 */ 15059 perf_event__header_size(child_event); 15060 perf_event__id_header_size(child_event); 15061 15062 /* 15063 * Link it up in the child's context: 15064 */ 15065 raw_spin_lock_irqsave(&child_ctx->lock, flags); 15066 add_event_to_ctx(child_event, child_ctx); 15067 child_event->attach_state |= PERF_ATTACH_CHILD; 15068 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 15069 15070 /* 15071 * Link this into the parent event's child list 15072 */ 15073 list_add_tail(&child_event->child_list, &parent_event->child_list); 15074 mutex_unlock(&parent_event->child_mutex); 15075 15076 return child_event; 15077 } 15078 15079 /* 15080 * Inherits an event group. 15081 * 15082 * This will quietly suppress orphaned events; !inherit_event() is not an error. 15083 * This matches with perf_event_release_kernel() removing all child events. 15084 * 15085 * Returns: 15086 * - 0 on success 15087 * - <0 on error 15088 */ 15089 static int inherit_group(struct perf_event *parent_event, 15090 struct task_struct *parent, 15091 struct perf_event_context *parent_ctx, 15092 struct task_struct *child, 15093 struct perf_event_context *child_ctx) 15094 { 15095 struct perf_event *leader; 15096 struct perf_event *sub; 15097 struct perf_event *child_ctr; 15098 15099 leader = inherit_event(parent_event, parent, parent_ctx, 15100 child, NULL, child_ctx); 15101 if (IS_ERR(leader)) 15102 return PTR_ERR(leader); 15103 /* 15104 * @leader can be NULL here because of is_orphaned_event(). In this 15105 * case inherit_event() will create individual events, similar to what 15106 * perf_group_detach() would do anyway. 15107 */ 15108 for_each_sibling_event(sub, parent_event) { 15109 child_ctr = inherit_event(sub, parent, parent_ctx, 15110 child, leader, child_ctx); 15111 if (IS_ERR(child_ctr)) 15112 return PTR_ERR(child_ctr); 15113 15114 if (sub->aux_event == parent_event && child_ctr && 15115 !perf_get_aux_event(child_ctr, leader)) 15116 return -EINVAL; 15117 } 15118 if (leader) 15119 leader->group_generation = parent_event->group_generation; 15120 return 0; 15121 } 15122 15123 /* 15124 * Creates the child task context and tries to inherit the event-group. 15125 * 15126 * Clears @inherited_all on !attr.inherited or error. Note that we'll leave 15127 * inherited_all set when we 'fail' to inherit an orphaned event; this is 15128 * consistent with perf_event_release_kernel() removing all child events. 15129 * 15130 * Returns: 15131 * - 0 on success 15132 * - <0 on error 15133 */ 15134 static int 15135 inherit_task_group(struct perf_event *event, struct task_struct *parent, 15136 struct perf_event_context *parent_ctx, 15137 struct task_struct *child, 15138 u64 clone_flags, int *inherited_all) 15139 { 15140 struct perf_event_context *child_ctx; 15141 int ret; 15142 15143 if (!event->attr.inherit || 15144 (event->attr.inherit_thread && !(clone_flags & CLONE_THREAD)) || 15145 /* Do not inherit if sigtrap and signal handlers were cleared. */ 15146 (event->attr.sigtrap && (clone_flags & CLONE_CLEAR_SIGHAND))) { 15147 *inherited_all = 0; 15148 return 0; 15149 } 15150 15151 child_ctx = child->perf_event_ctxp; 15152 if (!child_ctx) { 15153 /* 15154 * This is executed from the parent task context, so 15155 * inherit events that have been marked for cloning. 15156 * First allocate and initialize a context for the 15157 * child. 15158 */ 15159 child_ctx = alloc_perf_context(child); 15160 if (!child_ctx) 15161 return -ENOMEM; 15162 15163 child->perf_event_ctxp = child_ctx; 15164 } 15165 15166 ret = inherit_group(event, parent, parent_ctx, child, child_ctx); 15167 if (ret) 15168 *inherited_all = 0; 15169 15170 return ret; 15171 } 15172 15173 /* 15174 * Initialize the perf_event context in task_struct 15175 */ 15176 static int perf_event_init_context(struct task_struct *child, u64 clone_flags) 15177 { 15178 struct perf_event_context *child_ctx, *parent_ctx; 15179 struct perf_event_context *cloned_ctx; 15180 struct perf_event *event; 15181 struct task_struct *parent = current; 15182 int inherited_all = 1; 15183 unsigned long flags; 15184 int ret = 0; 15185 15186 if (likely(!parent->perf_event_ctxp)) 15187 return 0; 15188 15189 /* 15190 * If the parent's context is a clone, pin it so it won't get 15191 * swapped under us. 15192 */ 15193 parent_ctx = perf_pin_task_context(parent); 15194 if (!parent_ctx) 15195 return 0; 15196 15197 /* 15198 * No need to check if parent_ctx != NULL here; since we saw 15199 * it non-NULL earlier, the only reason for it to become NULL 15200 * is if we exit, and since we're currently in the middle of 15201 * a fork we can't be exiting at the same time. 15202 */ 15203 15204 /* 15205 * Lock the parent list. No need to lock the child - not PID 15206 * hashed yet and not running, so nobody can access it. 15207 */ 15208 mutex_lock(&parent_ctx->mutex); 15209 15210 /* 15211 * We dont have to disable NMIs - we are only looking at 15212 * the list, not manipulating it: 15213 */ 15214 perf_event_groups_for_each(event, &parent_ctx->pinned_groups) { 15215 ret = inherit_task_group(event, parent, parent_ctx, 15216 child, clone_flags, &inherited_all); 15217 if (ret) 15218 goto out_unlock; 15219 } 15220 15221 /* 15222 * We can't hold ctx->lock when iterating the ->flexible_group list due 15223 * to allocations, but we need to prevent rotation because 15224 * rotate_ctx() will change the list from interrupt context. 15225 */ 15226 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 15227 parent_ctx->rotate_disable = 1; 15228 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 15229 15230 perf_event_groups_for_each(event, &parent_ctx->flexible_groups) { 15231 ret = inherit_task_group(event, parent, parent_ctx, 15232 child, clone_flags, &inherited_all); 15233 if (ret) 15234 goto out_unlock; 15235 } 15236 15237 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 15238 parent_ctx->rotate_disable = 0; 15239 15240 child_ctx = child->perf_event_ctxp; 15241 15242 if (child_ctx && inherited_all) { 15243 /* 15244 * Mark the child context as a clone of the parent 15245 * context, or of whatever the parent is a clone of. 15246 * 15247 * Note that if the parent is a clone, the holding of 15248 * parent_ctx->lock avoids it from being uncloned. 15249 */ 15250 cloned_ctx = parent_ctx->parent_ctx; 15251 if (cloned_ctx) { 15252 child_ctx->parent_ctx = cloned_ctx; 15253 child_ctx->parent_gen = parent_ctx->parent_gen; 15254 } else { 15255 child_ctx->parent_ctx = parent_ctx; 15256 child_ctx->parent_gen = parent_ctx->generation; 15257 } 15258 get_ctx(child_ctx->parent_ctx); 15259 } 15260 15261 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 15262 out_unlock: 15263 mutex_unlock(&parent_ctx->mutex); 15264 15265 perf_unpin_context(parent_ctx); 15266 put_ctx(parent_ctx); 15267 15268 return ret; 15269 } 15270 15271 /* 15272 * Initialize the perf_event context in task_struct 15273 */ 15274 int perf_event_init_task(struct task_struct *child, u64 clone_flags) 15275 { 15276 int ret; 15277 15278 memset(child->perf_recursion, 0, sizeof(child->perf_recursion)); 15279 child->perf_event_ctxp = NULL; 15280 mutex_init(&child->perf_event_mutex); 15281 INIT_LIST_HEAD(&child->perf_event_list); 15282 child->perf_ctx_data = NULL; 15283 15284 ret = perf_event_init_context(child, clone_flags); 15285 if (ret) { 15286 perf_event_free_task(child); 15287 return ret; 15288 } 15289 15290 return 0; 15291 } 15292 15293 static void __init perf_event_init_all_cpus(void) 15294 { 15295 struct swevent_htable *swhash; 15296 struct perf_cpu_context *cpuctx; 15297 int cpu; 15298 15299 zalloc_cpumask_var(&perf_online_mask, GFP_KERNEL); 15300 zalloc_cpumask_var(&perf_online_core_mask, GFP_KERNEL); 15301 zalloc_cpumask_var(&perf_online_die_mask, GFP_KERNEL); 15302 zalloc_cpumask_var(&perf_online_cluster_mask, GFP_KERNEL); 15303 zalloc_cpumask_var(&perf_online_pkg_mask, GFP_KERNEL); 15304 zalloc_cpumask_var(&perf_online_sys_mask, GFP_KERNEL); 15305 15306 15307 for_each_possible_cpu(cpu) { 15308 swhash = &per_cpu(swevent_htable, cpu); 15309 mutex_init(&swhash->hlist_mutex); 15310 15311 INIT_LIST_HEAD(&per_cpu(pmu_sb_events.list, cpu)); 15312 raw_spin_lock_init(&per_cpu(pmu_sb_events.lock, cpu)); 15313 15314 INIT_LIST_HEAD(&per_cpu(sched_cb_list, cpu)); 15315 15316 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); 15317 __perf_event_init_context(&cpuctx->ctx); 15318 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); 15319 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); 15320 cpuctx->online = cpumask_test_cpu(cpu, perf_online_mask); 15321 cpuctx->heap_size = ARRAY_SIZE(cpuctx->heap_default); 15322 cpuctx->heap = cpuctx->heap_default; 15323 } 15324 } 15325 15326 static void perf_swevent_init_cpu(unsigned int cpu) 15327 { 15328 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 15329 15330 mutex_lock(&swhash->hlist_mutex); 15331 if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) { 15332 struct swevent_hlist *hlist; 15333 15334 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu)); 15335 WARN_ON(!hlist); 15336 rcu_assign_pointer(swhash->swevent_hlist, hlist); 15337 } 15338 mutex_unlock(&swhash->hlist_mutex); 15339 } 15340 15341 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE 15342 static void __perf_event_exit_context(void *__info) 15343 { 15344 struct perf_cpu_context *cpuctx = this_cpu_ptr(&perf_cpu_context); 15345 struct perf_event_context *ctx = __info; 15346 struct perf_event *event; 15347 15348 raw_spin_lock(&ctx->lock); 15349 ctx_sched_out(ctx, NULL, EVENT_TIME); 15350 list_for_each_entry(event, &ctx->event_list, event_entry) 15351 __perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP); 15352 raw_spin_unlock(&ctx->lock); 15353 } 15354 15355 static void perf_event_clear_cpumask(unsigned int cpu) 15356 { 15357 int target[PERF_PMU_MAX_SCOPE]; 15358 unsigned int scope; 15359 struct pmu *pmu; 15360 15361 cpumask_clear_cpu(cpu, perf_online_mask); 15362 15363 for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) { 15364 const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(scope, cpu); 15365 struct cpumask *pmu_cpumask = perf_scope_cpumask(scope); 15366 15367 target[scope] = -1; 15368 if (WARN_ON_ONCE(!pmu_cpumask || !cpumask)) 15369 continue; 15370 15371 if (!cpumask_test_and_clear_cpu(cpu, pmu_cpumask)) 15372 continue; 15373 target[scope] = cpumask_any_but(cpumask, cpu); 15374 if (target[scope] < nr_cpu_ids) 15375 cpumask_set_cpu(target[scope], pmu_cpumask); 15376 } 15377 15378 /* migrate */ 15379 list_for_each_entry(pmu, &pmus, entry) { 15380 if (pmu->scope == PERF_PMU_SCOPE_NONE || 15381 WARN_ON_ONCE(pmu->scope >= PERF_PMU_MAX_SCOPE)) 15382 continue; 15383 15384 if (target[pmu->scope] >= 0 && target[pmu->scope] < nr_cpu_ids) 15385 perf_pmu_migrate_context(pmu, cpu, target[pmu->scope]); 15386 } 15387 } 15388 15389 static void perf_event_exit_cpu_context(int cpu) 15390 { 15391 struct perf_cpu_context *cpuctx; 15392 struct perf_event_context *ctx; 15393 15394 // XXX simplify cpuctx->online 15395 mutex_lock(&pmus_lock); 15396 /* 15397 * Clear the cpumasks, and migrate to other CPUs if possible. 15398 * Must be invoked before the __perf_event_exit_context. 15399 */ 15400 perf_event_clear_cpumask(cpu); 15401 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); 15402 ctx = &cpuctx->ctx; 15403 15404 mutex_lock(&ctx->mutex); 15405 if (ctx->nr_events) 15406 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1); 15407 cpuctx->online = 0; 15408 mutex_unlock(&ctx->mutex); 15409 mutex_unlock(&pmus_lock); 15410 } 15411 #else 15412 15413 static void perf_event_exit_cpu_context(int cpu) { } 15414 15415 #endif 15416 15417 static void perf_event_setup_cpumask(unsigned int cpu) 15418 { 15419 struct cpumask *pmu_cpumask; 15420 unsigned int scope; 15421 15422 /* 15423 * Early boot stage, the cpumask hasn't been set yet. 15424 * The perf_online_<domain>_masks includes the first CPU of each domain. 15425 * Always unconditionally set the boot CPU for the perf_online_<domain>_masks. 15426 */ 15427 if (cpumask_empty(perf_online_mask)) { 15428 for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) { 15429 pmu_cpumask = perf_scope_cpumask(scope); 15430 if (WARN_ON_ONCE(!pmu_cpumask)) 15431 continue; 15432 cpumask_set_cpu(cpu, pmu_cpumask); 15433 } 15434 goto end; 15435 } 15436 15437 for (scope = PERF_PMU_SCOPE_NONE + 1; scope < PERF_PMU_MAX_SCOPE; scope++) { 15438 const struct cpumask *cpumask = perf_scope_cpu_topology_cpumask(scope, cpu); 15439 15440 pmu_cpumask = perf_scope_cpumask(scope); 15441 15442 if (WARN_ON_ONCE(!pmu_cpumask || !cpumask)) 15443 continue; 15444 15445 if (!cpumask_empty(cpumask) && 15446 cpumask_any_and(pmu_cpumask, cpumask) >= nr_cpu_ids) 15447 cpumask_set_cpu(cpu, pmu_cpumask); 15448 } 15449 end: 15450 cpumask_set_cpu(cpu, perf_online_mask); 15451 } 15452 15453 int perf_event_init_cpu(unsigned int cpu) 15454 { 15455 struct perf_cpu_context *cpuctx; 15456 struct perf_event_context *ctx; 15457 15458 perf_swevent_init_cpu(cpu); 15459 15460 mutex_lock(&pmus_lock); 15461 perf_event_setup_cpumask(cpu); 15462 cpuctx = per_cpu_ptr(&perf_cpu_context, cpu); 15463 ctx = &cpuctx->ctx; 15464 15465 mutex_lock(&ctx->mutex); 15466 cpuctx->online = 1; 15467 mutex_unlock(&ctx->mutex); 15468 mutex_unlock(&pmus_lock); 15469 15470 return 0; 15471 } 15472 15473 int perf_event_exit_cpu(unsigned int cpu) 15474 { 15475 perf_event_exit_cpu_context(cpu); 15476 return 0; 15477 } 15478 15479 static int 15480 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v) 15481 { 15482 int cpu; 15483 15484 for_each_online_cpu(cpu) 15485 perf_event_exit_cpu(cpu); 15486 15487 return NOTIFY_OK; 15488 } 15489 15490 /* 15491 * Run the perf reboot notifier at the very last possible moment so that 15492 * the generic watchdog code runs as long as possible. 15493 */ 15494 static struct notifier_block perf_reboot_notifier = { 15495 .notifier_call = perf_reboot, 15496 .priority = INT_MIN, 15497 }; 15498 15499 void __init perf_event_init(void) 15500 { 15501 int ret; 15502 15503 idr_init(&pmu_idr); 15504 15505 unwind_deferred_init(&perf_unwind_work, 15506 perf_unwind_deferred_callback); 15507 15508 perf_event_init_all_cpus(); 15509 init_srcu_struct(&pmus_srcu); 15510 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); 15511 perf_pmu_register(&perf_cpu_clock, "cpu_clock", -1); 15512 perf_pmu_register(&perf_task_clock, "task_clock", -1); 15513 perf_tp_register(); 15514 perf_event_init_cpu(smp_processor_id()); 15515 register_reboot_notifier(&perf_reboot_notifier); 15516 15517 ret = init_hw_breakpoint(); 15518 WARN(ret, "hw_breakpoint initialization failed with: %d", ret); 15519 15520 perf_event_cache = KMEM_CACHE(perf_event, SLAB_PANIC); 15521 15522 /* 15523 * Build time assertion that we keep the data_head at the intended 15524 * location. IOW, validation we got the __reserved[] size right. 15525 */ 15526 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head)) 15527 != 1024); 15528 } 15529 15530 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr, 15531 char *page) 15532 { 15533 struct perf_pmu_events_attr *pmu_attr = 15534 container_of(attr, struct perf_pmu_events_attr, attr); 15535 15536 if (pmu_attr->event_str) 15537 return sprintf(page, "%s\n", pmu_attr->event_str); 15538 15539 return 0; 15540 } 15541 EXPORT_SYMBOL_GPL(perf_event_sysfs_show); 15542 15543 static int __init perf_event_sysfs_init(void) 15544 { 15545 struct pmu *pmu; 15546 int ret; 15547 15548 mutex_lock(&pmus_lock); 15549 15550 ret = bus_register(&pmu_bus); 15551 if (ret) 15552 goto unlock; 15553 15554 list_for_each_entry(pmu, &pmus, entry) { 15555 if (pmu->dev) 15556 continue; 15557 15558 ret = pmu_dev_alloc(pmu); 15559 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); 15560 } 15561 pmu_bus_running = 1; 15562 ret = 0; 15563 15564 unlock: 15565 mutex_unlock(&pmus_lock); 15566 15567 return ret; 15568 } 15569 device_initcall(perf_event_sysfs_init); 15570 15571 #ifdef CONFIG_CGROUP_PERF 15572 static struct cgroup_subsys_state * 15573 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 15574 { 15575 struct perf_cgroup *jc; 15576 15577 jc = kzalloc_obj(*jc); 15578 if (!jc) 15579 return ERR_PTR(-ENOMEM); 15580 15581 jc->info = alloc_percpu(struct perf_cgroup_info); 15582 if (!jc->info) { 15583 kfree(jc); 15584 return ERR_PTR(-ENOMEM); 15585 } 15586 15587 return &jc->css; 15588 } 15589 15590 static void perf_cgroup_css_free(struct cgroup_subsys_state *css) 15591 { 15592 struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css); 15593 15594 free_percpu(jc->info); 15595 kfree(jc); 15596 } 15597 15598 static int perf_cgroup_css_online(struct cgroup_subsys_state *css) 15599 { 15600 perf_event_cgroup(css->cgroup); 15601 return 0; 15602 } 15603 15604 static int __perf_cgroup_move(void *info) 15605 { 15606 struct task_struct *task = info; 15607 15608 preempt_disable(); 15609 perf_cgroup_switch(task); 15610 preempt_enable(); 15611 15612 return 0; 15613 } 15614 15615 static void perf_cgroup_attach(struct cgroup_taskset *tset) 15616 { 15617 struct task_struct *task; 15618 struct cgroup_subsys_state *css; 15619 15620 cgroup_taskset_for_each(task, css, tset) 15621 task_function_call(task, __perf_cgroup_move, task); 15622 } 15623 15624 struct cgroup_subsys perf_event_cgrp_subsys = { 15625 .css_alloc = perf_cgroup_css_alloc, 15626 .css_free = perf_cgroup_css_free, 15627 .css_online = perf_cgroup_css_online, 15628 .attach = perf_cgroup_attach, 15629 /* 15630 * Implicitly enable on dfl hierarchy so that perf events can 15631 * always be filtered by cgroup2 path as long as perf_event 15632 * controller is not mounted on a legacy hierarchy. 15633 */ 15634 .implicit_on_dfl = true, 15635 .threaded = true, 15636 }; 15637 #endif /* CONFIG_CGROUP_PERF */ 15638 15639 DEFINE_STATIC_CALL_RET0(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t); 15640