1 /* 2 * Performance events core code: 3 * 4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de> 5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar 6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra 7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 8 * 9 * For licensing details see kernel-base/COPYING 10 */ 11 12 #include <linux/fs.h> 13 #include <linux/mm.h> 14 #include <linux/cpu.h> 15 #include <linux/smp.h> 16 #include <linux/idr.h> 17 #include <linux/file.h> 18 #include <linux/poll.h> 19 #include <linux/slab.h> 20 #include <linux/hash.h> 21 #include <linux/tick.h> 22 #include <linux/sysfs.h> 23 #include <linux/dcache.h> 24 #include <linux/percpu.h> 25 #include <linux/ptrace.h> 26 #include <linux/reboot.h> 27 #include <linux/vmstat.h> 28 #include <linux/device.h> 29 #include <linux/export.h> 30 #include <linux/vmalloc.h> 31 #include <linux/hardirq.h> 32 #include <linux/rculist.h> 33 #include <linux/uaccess.h> 34 #include <linux/syscalls.h> 35 #include <linux/anon_inodes.h> 36 #include <linux/kernel_stat.h> 37 #include <linux/cgroup.h> 38 #include <linux/perf_event.h> 39 #include <linux/trace_events.h> 40 #include <linux/hw_breakpoint.h> 41 #include <linux/mm_types.h> 42 #include <linux/module.h> 43 #include <linux/mman.h> 44 #include <linux/compat.h> 45 #include <linux/bpf.h> 46 #include <linux/filter.h> 47 #include <linux/namei.h> 48 #include <linux/parser.h> 49 50 #include "internal.h" 51 52 #include <asm/irq_regs.h> 53 54 typedef int (*remote_function_f)(void *); 55 56 struct remote_function_call { 57 struct task_struct *p; 58 remote_function_f func; 59 void *info; 60 int ret; 61 }; 62 63 static void remote_function(void *data) 64 { 65 struct remote_function_call *tfc = data; 66 struct task_struct *p = tfc->p; 67 68 if (p) { 69 /* -EAGAIN */ 70 if (task_cpu(p) != smp_processor_id()) 71 return; 72 73 /* 74 * Now that we're on right CPU with IRQs disabled, we can test 75 * if we hit the right task without races. 76 */ 77 78 tfc->ret = -ESRCH; /* No such (running) process */ 79 if (p != current) 80 return; 81 } 82 83 tfc->ret = tfc->func(tfc->info); 84 } 85 86 /** 87 * task_function_call - call a function on the cpu on which a task runs 88 * @p: the task to evaluate 89 * @func: the function to be called 90 * @info: the function call argument 91 * 92 * Calls the function @func when the task is currently running. This might 93 * be on the current CPU, which just calls the function directly 94 * 95 * returns: @func return value, or 96 * -ESRCH - when the process isn't running 97 * -EAGAIN - when the process moved away 98 */ 99 static int 100 task_function_call(struct task_struct *p, remote_function_f func, void *info) 101 { 102 struct remote_function_call data = { 103 .p = p, 104 .func = func, 105 .info = info, 106 .ret = -EAGAIN, 107 }; 108 int ret; 109 110 do { 111 ret = smp_call_function_single(task_cpu(p), remote_function, &data, 1); 112 if (!ret) 113 ret = data.ret; 114 } while (ret == -EAGAIN); 115 116 return ret; 117 } 118 119 /** 120 * cpu_function_call - call a function on the cpu 121 * @func: the function to be called 122 * @info: the function call argument 123 * 124 * Calls the function @func on the remote cpu. 125 * 126 * returns: @func return value or -ENXIO when the cpu is offline 127 */ 128 static int cpu_function_call(int cpu, remote_function_f func, void *info) 129 { 130 struct remote_function_call data = { 131 .p = NULL, 132 .func = func, 133 .info = info, 134 .ret = -ENXIO, /* No such CPU */ 135 }; 136 137 smp_call_function_single(cpu, remote_function, &data, 1); 138 139 return data.ret; 140 } 141 142 static inline struct perf_cpu_context * 143 __get_cpu_context(struct perf_event_context *ctx) 144 { 145 return this_cpu_ptr(ctx->pmu->pmu_cpu_context); 146 } 147 148 static void perf_ctx_lock(struct perf_cpu_context *cpuctx, 149 struct perf_event_context *ctx) 150 { 151 raw_spin_lock(&cpuctx->ctx.lock); 152 if (ctx) 153 raw_spin_lock(&ctx->lock); 154 } 155 156 static void perf_ctx_unlock(struct perf_cpu_context *cpuctx, 157 struct perf_event_context *ctx) 158 { 159 if (ctx) 160 raw_spin_unlock(&ctx->lock); 161 raw_spin_unlock(&cpuctx->ctx.lock); 162 } 163 164 #define TASK_TOMBSTONE ((void *)-1L) 165 166 static bool is_kernel_event(struct perf_event *event) 167 { 168 return READ_ONCE(event->owner) == TASK_TOMBSTONE; 169 } 170 171 /* 172 * On task ctx scheduling... 173 * 174 * When !ctx->nr_events a task context will not be scheduled. This means 175 * we can disable the scheduler hooks (for performance) without leaving 176 * pending task ctx state. 177 * 178 * This however results in two special cases: 179 * 180 * - removing the last event from a task ctx; this is relatively straight 181 * forward and is done in __perf_remove_from_context. 182 * 183 * - adding the first event to a task ctx; this is tricky because we cannot 184 * rely on ctx->is_active and therefore cannot use event_function_call(). 185 * See perf_install_in_context(). 186 * 187 * If ctx->nr_events, then ctx->is_active and cpuctx->task_ctx are set. 188 */ 189 190 typedef void (*event_f)(struct perf_event *, struct perf_cpu_context *, 191 struct perf_event_context *, void *); 192 193 struct event_function_struct { 194 struct perf_event *event; 195 event_f func; 196 void *data; 197 }; 198 199 static int event_function(void *info) 200 { 201 struct event_function_struct *efs = info; 202 struct perf_event *event = efs->event; 203 struct perf_event_context *ctx = event->ctx; 204 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 205 struct perf_event_context *task_ctx = cpuctx->task_ctx; 206 int ret = 0; 207 208 WARN_ON_ONCE(!irqs_disabled()); 209 210 perf_ctx_lock(cpuctx, task_ctx); 211 /* 212 * Since we do the IPI call without holding ctx->lock things can have 213 * changed, double check we hit the task we set out to hit. 214 */ 215 if (ctx->task) { 216 if (ctx->task != current) { 217 ret = -ESRCH; 218 goto unlock; 219 } 220 221 /* 222 * We only use event_function_call() on established contexts, 223 * and event_function() is only ever called when active (or 224 * rather, we'll have bailed in task_function_call() or the 225 * above ctx->task != current test), therefore we must have 226 * ctx->is_active here. 227 */ 228 WARN_ON_ONCE(!ctx->is_active); 229 /* 230 * And since we have ctx->is_active, cpuctx->task_ctx must 231 * match. 232 */ 233 WARN_ON_ONCE(task_ctx != ctx); 234 } else { 235 WARN_ON_ONCE(&cpuctx->ctx != ctx); 236 } 237 238 efs->func(event, cpuctx, ctx, efs->data); 239 unlock: 240 perf_ctx_unlock(cpuctx, task_ctx); 241 242 return ret; 243 } 244 245 static void event_function_local(struct perf_event *event, event_f func, void *data) 246 { 247 struct event_function_struct efs = { 248 .event = event, 249 .func = func, 250 .data = data, 251 }; 252 253 int ret = event_function(&efs); 254 WARN_ON_ONCE(ret); 255 } 256 257 static void event_function_call(struct perf_event *event, event_f func, void *data) 258 { 259 struct perf_event_context *ctx = event->ctx; 260 struct task_struct *task = READ_ONCE(ctx->task); /* verified in event_function */ 261 struct event_function_struct efs = { 262 .event = event, 263 .func = func, 264 .data = data, 265 }; 266 267 if (!event->parent) { 268 /* 269 * If this is a !child event, we must hold ctx::mutex to 270 * stabilize the the event->ctx relation. See 271 * perf_event_ctx_lock(). 272 */ 273 lockdep_assert_held(&ctx->mutex); 274 } 275 276 if (!task) { 277 cpu_function_call(event->cpu, event_function, &efs); 278 return; 279 } 280 281 if (task == TASK_TOMBSTONE) 282 return; 283 284 again: 285 if (!task_function_call(task, event_function, &efs)) 286 return; 287 288 raw_spin_lock_irq(&ctx->lock); 289 /* 290 * Reload the task pointer, it might have been changed by 291 * a concurrent perf_event_context_sched_out(). 292 */ 293 task = ctx->task; 294 if (task == TASK_TOMBSTONE) { 295 raw_spin_unlock_irq(&ctx->lock); 296 return; 297 } 298 if (ctx->is_active) { 299 raw_spin_unlock_irq(&ctx->lock); 300 goto again; 301 } 302 func(event, NULL, ctx, data); 303 raw_spin_unlock_irq(&ctx->lock); 304 } 305 306 #define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\ 307 PERF_FLAG_FD_OUTPUT |\ 308 PERF_FLAG_PID_CGROUP |\ 309 PERF_FLAG_FD_CLOEXEC) 310 311 /* 312 * branch priv levels that need permission checks 313 */ 314 #define PERF_SAMPLE_BRANCH_PERM_PLM \ 315 (PERF_SAMPLE_BRANCH_KERNEL |\ 316 PERF_SAMPLE_BRANCH_HV) 317 318 enum event_type_t { 319 EVENT_FLEXIBLE = 0x1, 320 EVENT_PINNED = 0x2, 321 EVENT_TIME = 0x4, 322 EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED, 323 }; 324 325 /* 326 * perf_sched_events : >0 events exist 327 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu 328 */ 329 330 static void perf_sched_delayed(struct work_struct *work); 331 DEFINE_STATIC_KEY_FALSE(perf_sched_events); 332 static DECLARE_DELAYED_WORK(perf_sched_work, perf_sched_delayed); 333 static DEFINE_MUTEX(perf_sched_mutex); 334 static atomic_t perf_sched_count; 335 336 static DEFINE_PER_CPU(atomic_t, perf_cgroup_events); 337 static DEFINE_PER_CPU(int, perf_sched_cb_usages); 338 339 static atomic_t nr_mmap_events __read_mostly; 340 static atomic_t nr_comm_events __read_mostly; 341 static atomic_t nr_task_events __read_mostly; 342 static atomic_t nr_freq_events __read_mostly; 343 static atomic_t nr_switch_events __read_mostly; 344 345 static LIST_HEAD(pmus); 346 static DEFINE_MUTEX(pmus_lock); 347 static struct srcu_struct pmus_srcu; 348 349 /* 350 * perf event paranoia level: 351 * -1 - not paranoid at all 352 * 0 - disallow raw tracepoint access for unpriv 353 * 1 - disallow cpu events for unpriv 354 * 2 - disallow kernel profiling for unpriv 355 */ 356 int sysctl_perf_event_paranoid __read_mostly = 2; 357 358 /* Minimum for 512 kiB + 1 user control page */ 359 int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */ 360 361 /* 362 * max perf event sample rate 363 */ 364 #define DEFAULT_MAX_SAMPLE_RATE 100000 365 #define DEFAULT_SAMPLE_PERIOD_NS (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE) 366 #define DEFAULT_CPU_TIME_MAX_PERCENT 25 367 368 int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE; 369 370 static int max_samples_per_tick __read_mostly = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ); 371 static int perf_sample_period_ns __read_mostly = DEFAULT_SAMPLE_PERIOD_NS; 372 373 static int perf_sample_allowed_ns __read_mostly = 374 DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100; 375 376 static void update_perf_cpu_limits(void) 377 { 378 u64 tmp = perf_sample_period_ns; 379 380 tmp *= sysctl_perf_cpu_time_max_percent; 381 tmp = div_u64(tmp, 100); 382 if (!tmp) 383 tmp = 1; 384 385 WRITE_ONCE(perf_sample_allowed_ns, tmp); 386 } 387 388 static int perf_rotate_context(struct perf_cpu_context *cpuctx); 389 390 int perf_proc_update_handler(struct ctl_table *table, int write, 391 void __user *buffer, size_t *lenp, 392 loff_t *ppos) 393 { 394 int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos); 395 396 if (ret || !write) 397 return ret; 398 399 max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ); 400 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 401 update_perf_cpu_limits(); 402 403 return 0; 404 } 405 406 int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT; 407 408 int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write, 409 void __user *buffer, size_t *lenp, 410 loff_t *ppos) 411 { 412 int ret = proc_dointvec(table, write, buffer, lenp, ppos); 413 414 if (ret || !write) 415 return ret; 416 417 if (sysctl_perf_cpu_time_max_percent == 100 || 418 sysctl_perf_cpu_time_max_percent == 0) { 419 printk(KERN_WARNING 420 "perf: Dynamic interrupt throttling disabled, can hang your system!\n"); 421 WRITE_ONCE(perf_sample_allowed_ns, 0); 422 } else { 423 update_perf_cpu_limits(); 424 } 425 426 return 0; 427 } 428 429 /* 430 * perf samples are done in some very critical code paths (NMIs). 431 * If they take too much CPU time, the system can lock up and not 432 * get any real work done. This will drop the sample rate when 433 * we detect that events are taking too long. 434 */ 435 #define NR_ACCUMULATED_SAMPLES 128 436 static DEFINE_PER_CPU(u64, running_sample_length); 437 438 static u64 __report_avg; 439 static u64 __report_allowed; 440 441 static void perf_duration_warn(struct irq_work *w) 442 { 443 printk_ratelimited(KERN_WARNING 444 "perf: interrupt took too long (%lld > %lld), lowering " 445 "kernel.perf_event_max_sample_rate to %d\n", 446 __report_avg, __report_allowed, 447 sysctl_perf_event_sample_rate); 448 } 449 450 static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn); 451 452 void perf_sample_event_took(u64 sample_len_ns) 453 { 454 u64 max_len = READ_ONCE(perf_sample_allowed_ns); 455 u64 running_len; 456 u64 avg_len; 457 u32 max; 458 459 if (max_len == 0) 460 return; 461 462 /* Decay the counter by 1 average sample. */ 463 running_len = __this_cpu_read(running_sample_length); 464 running_len -= running_len/NR_ACCUMULATED_SAMPLES; 465 running_len += sample_len_ns; 466 __this_cpu_write(running_sample_length, running_len); 467 468 /* 469 * Note: this will be biased artifically low until we have 470 * seen NR_ACCUMULATED_SAMPLES. Doing it this way keeps us 471 * from having to maintain a count. 472 */ 473 avg_len = running_len/NR_ACCUMULATED_SAMPLES; 474 if (avg_len <= max_len) 475 return; 476 477 __report_avg = avg_len; 478 __report_allowed = max_len; 479 480 /* 481 * Compute a throttle threshold 25% below the current duration. 482 */ 483 avg_len += avg_len / 4; 484 max = (TICK_NSEC / 100) * sysctl_perf_cpu_time_max_percent; 485 if (avg_len < max) 486 max /= (u32)avg_len; 487 else 488 max = 1; 489 490 WRITE_ONCE(perf_sample_allowed_ns, avg_len); 491 WRITE_ONCE(max_samples_per_tick, max); 492 493 sysctl_perf_event_sample_rate = max * HZ; 494 perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate; 495 496 if (!irq_work_queue(&perf_duration_work)) { 497 early_printk("perf: interrupt took too long (%lld > %lld), lowering " 498 "kernel.perf_event_max_sample_rate to %d\n", 499 __report_avg, __report_allowed, 500 sysctl_perf_event_sample_rate); 501 } 502 } 503 504 static atomic64_t perf_event_id; 505 506 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 507 enum event_type_t event_type); 508 509 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 510 enum event_type_t event_type, 511 struct task_struct *task); 512 513 static void update_context_time(struct perf_event_context *ctx); 514 static u64 perf_event_time(struct perf_event *event); 515 516 void __weak perf_event_print_debug(void) { } 517 518 extern __weak const char *perf_pmu_name(void) 519 { 520 return "pmu"; 521 } 522 523 static inline u64 perf_clock(void) 524 { 525 return local_clock(); 526 } 527 528 static inline u64 perf_event_clock(struct perf_event *event) 529 { 530 return event->clock(); 531 } 532 533 #ifdef CONFIG_CGROUP_PERF 534 535 static inline bool 536 perf_cgroup_match(struct perf_event *event) 537 { 538 struct perf_event_context *ctx = event->ctx; 539 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 540 541 /* @event doesn't care about cgroup */ 542 if (!event->cgrp) 543 return true; 544 545 /* wants specific cgroup scope but @cpuctx isn't associated with any */ 546 if (!cpuctx->cgrp) 547 return false; 548 549 /* 550 * Cgroup scoping is recursive. An event enabled for a cgroup is 551 * also enabled for all its descendant cgroups. If @cpuctx's 552 * cgroup is a descendant of @event's (the test covers identity 553 * case), it's a match. 554 */ 555 return cgroup_is_descendant(cpuctx->cgrp->css.cgroup, 556 event->cgrp->css.cgroup); 557 } 558 559 static inline void perf_detach_cgroup(struct perf_event *event) 560 { 561 css_put(&event->cgrp->css); 562 event->cgrp = NULL; 563 } 564 565 static inline int is_cgroup_event(struct perf_event *event) 566 { 567 return event->cgrp != NULL; 568 } 569 570 static inline u64 perf_cgroup_event_time(struct perf_event *event) 571 { 572 struct perf_cgroup_info *t; 573 574 t = per_cpu_ptr(event->cgrp->info, event->cpu); 575 return t->time; 576 } 577 578 static inline void __update_cgrp_time(struct perf_cgroup *cgrp) 579 { 580 struct perf_cgroup_info *info; 581 u64 now; 582 583 now = perf_clock(); 584 585 info = this_cpu_ptr(cgrp->info); 586 587 info->time += now - info->timestamp; 588 info->timestamp = now; 589 } 590 591 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 592 { 593 struct perf_cgroup *cgrp_out = cpuctx->cgrp; 594 if (cgrp_out) 595 __update_cgrp_time(cgrp_out); 596 } 597 598 static inline void update_cgrp_time_from_event(struct perf_event *event) 599 { 600 struct perf_cgroup *cgrp; 601 602 /* 603 * ensure we access cgroup data only when needed and 604 * when we know the cgroup is pinned (css_get) 605 */ 606 if (!is_cgroup_event(event)) 607 return; 608 609 cgrp = perf_cgroup_from_task(current, event->ctx); 610 /* 611 * Do not update time when cgroup is not active 612 */ 613 if (cgrp == event->cgrp) 614 __update_cgrp_time(event->cgrp); 615 } 616 617 static inline void 618 perf_cgroup_set_timestamp(struct task_struct *task, 619 struct perf_event_context *ctx) 620 { 621 struct perf_cgroup *cgrp; 622 struct perf_cgroup_info *info; 623 624 /* 625 * ctx->lock held by caller 626 * ensure we do not access cgroup data 627 * unless we have the cgroup pinned (css_get) 628 */ 629 if (!task || !ctx->nr_cgroups) 630 return; 631 632 cgrp = perf_cgroup_from_task(task, ctx); 633 info = this_cpu_ptr(cgrp->info); 634 info->timestamp = ctx->timestamp; 635 } 636 637 #define PERF_CGROUP_SWOUT 0x1 /* cgroup switch out every event */ 638 #define PERF_CGROUP_SWIN 0x2 /* cgroup switch in events based on task */ 639 640 /* 641 * reschedule events based on the cgroup constraint of task. 642 * 643 * mode SWOUT : schedule out everything 644 * mode SWIN : schedule in based on cgroup for next 645 */ 646 static void perf_cgroup_switch(struct task_struct *task, int mode) 647 { 648 struct perf_cpu_context *cpuctx; 649 struct pmu *pmu; 650 unsigned long flags; 651 652 /* 653 * disable interrupts to avoid geting nr_cgroup 654 * changes via __perf_event_disable(). Also 655 * avoids preemption. 656 */ 657 local_irq_save(flags); 658 659 /* 660 * we reschedule only in the presence of cgroup 661 * constrained events. 662 */ 663 664 list_for_each_entry_rcu(pmu, &pmus, entry) { 665 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 666 if (cpuctx->unique_pmu != pmu) 667 continue; /* ensure we process each cpuctx once */ 668 669 /* 670 * perf_cgroup_events says at least one 671 * context on this CPU has cgroup events. 672 * 673 * ctx->nr_cgroups reports the number of cgroup 674 * events for a context. 675 */ 676 if (cpuctx->ctx.nr_cgroups > 0) { 677 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 678 perf_pmu_disable(cpuctx->ctx.pmu); 679 680 if (mode & PERF_CGROUP_SWOUT) { 681 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 682 /* 683 * must not be done before ctxswout due 684 * to event_filter_match() in event_sched_out() 685 */ 686 cpuctx->cgrp = NULL; 687 } 688 689 if (mode & PERF_CGROUP_SWIN) { 690 WARN_ON_ONCE(cpuctx->cgrp); 691 /* 692 * set cgrp before ctxsw in to allow 693 * event_filter_match() to not have to pass 694 * task around 695 * we pass the cpuctx->ctx to perf_cgroup_from_task() 696 * because cgorup events are only per-cpu 697 */ 698 cpuctx->cgrp = perf_cgroup_from_task(task, &cpuctx->ctx); 699 cpu_ctx_sched_in(cpuctx, EVENT_ALL, task); 700 } 701 perf_pmu_enable(cpuctx->ctx.pmu); 702 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 703 } 704 } 705 706 local_irq_restore(flags); 707 } 708 709 static inline void perf_cgroup_sched_out(struct task_struct *task, 710 struct task_struct *next) 711 { 712 struct perf_cgroup *cgrp1; 713 struct perf_cgroup *cgrp2 = NULL; 714 715 rcu_read_lock(); 716 /* 717 * we come here when we know perf_cgroup_events > 0 718 * we do not need to pass the ctx here because we know 719 * we are holding the rcu lock 720 */ 721 cgrp1 = perf_cgroup_from_task(task, NULL); 722 cgrp2 = perf_cgroup_from_task(next, NULL); 723 724 /* 725 * only schedule out current cgroup events if we know 726 * that we are switching to a different cgroup. Otherwise, 727 * do no touch the cgroup events. 728 */ 729 if (cgrp1 != cgrp2) 730 perf_cgroup_switch(task, PERF_CGROUP_SWOUT); 731 732 rcu_read_unlock(); 733 } 734 735 static inline void perf_cgroup_sched_in(struct task_struct *prev, 736 struct task_struct *task) 737 { 738 struct perf_cgroup *cgrp1; 739 struct perf_cgroup *cgrp2 = NULL; 740 741 rcu_read_lock(); 742 /* 743 * we come here when we know perf_cgroup_events > 0 744 * we do not need to pass the ctx here because we know 745 * we are holding the rcu lock 746 */ 747 cgrp1 = perf_cgroup_from_task(task, NULL); 748 cgrp2 = perf_cgroup_from_task(prev, NULL); 749 750 /* 751 * only need to schedule in cgroup events if we are changing 752 * cgroup during ctxsw. Cgroup events were not scheduled 753 * out of ctxsw out if that was not the case. 754 */ 755 if (cgrp1 != cgrp2) 756 perf_cgroup_switch(task, PERF_CGROUP_SWIN); 757 758 rcu_read_unlock(); 759 } 760 761 static inline int perf_cgroup_connect(int fd, struct perf_event *event, 762 struct perf_event_attr *attr, 763 struct perf_event *group_leader) 764 { 765 struct perf_cgroup *cgrp; 766 struct cgroup_subsys_state *css; 767 struct fd f = fdget(fd); 768 int ret = 0; 769 770 if (!f.file) 771 return -EBADF; 772 773 css = css_tryget_online_from_dir(f.file->f_path.dentry, 774 &perf_event_cgrp_subsys); 775 if (IS_ERR(css)) { 776 ret = PTR_ERR(css); 777 goto out; 778 } 779 780 cgrp = container_of(css, struct perf_cgroup, css); 781 event->cgrp = cgrp; 782 783 /* 784 * all events in a group must monitor 785 * the same cgroup because a task belongs 786 * to only one perf cgroup at a time 787 */ 788 if (group_leader && group_leader->cgrp != cgrp) { 789 perf_detach_cgroup(event); 790 ret = -EINVAL; 791 } 792 out: 793 fdput(f); 794 return ret; 795 } 796 797 static inline void 798 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 799 { 800 struct perf_cgroup_info *t; 801 t = per_cpu_ptr(event->cgrp->info, event->cpu); 802 event->shadow_ctx_time = now - t->timestamp; 803 } 804 805 static inline void 806 perf_cgroup_defer_enabled(struct perf_event *event) 807 { 808 /* 809 * when the current task's perf cgroup does not match 810 * the event's, we need to remember to call the 811 * perf_mark_enable() function the first time a task with 812 * a matching perf cgroup is scheduled in. 813 */ 814 if (is_cgroup_event(event) && !perf_cgroup_match(event)) 815 event->cgrp_defer_enabled = 1; 816 } 817 818 static inline void 819 perf_cgroup_mark_enabled(struct perf_event *event, 820 struct perf_event_context *ctx) 821 { 822 struct perf_event *sub; 823 u64 tstamp = perf_event_time(event); 824 825 if (!event->cgrp_defer_enabled) 826 return; 827 828 event->cgrp_defer_enabled = 0; 829 830 event->tstamp_enabled = tstamp - event->total_time_enabled; 831 list_for_each_entry(sub, &event->sibling_list, group_entry) { 832 if (sub->state >= PERF_EVENT_STATE_INACTIVE) { 833 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 834 sub->cgrp_defer_enabled = 0; 835 } 836 } 837 } 838 #else /* !CONFIG_CGROUP_PERF */ 839 840 static inline bool 841 perf_cgroup_match(struct perf_event *event) 842 { 843 return true; 844 } 845 846 static inline void perf_detach_cgroup(struct perf_event *event) 847 {} 848 849 static inline int is_cgroup_event(struct perf_event *event) 850 { 851 return 0; 852 } 853 854 static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event) 855 { 856 return 0; 857 } 858 859 static inline void update_cgrp_time_from_event(struct perf_event *event) 860 { 861 } 862 863 static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx) 864 { 865 } 866 867 static inline void perf_cgroup_sched_out(struct task_struct *task, 868 struct task_struct *next) 869 { 870 } 871 872 static inline void perf_cgroup_sched_in(struct task_struct *prev, 873 struct task_struct *task) 874 { 875 } 876 877 static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event, 878 struct perf_event_attr *attr, 879 struct perf_event *group_leader) 880 { 881 return -EINVAL; 882 } 883 884 static inline void 885 perf_cgroup_set_timestamp(struct task_struct *task, 886 struct perf_event_context *ctx) 887 { 888 } 889 890 void 891 perf_cgroup_switch(struct task_struct *task, struct task_struct *next) 892 { 893 } 894 895 static inline void 896 perf_cgroup_set_shadow_time(struct perf_event *event, u64 now) 897 { 898 } 899 900 static inline u64 perf_cgroup_event_time(struct perf_event *event) 901 { 902 return 0; 903 } 904 905 static inline void 906 perf_cgroup_defer_enabled(struct perf_event *event) 907 { 908 } 909 910 static inline void 911 perf_cgroup_mark_enabled(struct perf_event *event, 912 struct perf_event_context *ctx) 913 { 914 } 915 #endif 916 917 /* 918 * set default to be dependent on timer tick just 919 * like original code 920 */ 921 #define PERF_CPU_HRTIMER (1000 / HZ) 922 /* 923 * function must be called with interrupts disbled 924 */ 925 static enum hrtimer_restart perf_mux_hrtimer_handler(struct hrtimer *hr) 926 { 927 struct perf_cpu_context *cpuctx; 928 int rotations = 0; 929 930 WARN_ON(!irqs_disabled()); 931 932 cpuctx = container_of(hr, struct perf_cpu_context, hrtimer); 933 rotations = perf_rotate_context(cpuctx); 934 935 raw_spin_lock(&cpuctx->hrtimer_lock); 936 if (rotations) 937 hrtimer_forward_now(hr, cpuctx->hrtimer_interval); 938 else 939 cpuctx->hrtimer_active = 0; 940 raw_spin_unlock(&cpuctx->hrtimer_lock); 941 942 return rotations ? HRTIMER_RESTART : HRTIMER_NORESTART; 943 } 944 945 static void __perf_mux_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu) 946 { 947 struct hrtimer *timer = &cpuctx->hrtimer; 948 struct pmu *pmu = cpuctx->ctx.pmu; 949 u64 interval; 950 951 /* no multiplexing needed for SW PMU */ 952 if (pmu->task_ctx_nr == perf_sw_context) 953 return; 954 955 /* 956 * check default is sane, if not set then force to 957 * default interval (1/tick) 958 */ 959 interval = pmu->hrtimer_interval_ms; 960 if (interval < 1) 961 interval = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER; 962 963 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval); 964 965 raw_spin_lock_init(&cpuctx->hrtimer_lock); 966 hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 967 timer->function = perf_mux_hrtimer_handler; 968 } 969 970 static int perf_mux_hrtimer_restart(struct perf_cpu_context *cpuctx) 971 { 972 struct hrtimer *timer = &cpuctx->hrtimer; 973 struct pmu *pmu = cpuctx->ctx.pmu; 974 unsigned long flags; 975 976 /* not for SW PMU */ 977 if (pmu->task_ctx_nr == perf_sw_context) 978 return 0; 979 980 raw_spin_lock_irqsave(&cpuctx->hrtimer_lock, flags); 981 if (!cpuctx->hrtimer_active) { 982 cpuctx->hrtimer_active = 1; 983 hrtimer_forward_now(timer, cpuctx->hrtimer_interval); 984 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_PINNED); 985 } 986 raw_spin_unlock_irqrestore(&cpuctx->hrtimer_lock, flags); 987 988 return 0; 989 } 990 991 void perf_pmu_disable(struct pmu *pmu) 992 { 993 int *count = this_cpu_ptr(pmu->pmu_disable_count); 994 if (!(*count)++) 995 pmu->pmu_disable(pmu); 996 } 997 998 void perf_pmu_enable(struct pmu *pmu) 999 { 1000 int *count = this_cpu_ptr(pmu->pmu_disable_count); 1001 if (!--(*count)) 1002 pmu->pmu_enable(pmu); 1003 } 1004 1005 static DEFINE_PER_CPU(struct list_head, active_ctx_list); 1006 1007 /* 1008 * perf_event_ctx_activate(), perf_event_ctx_deactivate(), and 1009 * perf_event_task_tick() are fully serialized because they're strictly cpu 1010 * affine and perf_event_ctx{activate,deactivate} are called with IRQs 1011 * disabled, while perf_event_task_tick is called from IRQ context. 1012 */ 1013 static void perf_event_ctx_activate(struct perf_event_context *ctx) 1014 { 1015 struct list_head *head = this_cpu_ptr(&active_ctx_list); 1016 1017 WARN_ON(!irqs_disabled()); 1018 1019 WARN_ON(!list_empty(&ctx->active_ctx_list)); 1020 1021 list_add(&ctx->active_ctx_list, head); 1022 } 1023 1024 static void perf_event_ctx_deactivate(struct perf_event_context *ctx) 1025 { 1026 WARN_ON(!irqs_disabled()); 1027 1028 WARN_ON(list_empty(&ctx->active_ctx_list)); 1029 1030 list_del_init(&ctx->active_ctx_list); 1031 } 1032 1033 static void get_ctx(struct perf_event_context *ctx) 1034 { 1035 WARN_ON(!atomic_inc_not_zero(&ctx->refcount)); 1036 } 1037 1038 static void free_ctx(struct rcu_head *head) 1039 { 1040 struct perf_event_context *ctx; 1041 1042 ctx = container_of(head, struct perf_event_context, rcu_head); 1043 kfree(ctx->task_ctx_data); 1044 kfree(ctx); 1045 } 1046 1047 static void put_ctx(struct perf_event_context *ctx) 1048 { 1049 if (atomic_dec_and_test(&ctx->refcount)) { 1050 if (ctx->parent_ctx) 1051 put_ctx(ctx->parent_ctx); 1052 if (ctx->task && ctx->task != TASK_TOMBSTONE) 1053 put_task_struct(ctx->task); 1054 call_rcu(&ctx->rcu_head, free_ctx); 1055 } 1056 } 1057 1058 /* 1059 * Because of perf_event::ctx migration in sys_perf_event_open::move_group and 1060 * perf_pmu_migrate_context() we need some magic. 1061 * 1062 * Those places that change perf_event::ctx will hold both 1063 * perf_event_ctx::mutex of the 'old' and 'new' ctx value. 1064 * 1065 * Lock ordering is by mutex address. There are two other sites where 1066 * perf_event_context::mutex nests and those are: 1067 * 1068 * - perf_event_exit_task_context() [ child , 0 ] 1069 * perf_event_exit_event() 1070 * put_event() [ parent, 1 ] 1071 * 1072 * - perf_event_init_context() [ parent, 0 ] 1073 * inherit_task_group() 1074 * inherit_group() 1075 * inherit_event() 1076 * perf_event_alloc() 1077 * perf_init_event() 1078 * perf_try_init_event() [ child , 1 ] 1079 * 1080 * While it appears there is an obvious deadlock here -- the parent and child 1081 * nesting levels are inverted between the two. This is in fact safe because 1082 * life-time rules separate them. That is an exiting task cannot fork, and a 1083 * spawning task cannot (yet) exit. 1084 * 1085 * But remember that that these are parent<->child context relations, and 1086 * migration does not affect children, therefore these two orderings should not 1087 * interact. 1088 * 1089 * The change in perf_event::ctx does not affect children (as claimed above) 1090 * because the sys_perf_event_open() case will install a new event and break 1091 * the ctx parent<->child relation, and perf_pmu_migrate_context() is only 1092 * concerned with cpuctx and that doesn't have children. 1093 * 1094 * The places that change perf_event::ctx will issue: 1095 * 1096 * perf_remove_from_context(); 1097 * synchronize_rcu(); 1098 * perf_install_in_context(); 1099 * 1100 * to affect the change. The remove_from_context() + synchronize_rcu() should 1101 * quiesce the event, after which we can install it in the new location. This 1102 * means that only external vectors (perf_fops, prctl) can perturb the event 1103 * while in transit. Therefore all such accessors should also acquire 1104 * perf_event_context::mutex to serialize against this. 1105 * 1106 * However; because event->ctx can change while we're waiting to acquire 1107 * ctx->mutex we must be careful and use the below perf_event_ctx_lock() 1108 * function. 1109 * 1110 * Lock order: 1111 * cred_guard_mutex 1112 * task_struct::perf_event_mutex 1113 * perf_event_context::mutex 1114 * perf_event::child_mutex; 1115 * perf_event_context::lock 1116 * perf_event::mmap_mutex 1117 * mmap_sem 1118 */ 1119 static struct perf_event_context * 1120 perf_event_ctx_lock_nested(struct perf_event *event, int nesting) 1121 { 1122 struct perf_event_context *ctx; 1123 1124 again: 1125 rcu_read_lock(); 1126 ctx = ACCESS_ONCE(event->ctx); 1127 if (!atomic_inc_not_zero(&ctx->refcount)) { 1128 rcu_read_unlock(); 1129 goto again; 1130 } 1131 rcu_read_unlock(); 1132 1133 mutex_lock_nested(&ctx->mutex, nesting); 1134 if (event->ctx != ctx) { 1135 mutex_unlock(&ctx->mutex); 1136 put_ctx(ctx); 1137 goto again; 1138 } 1139 1140 return ctx; 1141 } 1142 1143 static inline struct perf_event_context * 1144 perf_event_ctx_lock(struct perf_event *event) 1145 { 1146 return perf_event_ctx_lock_nested(event, 0); 1147 } 1148 1149 static void perf_event_ctx_unlock(struct perf_event *event, 1150 struct perf_event_context *ctx) 1151 { 1152 mutex_unlock(&ctx->mutex); 1153 put_ctx(ctx); 1154 } 1155 1156 /* 1157 * This must be done under the ctx->lock, such as to serialize against 1158 * context_equiv(), therefore we cannot call put_ctx() since that might end up 1159 * calling scheduler related locks and ctx->lock nests inside those. 1160 */ 1161 static __must_check struct perf_event_context * 1162 unclone_ctx(struct perf_event_context *ctx) 1163 { 1164 struct perf_event_context *parent_ctx = ctx->parent_ctx; 1165 1166 lockdep_assert_held(&ctx->lock); 1167 1168 if (parent_ctx) 1169 ctx->parent_ctx = NULL; 1170 ctx->generation++; 1171 1172 return parent_ctx; 1173 } 1174 1175 static u32 perf_event_pid(struct perf_event *event, struct task_struct *p) 1176 { 1177 /* 1178 * only top level events have the pid namespace they were created in 1179 */ 1180 if (event->parent) 1181 event = event->parent; 1182 1183 return task_tgid_nr_ns(p, event->ns); 1184 } 1185 1186 static u32 perf_event_tid(struct perf_event *event, struct task_struct *p) 1187 { 1188 /* 1189 * only top level events have the pid namespace they were created in 1190 */ 1191 if (event->parent) 1192 event = event->parent; 1193 1194 return task_pid_nr_ns(p, event->ns); 1195 } 1196 1197 /* 1198 * If we inherit events we want to return the parent event id 1199 * to userspace. 1200 */ 1201 static u64 primary_event_id(struct perf_event *event) 1202 { 1203 u64 id = event->id; 1204 1205 if (event->parent) 1206 id = event->parent->id; 1207 1208 return id; 1209 } 1210 1211 /* 1212 * Get the perf_event_context for a task and lock it. 1213 * 1214 * This has to cope with with the fact that until it is locked, 1215 * the context could get moved to another task. 1216 */ 1217 static struct perf_event_context * 1218 perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags) 1219 { 1220 struct perf_event_context *ctx; 1221 1222 retry: 1223 /* 1224 * One of the few rules of preemptible RCU is that one cannot do 1225 * rcu_read_unlock() while holding a scheduler (or nested) lock when 1226 * part of the read side critical section was irqs-enabled -- see 1227 * rcu_read_unlock_special(). 1228 * 1229 * Since ctx->lock nests under rq->lock we must ensure the entire read 1230 * side critical section has interrupts disabled. 1231 */ 1232 local_irq_save(*flags); 1233 rcu_read_lock(); 1234 ctx = rcu_dereference(task->perf_event_ctxp[ctxn]); 1235 if (ctx) { 1236 /* 1237 * If this context is a clone of another, it might 1238 * get swapped for another underneath us by 1239 * perf_event_task_sched_out, though the 1240 * rcu_read_lock() protects us from any context 1241 * getting freed. Lock the context and check if it 1242 * got swapped before we could get the lock, and retry 1243 * if so. If we locked the right context, then it 1244 * can't get swapped on us any more. 1245 */ 1246 raw_spin_lock(&ctx->lock); 1247 if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) { 1248 raw_spin_unlock(&ctx->lock); 1249 rcu_read_unlock(); 1250 local_irq_restore(*flags); 1251 goto retry; 1252 } 1253 1254 if (ctx->task == TASK_TOMBSTONE || 1255 !atomic_inc_not_zero(&ctx->refcount)) { 1256 raw_spin_unlock(&ctx->lock); 1257 ctx = NULL; 1258 } else { 1259 WARN_ON_ONCE(ctx->task != task); 1260 } 1261 } 1262 rcu_read_unlock(); 1263 if (!ctx) 1264 local_irq_restore(*flags); 1265 return ctx; 1266 } 1267 1268 /* 1269 * Get the context for a task and increment its pin_count so it 1270 * can't get swapped to another task. This also increments its 1271 * reference count so that the context can't get freed. 1272 */ 1273 static struct perf_event_context * 1274 perf_pin_task_context(struct task_struct *task, int ctxn) 1275 { 1276 struct perf_event_context *ctx; 1277 unsigned long flags; 1278 1279 ctx = perf_lock_task_context(task, ctxn, &flags); 1280 if (ctx) { 1281 ++ctx->pin_count; 1282 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1283 } 1284 return ctx; 1285 } 1286 1287 static void perf_unpin_context(struct perf_event_context *ctx) 1288 { 1289 unsigned long flags; 1290 1291 raw_spin_lock_irqsave(&ctx->lock, flags); 1292 --ctx->pin_count; 1293 raw_spin_unlock_irqrestore(&ctx->lock, flags); 1294 } 1295 1296 /* 1297 * Update the record of the current time in a context. 1298 */ 1299 static void update_context_time(struct perf_event_context *ctx) 1300 { 1301 u64 now = perf_clock(); 1302 1303 ctx->time += now - ctx->timestamp; 1304 ctx->timestamp = now; 1305 } 1306 1307 static u64 perf_event_time(struct perf_event *event) 1308 { 1309 struct perf_event_context *ctx = event->ctx; 1310 1311 if (is_cgroup_event(event)) 1312 return perf_cgroup_event_time(event); 1313 1314 return ctx ? ctx->time : 0; 1315 } 1316 1317 /* 1318 * Update the total_time_enabled and total_time_running fields for a event. 1319 */ 1320 static void update_event_times(struct perf_event *event) 1321 { 1322 struct perf_event_context *ctx = event->ctx; 1323 u64 run_end; 1324 1325 lockdep_assert_held(&ctx->lock); 1326 1327 if (event->state < PERF_EVENT_STATE_INACTIVE || 1328 event->group_leader->state < PERF_EVENT_STATE_INACTIVE) 1329 return; 1330 1331 /* 1332 * in cgroup mode, time_enabled represents 1333 * the time the event was enabled AND active 1334 * tasks were in the monitored cgroup. This is 1335 * independent of the activity of the context as 1336 * there may be a mix of cgroup and non-cgroup events. 1337 * 1338 * That is why we treat cgroup events differently 1339 * here. 1340 */ 1341 if (is_cgroup_event(event)) 1342 run_end = perf_cgroup_event_time(event); 1343 else if (ctx->is_active) 1344 run_end = ctx->time; 1345 else 1346 run_end = event->tstamp_stopped; 1347 1348 event->total_time_enabled = run_end - event->tstamp_enabled; 1349 1350 if (event->state == PERF_EVENT_STATE_INACTIVE) 1351 run_end = event->tstamp_stopped; 1352 else 1353 run_end = perf_event_time(event); 1354 1355 event->total_time_running = run_end - event->tstamp_running; 1356 1357 } 1358 1359 /* 1360 * Update total_time_enabled and total_time_running for all events in a group. 1361 */ 1362 static void update_group_times(struct perf_event *leader) 1363 { 1364 struct perf_event *event; 1365 1366 update_event_times(leader); 1367 list_for_each_entry(event, &leader->sibling_list, group_entry) 1368 update_event_times(event); 1369 } 1370 1371 static struct list_head * 1372 ctx_group_list(struct perf_event *event, struct perf_event_context *ctx) 1373 { 1374 if (event->attr.pinned) 1375 return &ctx->pinned_groups; 1376 else 1377 return &ctx->flexible_groups; 1378 } 1379 1380 /* 1381 * Add a event from the lists for its context. 1382 * Must be called with ctx->mutex and ctx->lock held. 1383 */ 1384 static void 1385 list_add_event(struct perf_event *event, struct perf_event_context *ctx) 1386 { 1387 lockdep_assert_held(&ctx->lock); 1388 1389 WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT); 1390 event->attach_state |= PERF_ATTACH_CONTEXT; 1391 1392 /* 1393 * If we're a stand alone event or group leader, we go to the context 1394 * list, group events are kept attached to the group so that 1395 * perf_group_detach can, at all times, locate all siblings. 1396 */ 1397 if (event->group_leader == event) { 1398 struct list_head *list; 1399 1400 if (is_software_event(event)) 1401 event->group_flags |= PERF_GROUP_SOFTWARE; 1402 1403 list = ctx_group_list(event, ctx); 1404 list_add_tail(&event->group_entry, list); 1405 } 1406 1407 if (is_cgroup_event(event)) 1408 ctx->nr_cgroups++; 1409 1410 list_add_rcu(&event->event_entry, &ctx->event_list); 1411 ctx->nr_events++; 1412 if (event->attr.inherit_stat) 1413 ctx->nr_stat++; 1414 1415 ctx->generation++; 1416 } 1417 1418 /* 1419 * Initialize event state based on the perf_event_attr::disabled. 1420 */ 1421 static inline void perf_event__state_init(struct perf_event *event) 1422 { 1423 event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF : 1424 PERF_EVENT_STATE_INACTIVE; 1425 } 1426 1427 static void __perf_event_read_size(struct perf_event *event, int nr_siblings) 1428 { 1429 int entry = sizeof(u64); /* value */ 1430 int size = 0; 1431 int nr = 1; 1432 1433 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 1434 size += sizeof(u64); 1435 1436 if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 1437 size += sizeof(u64); 1438 1439 if (event->attr.read_format & PERF_FORMAT_ID) 1440 entry += sizeof(u64); 1441 1442 if (event->attr.read_format & PERF_FORMAT_GROUP) { 1443 nr += nr_siblings; 1444 size += sizeof(u64); 1445 } 1446 1447 size += entry * nr; 1448 event->read_size = size; 1449 } 1450 1451 static void __perf_event_header_size(struct perf_event *event, u64 sample_type) 1452 { 1453 struct perf_sample_data *data; 1454 u16 size = 0; 1455 1456 if (sample_type & PERF_SAMPLE_IP) 1457 size += sizeof(data->ip); 1458 1459 if (sample_type & PERF_SAMPLE_ADDR) 1460 size += sizeof(data->addr); 1461 1462 if (sample_type & PERF_SAMPLE_PERIOD) 1463 size += sizeof(data->period); 1464 1465 if (sample_type & PERF_SAMPLE_WEIGHT) 1466 size += sizeof(data->weight); 1467 1468 if (sample_type & PERF_SAMPLE_READ) 1469 size += event->read_size; 1470 1471 if (sample_type & PERF_SAMPLE_DATA_SRC) 1472 size += sizeof(data->data_src.val); 1473 1474 if (sample_type & PERF_SAMPLE_TRANSACTION) 1475 size += sizeof(data->txn); 1476 1477 event->header_size = size; 1478 } 1479 1480 /* 1481 * Called at perf_event creation and when events are attached/detached from a 1482 * group. 1483 */ 1484 static void perf_event__header_size(struct perf_event *event) 1485 { 1486 __perf_event_read_size(event, 1487 event->group_leader->nr_siblings); 1488 __perf_event_header_size(event, event->attr.sample_type); 1489 } 1490 1491 static void perf_event__id_header_size(struct perf_event *event) 1492 { 1493 struct perf_sample_data *data; 1494 u64 sample_type = event->attr.sample_type; 1495 u16 size = 0; 1496 1497 if (sample_type & PERF_SAMPLE_TID) 1498 size += sizeof(data->tid_entry); 1499 1500 if (sample_type & PERF_SAMPLE_TIME) 1501 size += sizeof(data->time); 1502 1503 if (sample_type & PERF_SAMPLE_IDENTIFIER) 1504 size += sizeof(data->id); 1505 1506 if (sample_type & PERF_SAMPLE_ID) 1507 size += sizeof(data->id); 1508 1509 if (sample_type & PERF_SAMPLE_STREAM_ID) 1510 size += sizeof(data->stream_id); 1511 1512 if (sample_type & PERF_SAMPLE_CPU) 1513 size += sizeof(data->cpu_entry); 1514 1515 event->id_header_size = size; 1516 } 1517 1518 static bool perf_event_validate_size(struct perf_event *event) 1519 { 1520 /* 1521 * The values computed here will be over-written when we actually 1522 * attach the event. 1523 */ 1524 __perf_event_read_size(event, event->group_leader->nr_siblings + 1); 1525 __perf_event_header_size(event, event->attr.sample_type & ~PERF_SAMPLE_READ); 1526 perf_event__id_header_size(event); 1527 1528 /* 1529 * Sum the lot; should not exceed the 64k limit we have on records. 1530 * Conservative limit to allow for callchains and other variable fields. 1531 */ 1532 if (event->read_size + event->header_size + 1533 event->id_header_size + sizeof(struct perf_event_header) >= 16*1024) 1534 return false; 1535 1536 return true; 1537 } 1538 1539 static void perf_group_attach(struct perf_event *event) 1540 { 1541 struct perf_event *group_leader = event->group_leader, *pos; 1542 1543 /* 1544 * We can have double attach due to group movement in perf_event_open. 1545 */ 1546 if (event->attach_state & PERF_ATTACH_GROUP) 1547 return; 1548 1549 event->attach_state |= PERF_ATTACH_GROUP; 1550 1551 if (group_leader == event) 1552 return; 1553 1554 WARN_ON_ONCE(group_leader->ctx != event->ctx); 1555 1556 if (group_leader->group_flags & PERF_GROUP_SOFTWARE && 1557 !is_software_event(event)) 1558 group_leader->group_flags &= ~PERF_GROUP_SOFTWARE; 1559 1560 list_add_tail(&event->group_entry, &group_leader->sibling_list); 1561 group_leader->nr_siblings++; 1562 1563 perf_event__header_size(group_leader); 1564 1565 list_for_each_entry(pos, &group_leader->sibling_list, group_entry) 1566 perf_event__header_size(pos); 1567 } 1568 1569 /* 1570 * Remove a event from the lists for its context. 1571 * Must be called with ctx->mutex and ctx->lock held. 1572 */ 1573 static void 1574 list_del_event(struct perf_event *event, struct perf_event_context *ctx) 1575 { 1576 struct perf_cpu_context *cpuctx; 1577 1578 WARN_ON_ONCE(event->ctx != ctx); 1579 lockdep_assert_held(&ctx->lock); 1580 1581 /* 1582 * We can have double detach due to exit/hot-unplug + close. 1583 */ 1584 if (!(event->attach_state & PERF_ATTACH_CONTEXT)) 1585 return; 1586 1587 event->attach_state &= ~PERF_ATTACH_CONTEXT; 1588 1589 if (is_cgroup_event(event)) { 1590 ctx->nr_cgroups--; 1591 /* 1592 * Because cgroup events are always per-cpu events, this will 1593 * always be called from the right CPU. 1594 */ 1595 cpuctx = __get_cpu_context(ctx); 1596 /* 1597 * If there are no more cgroup events then clear cgrp to avoid 1598 * stale pointer in update_cgrp_time_from_cpuctx(). 1599 */ 1600 if (!ctx->nr_cgroups) 1601 cpuctx->cgrp = NULL; 1602 } 1603 1604 ctx->nr_events--; 1605 if (event->attr.inherit_stat) 1606 ctx->nr_stat--; 1607 1608 list_del_rcu(&event->event_entry); 1609 1610 if (event->group_leader == event) 1611 list_del_init(&event->group_entry); 1612 1613 update_group_times(event); 1614 1615 /* 1616 * If event was in error state, then keep it 1617 * that way, otherwise bogus counts will be 1618 * returned on read(). The only way to get out 1619 * of error state is by explicit re-enabling 1620 * of the event 1621 */ 1622 if (event->state > PERF_EVENT_STATE_OFF) 1623 event->state = PERF_EVENT_STATE_OFF; 1624 1625 ctx->generation++; 1626 } 1627 1628 static void perf_group_detach(struct perf_event *event) 1629 { 1630 struct perf_event *sibling, *tmp; 1631 struct list_head *list = NULL; 1632 1633 /* 1634 * We can have double detach due to exit/hot-unplug + close. 1635 */ 1636 if (!(event->attach_state & PERF_ATTACH_GROUP)) 1637 return; 1638 1639 event->attach_state &= ~PERF_ATTACH_GROUP; 1640 1641 /* 1642 * If this is a sibling, remove it from its group. 1643 */ 1644 if (event->group_leader != event) { 1645 list_del_init(&event->group_entry); 1646 event->group_leader->nr_siblings--; 1647 goto out; 1648 } 1649 1650 if (!list_empty(&event->group_entry)) 1651 list = &event->group_entry; 1652 1653 /* 1654 * If this was a group event with sibling events then 1655 * upgrade the siblings to singleton events by adding them 1656 * to whatever list we are on. 1657 */ 1658 list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) { 1659 if (list) 1660 list_move_tail(&sibling->group_entry, list); 1661 sibling->group_leader = sibling; 1662 1663 /* Inherit group flags from the previous leader */ 1664 sibling->group_flags = event->group_flags; 1665 1666 WARN_ON_ONCE(sibling->ctx != event->ctx); 1667 } 1668 1669 out: 1670 perf_event__header_size(event->group_leader); 1671 1672 list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry) 1673 perf_event__header_size(tmp); 1674 } 1675 1676 static bool is_orphaned_event(struct perf_event *event) 1677 { 1678 return event->state == PERF_EVENT_STATE_DEAD; 1679 } 1680 1681 static inline int pmu_filter_match(struct perf_event *event) 1682 { 1683 struct pmu *pmu = event->pmu; 1684 return pmu->filter_match ? pmu->filter_match(event) : 1; 1685 } 1686 1687 static inline int 1688 event_filter_match(struct perf_event *event) 1689 { 1690 return (event->cpu == -1 || event->cpu == smp_processor_id()) 1691 && perf_cgroup_match(event) && pmu_filter_match(event); 1692 } 1693 1694 static void 1695 event_sched_out(struct perf_event *event, 1696 struct perf_cpu_context *cpuctx, 1697 struct perf_event_context *ctx) 1698 { 1699 u64 tstamp = perf_event_time(event); 1700 u64 delta; 1701 1702 WARN_ON_ONCE(event->ctx != ctx); 1703 lockdep_assert_held(&ctx->lock); 1704 1705 /* 1706 * An event which could not be activated because of 1707 * filter mismatch still needs to have its timings 1708 * maintained, otherwise bogus information is return 1709 * via read() for time_enabled, time_running: 1710 */ 1711 if (event->state == PERF_EVENT_STATE_INACTIVE 1712 && !event_filter_match(event)) { 1713 delta = tstamp - event->tstamp_stopped; 1714 event->tstamp_running += delta; 1715 event->tstamp_stopped = tstamp; 1716 } 1717 1718 if (event->state != PERF_EVENT_STATE_ACTIVE) 1719 return; 1720 1721 perf_pmu_disable(event->pmu); 1722 1723 event->tstamp_stopped = tstamp; 1724 event->pmu->del(event, 0); 1725 event->oncpu = -1; 1726 event->state = PERF_EVENT_STATE_INACTIVE; 1727 if (event->pending_disable) { 1728 event->pending_disable = 0; 1729 event->state = PERF_EVENT_STATE_OFF; 1730 } 1731 1732 if (!is_software_event(event)) 1733 cpuctx->active_oncpu--; 1734 if (!--ctx->nr_active) 1735 perf_event_ctx_deactivate(ctx); 1736 if (event->attr.freq && event->attr.sample_freq) 1737 ctx->nr_freq--; 1738 if (event->attr.exclusive || !cpuctx->active_oncpu) 1739 cpuctx->exclusive = 0; 1740 1741 perf_pmu_enable(event->pmu); 1742 } 1743 1744 static void 1745 group_sched_out(struct perf_event *group_event, 1746 struct perf_cpu_context *cpuctx, 1747 struct perf_event_context *ctx) 1748 { 1749 struct perf_event *event; 1750 int state = group_event->state; 1751 1752 event_sched_out(group_event, cpuctx, ctx); 1753 1754 /* 1755 * Schedule out siblings (if any): 1756 */ 1757 list_for_each_entry(event, &group_event->sibling_list, group_entry) 1758 event_sched_out(event, cpuctx, ctx); 1759 1760 if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive) 1761 cpuctx->exclusive = 0; 1762 } 1763 1764 #define DETACH_GROUP 0x01UL 1765 1766 /* 1767 * Cross CPU call to remove a performance event 1768 * 1769 * We disable the event on the hardware level first. After that we 1770 * remove it from the context list. 1771 */ 1772 static void 1773 __perf_remove_from_context(struct perf_event *event, 1774 struct perf_cpu_context *cpuctx, 1775 struct perf_event_context *ctx, 1776 void *info) 1777 { 1778 unsigned long flags = (unsigned long)info; 1779 1780 event_sched_out(event, cpuctx, ctx); 1781 if (flags & DETACH_GROUP) 1782 perf_group_detach(event); 1783 list_del_event(event, ctx); 1784 1785 if (!ctx->nr_events && ctx->is_active) { 1786 ctx->is_active = 0; 1787 if (ctx->task) { 1788 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 1789 cpuctx->task_ctx = NULL; 1790 } 1791 } 1792 } 1793 1794 /* 1795 * Remove the event from a task's (or a CPU's) list of events. 1796 * 1797 * If event->ctx is a cloned context, callers must make sure that 1798 * every task struct that event->ctx->task could possibly point to 1799 * remains valid. This is OK when called from perf_release since 1800 * that only calls us on the top-level context, which can't be a clone. 1801 * When called from perf_event_exit_task, it's OK because the 1802 * context has been detached from its task. 1803 */ 1804 static void perf_remove_from_context(struct perf_event *event, unsigned long flags) 1805 { 1806 lockdep_assert_held(&event->ctx->mutex); 1807 1808 event_function_call(event, __perf_remove_from_context, (void *)flags); 1809 } 1810 1811 /* 1812 * Cross CPU call to disable a performance event 1813 */ 1814 static void __perf_event_disable(struct perf_event *event, 1815 struct perf_cpu_context *cpuctx, 1816 struct perf_event_context *ctx, 1817 void *info) 1818 { 1819 if (event->state < PERF_EVENT_STATE_INACTIVE) 1820 return; 1821 1822 update_context_time(ctx); 1823 update_cgrp_time_from_event(event); 1824 update_group_times(event); 1825 if (event == event->group_leader) 1826 group_sched_out(event, cpuctx, ctx); 1827 else 1828 event_sched_out(event, cpuctx, ctx); 1829 event->state = PERF_EVENT_STATE_OFF; 1830 } 1831 1832 /* 1833 * Disable a event. 1834 * 1835 * If event->ctx is a cloned context, callers must make sure that 1836 * every task struct that event->ctx->task could possibly point to 1837 * remains valid. This condition is satisifed when called through 1838 * perf_event_for_each_child or perf_event_for_each because they 1839 * hold the top-level event's child_mutex, so any descendant that 1840 * goes to exit will block in perf_event_exit_event(). 1841 * 1842 * When called from perf_pending_event it's OK because event->ctx 1843 * is the current context on this CPU and preemption is disabled, 1844 * hence we can't get into perf_event_task_sched_out for this context. 1845 */ 1846 static void _perf_event_disable(struct perf_event *event) 1847 { 1848 struct perf_event_context *ctx = event->ctx; 1849 1850 raw_spin_lock_irq(&ctx->lock); 1851 if (event->state <= PERF_EVENT_STATE_OFF) { 1852 raw_spin_unlock_irq(&ctx->lock); 1853 return; 1854 } 1855 raw_spin_unlock_irq(&ctx->lock); 1856 1857 event_function_call(event, __perf_event_disable, NULL); 1858 } 1859 1860 void perf_event_disable_local(struct perf_event *event) 1861 { 1862 event_function_local(event, __perf_event_disable, NULL); 1863 } 1864 1865 /* 1866 * Strictly speaking kernel users cannot create groups and therefore this 1867 * interface does not need the perf_event_ctx_lock() magic. 1868 */ 1869 void perf_event_disable(struct perf_event *event) 1870 { 1871 struct perf_event_context *ctx; 1872 1873 ctx = perf_event_ctx_lock(event); 1874 _perf_event_disable(event); 1875 perf_event_ctx_unlock(event, ctx); 1876 } 1877 EXPORT_SYMBOL_GPL(perf_event_disable); 1878 1879 static void perf_set_shadow_time(struct perf_event *event, 1880 struct perf_event_context *ctx, 1881 u64 tstamp) 1882 { 1883 /* 1884 * use the correct time source for the time snapshot 1885 * 1886 * We could get by without this by leveraging the 1887 * fact that to get to this function, the caller 1888 * has most likely already called update_context_time() 1889 * and update_cgrp_time_xx() and thus both timestamp 1890 * are identical (or very close). Given that tstamp is, 1891 * already adjusted for cgroup, we could say that: 1892 * tstamp - ctx->timestamp 1893 * is equivalent to 1894 * tstamp - cgrp->timestamp. 1895 * 1896 * Then, in perf_output_read(), the calculation would 1897 * work with no changes because: 1898 * - event is guaranteed scheduled in 1899 * - no scheduled out in between 1900 * - thus the timestamp would be the same 1901 * 1902 * But this is a bit hairy. 1903 * 1904 * So instead, we have an explicit cgroup call to remain 1905 * within the time time source all along. We believe it 1906 * is cleaner and simpler to understand. 1907 */ 1908 if (is_cgroup_event(event)) 1909 perf_cgroup_set_shadow_time(event, tstamp); 1910 else 1911 event->shadow_ctx_time = tstamp - ctx->timestamp; 1912 } 1913 1914 #define MAX_INTERRUPTS (~0ULL) 1915 1916 static void perf_log_throttle(struct perf_event *event, int enable); 1917 static void perf_log_itrace_start(struct perf_event *event); 1918 1919 static int 1920 event_sched_in(struct perf_event *event, 1921 struct perf_cpu_context *cpuctx, 1922 struct perf_event_context *ctx) 1923 { 1924 u64 tstamp = perf_event_time(event); 1925 int ret = 0; 1926 1927 lockdep_assert_held(&ctx->lock); 1928 1929 if (event->state <= PERF_EVENT_STATE_OFF) 1930 return 0; 1931 1932 WRITE_ONCE(event->oncpu, smp_processor_id()); 1933 /* 1934 * Order event::oncpu write to happen before the ACTIVE state 1935 * is visible. 1936 */ 1937 smp_wmb(); 1938 WRITE_ONCE(event->state, PERF_EVENT_STATE_ACTIVE); 1939 1940 /* 1941 * Unthrottle events, since we scheduled we might have missed several 1942 * ticks already, also for a heavily scheduling task there is little 1943 * guarantee it'll get a tick in a timely manner. 1944 */ 1945 if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) { 1946 perf_log_throttle(event, 1); 1947 event->hw.interrupts = 0; 1948 } 1949 1950 /* 1951 * The new state must be visible before we turn it on in the hardware: 1952 */ 1953 smp_wmb(); 1954 1955 perf_pmu_disable(event->pmu); 1956 1957 perf_set_shadow_time(event, ctx, tstamp); 1958 1959 perf_log_itrace_start(event); 1960 1961 if (event->pmu->add(event, PERF_EF_START)) { 1962 event->state = PERF_EVENT_STATE_INACTIVE; 1963 event->oncpu = -1; 1964 ret = -EAGAIN; 1965 goto out; 1966 } 1967 1968 event->tstamp_running += tstamp - event->tstamp_stopped; 1969 1970 if (!is_software_event(event)) 1971 cpuctx->active_oncpu++; 1972 if (!ctx->nr_active++) 1973 perf_event_ctx_activate(ctx); 1974 if (event->attr.freq && event->attr.sample_freq) 1975 ctx->nr_freq++; 1976 1977 if (event->attr.exclusive) 1978 cpuctx->exclusive = 1; 1979 1980 out: 1981 perf_pmu_enable(event->pmu); 1982 1983 return ret; 1984 } 1985 1986 static int 1987 group_sched_in(struct perf_event *group_event, 1988 struct perf_cpu_context *cpuctx, 1989 struct perf_event_context *ctx) 1990 { 1991 struct perf_event *event, *partial_group = NULL; 1992 struct pmu *pmu = ctx->pmu; 1993 u64 now = ctx->time; 1994 bool simulate = false; 1995 1996 if (group_event->state == PERF_EVENT_STATE_OFF) 1997 return 0; 1998 1999 pmu->start_txn(pmu, PERF_PMU_TXN_ADD); 2000 2001 if (event_sched_in(group_event, cpuctx, ctx)) { 2002 pmu->cancel_txn(pmu); 2003 perf_mux_hrtimer_restart(cpuctx); 2004 return -EAGAIN; 2005 } 2006 2007 /* 2008 * Schedule in siblings as one group (if any): 2009 */ 2010 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 2011 if (event_sched_in(event, cpuctx, ctx)) { 2012 partial_group = event; 2013 goto group_error; 2014 } 2015 } 2016 2017 if (!pmu->commit_txn(pmu)) 2018 return 0; 2019 2020 group_error: 2021 /* 2022 * Groups can be scheduled in as one unit only, so undo any 2023 * partial group before returning: 2024 * The events up to the failed event are scheduled out normally, 2025 * tstamp_stopped will be updated. 2026 * 2027 * The failed events and the remaining siblings need to have 2028 * their timings updated as if they had gone thru event_sched_in() 2029 * and event_sched_out(). This is required to get consistent timings 2030 * across the group. This also takes care of the case where the group 2031 * could never be scheduled by ensuring tstamp_stopped is set to mark 2032 * the time the event was actually stopped, such that time delta 2033 * calculation in update_event_times() is correct. 2034 */ 2035 list_for_each_entry(event, &group_event->sibling_list, group_entry) { 2036 if (event == partial_group) 2037 simulate = true; 2038 2039 if (simulate) { 2040 event->tstamp_running += now - event->tstamp_stopped; 2041 event->tstamp_stopped = now; 2042 } else { 2043 event_sched_out(event, cpuctx, ctx); 2044 } 2045 } 2046 event_sched_out(group_event, cpuctx, ctx); 2047 2048 pmu->cancel_txn(pmu); 2049 2050 perf_mux_hrtimer_restart(cpuctx); 2051 2052 return -EAGAIN; 2053 } 2054 2055 /* 2056 * Work out whether we can put this event group on the CPU now. 2057 */ 2058 static int group_can_go_on(struct perf_event *event, 2059 struct perf_cpu_context *cpuctx, 2060 int can_add_hw) 2061 { 2062 /* 2063 * Groups consisting entirely of software events can always go on. 2064 */ 2065 if (event->group_flags & PERF_GROUP_SOFTWARE) 2066 return 1; 2067 /* 2068 * If an exclusive group is already on, no other hardware 2069 * events can go on. 2070 */ 2071 if (cpuctx->exclusive) 2072 return 0; 2073 /* 2074 * If this group is exclusive and there are already 2075 * events on the CPU, it can't go on. 2076 */ 2077 if (event->attr.exclusive && cpuctx->active_oncpu) 2078 return 0; 2079 /* 2080 * Otherwise, try to add it if all previous groups were able 2081 * to go on. 2082 */ 2083 return can_add_hw; 2084 } 2085 2086 static void add_event_to_ctx(struct perf_event *event, 2087 struct perf_event_context *ctx) 2088 { 2089 u64 tstamp = perf_event_time(event); 2090 2091 list_add_event(event, ctx); 2092 perf_group_attach(event); 2093 event->tstamp_enabled = tstamp; 2094 event->tstamp_running = tstamp; 2095 event->tstamp_stopped = tstamp; 2096 } 2097 2098 static void ctx_sched_out(struct perf_event_context *ctx, 2099 struct perf_cpu_context *cpuctx, 2100 enum event_type_t event_type); 2101 static void 2102 ctx_sched_in(struct perf_event_context *ctx, 2103 struct perf_cpu_context *cpuctx, 2104 enum event_type_t event_type, 2105 struct task_struct *task); 2106 2107 static void task_ctx_sched_out(struct perf_cpu_context *cpuctx, 2108 struct perf_event_context *ctx) 2109 { 2110 if (!cpuctx->task_ctx) 2111 return; 2112 2113 if (WARN_ON_ONCE(ctx != cpuctx->task_ctx)) 2114 return; 2115 2116 ctx_sched_out(ctx, cpuctx, EVENT_ALL); 2117 } 2118 2119 static void perf_event_sched_in(struct perf_cpu_context *cpuctx, 2120 struct perf_event_context *ctx, 2121 struct task_struct *task) 2122 { 2123 cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task); 2124 if (ctx) 2125 ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task); 2126 cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task); 2127 if (ctx) 2128 ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task); 2129 } 2130 2131 static void ctx_resched(struct perf_cpu_context *cpuctx, 2132 struct perf_event_context *task_ctx) 2133 { 2134 perf_pmu_disable(cpuctx->ctx.pmu); 2135 if (task_ctx) 2136 task_ctx_sched_out(cpuctx, task_ctx); 2137 cpu_ctx_sched_out(cpuctx, EVENT_ALL); 2138 perf_event_sched_in(cpuctx, task_ctx, current); 2139 perf_pmu_enable(cpuctx->ctx.pmu); 2140 } 2141 2142 /* 2143 * Cross CPU call to install and enable a performance event 2144 * 2145 * Very similar to remote_function() + event_function() but cannot assume that 2146 * things like ctx->is_active and cpuctx->task_ctx are set. 2147 */ 2148 static int __perf_install_in_context(void *info) 2149 { 2150 struct perf_event *event = info; 2151 struct perf_event_context *ctx = event->ctx; 2152 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 2153 struct perf_event_context *task_ctx = cpuctx->task_ctx; 2154 bool activate = true; 2155 int ret = 0; 2156 2157 raw_spin_lock(&cpuctx->ctx.lock); 2158 if (ctx->task) { 2159 raw_spin_lock(&ctx->lock); 2160 task_ctx = ctx; 2161 2162 /* If we're on the wrong CPU, try again */ 2163 if (task_cpu(ctx->task) != smp_processor_id()) { 2164 ret = -ESRCH; 2165 goto unlock; 2166 } 2167 2168 /* 2169 * If we're on the right CPU, see if the task we target is 2170 * current, if not we don't have to activate the ctx, a future 2171 * context switch will do that for us. 2172 */ 2173 if (ctx->task != current) 2174 activate = false; 2175 else 2176 WARN_ON_ONCE(cpuctx->task_ctx && cpuctx->task_ctx != ctx); 2177 2178 } else if (task_ctx) { 2179 raw_spin_lock(&task_ctx->lock); 2180 } 2181 2182 if (activate) { 2183 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 2184 add_event_to_ctx(event, ctx); 2185 ctx_resched(cpuctx, task_ctx); 2186 } else { 2187 add_event_to_ctx(event, ctx); 2188 } 2189 2190 unlock: 2191 perf_ctx_unlock(cpuctx, task_ctx); 2192 2193 return ret; 2194 } 2195 2196 /* 2197 * Attach a performance event to a context. 2198 * 2199 * Very similar to event_function_call, see comment there. 2200 */ 2201 static void 2202 perf_install_in_context(struct perf_event_context *ctx, 2203 struct perf_event *event, 2204 int cpu) 2205 { 2206 struct task_struct *task = READ_ONCE(ctx->task); 2207 2208 lockdep_assert_held(&ctx->mutex); 2209 2210 event->ctx = ctx; 2211 if (event->cpu != -1) 2212 event->cpu = cpu; 2213 2214 if (!task) { 2215 cpu_function_call(cpu, __perf_install_in_context, event); 2216 return; 2217 } 2218 2219 /* 2220 * Should not happen, we validate the ctx is still alive before calling. 2221 */ 2222 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) 2223 return; 2224 2225 /* 2226 * Installing events is tricky because we cannot rely on ctx->is_active 2227 * to be set in case this is the nr_events 0 -> 1 transition. 2228 */ 2229 again: 2230 /* 2231 * Cannot use task_function_call() because we need to run on the task's 2232 * CPU regardless of whether its current or not. 2233 */ 2234 if (!cpu_function_call(task_cpu(task), __perf_install_in_context, event)) 2235 return; 2236 2237 raw_spin_lock_irq(&ctx->lock); 2238 task = ctx->task; 2239 if (WARN_ON_ONCE(task == TASK_TOMBSTONE)) { 2240 /* 2241 * Cannot happen because we already checked above (which also 2242 * cannot happen), and we hold ctx->mutex, which serializes us 2243 * against perf_event_exit_task_context(). 2244 */ 2245 raw_spin_unlock_irq(&ctx->lock); 2246 return; 2247 } 2248 raw_spin_unlock_irq(&ctx->lock); 2249 /* 2250 * Since !ctx->is_active doesn't mean anything, we must IPI 2251 * unconditionally. 2252 */ 2253 goto again; 2254 } 2255 2256 /* 2257 * Put a event into inactive state and update time fields. 2258 * Enabling the leader of a group effectively enables all 2259 * the group members that aren't explicitly disabled, so we 2260 * have to update their ->tstamp_enabled also. 2261 * Note: this works for group members as well as group leaders 2262 * since the non-leader members' sibling_lists will be empty. 2263 */ 2264 static void __perf_event_mark_enabled(struct perf_event *event) 2265 { 2266 struct perf_event *sub; 2267 u64 tstamp = perf_event_time(event); 2268 2269 event->state = PERF_EVENT_STATE_INACTIVE; 2270 event->tstamp_enabled = tstamp - event->total_time_enabled; 2271 list_for_each_entry(sub, &event->sibling_list, group_entry) { 2272 if (sub->state >= PERF_EVENT_STATE_INACTIVE) 2273 sub->tstamp_enabled = tstamp - sub->total_time_enabled; 2274 } 2275 } 2276 2277 /* 2278 * Cross CPU call to enable a performance event 2279 */ 2280 static void __perf_event_enable(struct perf_event *event, 2281 struct perf_cpu_context *cpuctx, 2282 struct perf_event_context *ctx, 2283 void *info) 2284 { 2285 struct perf_event *leader = event->group_leader; 2286 struct perf_event_context *task_ctx; 2287 2288 if (event->state >= PERF_EVENT_STATE_INACTIVE || 2289 event->state <= PERF_EVENT_STATE_ERROR) 2290 return; 2291 2292 if (ctx->is_active) 2293 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 2294 2295 __perf_event_mark_enabled(event); 2296 2297 if (!ctx->is_active) 2298 return; 2299 2300 if (!event_filter_match(event)) { 2301 if (is_cgroup_event(event)) 2302 perf_cgroup_defer_enabled(event); 2303 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 2304 return; 2305 } 2306 2307 /* 2308 * If the event is in a group and isn't the group leader, 2309 * then don't put it on unless the group is on. 2310 */ 2311 if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) { 2312 ctx_sched_in(ctx, cpuctx, EVENT_TIME, current); 2313 return; 2314 } 2315 2316 task_ctx = cpuctx->task_ctx; 2317 if (ctx->task) 2318 WARN_ON_ONCE(task_ctx != ctx); 2319 2320 ctx_resched(cpuctx, task_ctx); 2321 } 2322 2323 /* 2324 * Enable a event. 2325 * 2326 * If event->ctx is a cloned context, callers must make sure that 2327 * every task struct that event->ctx->task could possibly point to 2328 * remains valid. This condition is satisfied when called through 2329 * perf_event_for_each_child or perf_event_for_each as described 2330 * for perf_event_disable. 2331 */ 2332 static void _perf_event_enable(struct perf_event *event) 2333 { 2334 struct perf_event_context *ctx = event->ctx; 2335 2336 raw_spin_lock_irq(&ctx->lock); 2337 if (event->state >= PERF_EVENT_STATE_INACTIVE || 2338 event->state < PERF_EVENT_STATE_ERROR) { 2339 raw_spin_unlock_irq(&ctx->lock); 2340 return; 2341 } 2342 2343 /* 2344 * If the event is in error state, clear that first. 2345 * 2346 * That way, if we see the event in error state below, we know that it 2347 * has gone back into error state, as distinct from the task having 2348 * been scheduled away before the cross-call arrived. 2349 */ 2350 if (event->state == PERF_EVENT_STATE_ERROR) 2351 event->state = PERF_EVENT_STATE_OFF; 2352 raw_spin_unlock_irq(&ctx->lock); 2353 2354 event_function_call(event, __perf_event_enable, NULL); 2355 } 2356 2357 /* 2358 * See perf_event_disable(); 2359 */ 2360 void perf_event_enable(struct perf_event *event) 2361 { 2362 struct perf_event_context *ctx; 2363 2364 ctx = perf_event_ctx_lock(event); 2365 _perf_event_enable(event); 2366 perf_event_ctx_unlock(event, ctx); 2367 } 2368 EXPORT_SYMBOL_GPL(perf_event_enable); 2369 2370 struct stop_event_data { 2371 struct perf_event *event; 2372 unsigned int restart; 2373 }; 2374 2375 static int __perf_event_stop(void *info) 2376 { 2377 struct stop_event_data *sd = info; 2378 struct perf_event *event = sd->event; 2379 2380 /* if it's already INACTIVE, do nothing */ 2381 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 2382 return 0; 2383 2384 /* matches smp_wmb() in event_sched_in() */ 2385 smp_rmb(); 2386 2387 /* 2388 * There is a window with interrupts enabled before we get here, 2389 * so we need to check again lest we try to stop another CPU's event. 2390 */ 2391 if (READ_ONCE(event->oncpu) != smp_processor_id()) 2392 return -EAGAIN; 2393 2394 event->pmu->stop(event, PERF_EF_UPDATE); 2395 2396 /* 2397 * May race with the actual stop (through perf_pmu_output_stop()), 2398 * but it is only used for events with AUX ring buffer, and such 2399 * events will refuse to restart because of rb::aux_mmap_count==0, 2400 * see comments in perf_aux_output_begin(). 2401 * 2402 * Since this is happening on a event-local CPU, no trace is lost 2403 * while restarting. 2404 */ 2405 if (sd->restart) 2406 event->pmu->start(event, PERF_EF_START); 2407 2408 return 0; 2409 } 2410 2411 static int perf_event_restart(struct perf_event *event) 2412 { 2413 struct stop_event_data sd = { 2414 .event = event, 2415 .restart = 1, 2416 }; 2417 int ret = 0; 2418 2419 do { 2420 if (READ_ONCE(event->state) != PERF_EVENT_STATE_ACTIVE) 2421 return 0; 2422 2423 /* matches smp_wmb() in event_sched_in() */ 2424 smp_rmb(); 2425 2426 /* 2427 * We only want to restart ACTIVE events, so if the event goes 2428 * inactive here (event->oncpu==-1), there's nothing more to do; 2429 * fall through with ret==-ENXIO. 2430 */ 2431 ret = cpu_function_call(READ_ONCE(event->oncpu), 2432 __perf_event_stop, &sd); 2433 } while (ret == -EAGAIN); 2434 2435 return ret; 2436 } 2437 2438 /* 2439 * In order to contain the amount of racy and tricky in the address filter 2440 * configuration management, it is a two part process: 2441 * 2442 * (p1) when userspace mappings change as a result of (1) or (2) or (3) below, 2443 * we update the addresses of corresponding vmas in 2444 * event::addr_filters_offs array and bump the event::addr_filters_gen; 2445 * (p2) when an event is scheduled in (pmu::add), it calls 2446 * perf_event_addr_filters_sync() which calls pmu::addr_filters_sync() 2447 * if the generation has changed since the previous call. 2448 * 2449 * If (p1) happens while the event is active, we restart it to force (p2). 2450 * 2451 * (1) perf_addr_filters_apply(): adjusting filters' offsets based on 2452 * pre-existing mappings, called once when new filters arrive via SET_FILTER 2453 * ioctl; 2454 * (2) perf_addr_filters_adjust(): adjusting filters' offsets based on newly 2455 * registered mapping, called for every new mmap(), with mm::mmap_sem down 2456 * for reading; 2457 * (3) perf_event_addr_filters_exec(): clearing filters' offsets in the process 2458 * of exec. 2459 */ 2460 void perf_event_addr_filters_sync(struct perf_event *event) 2461 { 2462 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 2463 2464 if (!has_addr_filter(event)) 2465 return; 2466 2467 raw_spin_lock(&ifh->lock); 2468 if (event->addr_filters_gen != event->hw.addr_filters_gen) { 2469 event->pmu->addr_filters_sync(event); 2470 event->hw.addr_filters_gen = event->addr_filters_gen; 2471 } 2472 raw_spin_unlock(&ifh->lock); 2473 } 2474 EXPORT_SYMBOL_GPL(perf_event_addr_filters_sync); 2475 2476 static int _perf_event_refresh(struct perf_event *event, int refresh) 2477 { 2478 /* 2479 * not supported on inherited events 2480 */ 2481 if (event->attr.inherit || !is_sampling_event(event)) 2482 return -EINVAL; 2483 2484 atomic_add(refresh, &event->event_limit); 2485 _perf_event_enable(event); 2486 2487 return 0; 2488 } 2489 2490 /* 2491 * See perf_event_disable() 2492 */ 2493 int perf_event_refresh(struct perf_event *event, int refresh) 2494 { 2495 struct perf_event_context *ctx; 2496 int ret; 2497 2498 ctx = perf_event_ctx_lock(event); 2499 ret = _perf_event_refresh(event, refresh); 2500 perf_event_ctx_unlock(event, ctx); 2501 2502 return ret; 2503 } 2504 EXPORT_SYMBOL_GPL(perf_event_refresh); 2505 2506 static void ctx_sched_out(struct perf_event_context *ctx, 2507 struct perf_cpu_context *cpuctx, 2508 enum event_type_t event_type) 2509 { 2510 int is_active = ctx->is_active; 2511 struct perf_event *event; 2512 2513 lockdep_assert_held(&ctx->lock); 2514 2515 if (likely(!ctx->nr_events)) { 2516 /* 2517 * See __perf_remove_from_context(). 2518 */ 2519 WARN_ON_ONCE(ctx->is_active); 2520 if (ctx->task) 2521 WARN_ON_ONCE(cpuctx->task_ctx); 2522 return; 2523 } 2524 2525 ctx->is_active &= ~event_type; 2526 if (!(ctx->is_active & EVENT_ALL)) 2527 ctx->is_active = 0; 2528 2529 if (ctx->task) { 2530 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 2531 if (!ctx->is_active) 2532 cpuctx->task_ctx = NULL; 2533 } 2534 2535 /* 2536 * Always update time if it was set; not only when it changes. 2537 * Otherwise we can 'forget' to update time for any but the last 2538 * context we sched out. For example: 2539 * 2540 * ctx_sched_out(.event_type = EVENT_FLEXIBLE) 2541 * ctx_sched_out(.event_type = EVENT_PINNED) 2542 * 2543 * would only update time for the pinned events. 2544 */ 2545 if (is_active & EVENT_TIME) { 2546 /* update (and stop) ctx time */ 2547 update_context_time(ctx); 2548 update_cgrp_time_from_cpuctx(cpuctx); 2549 } 2550 2551 is_active ^= ctx->is_active; /* changed bits */ 2552 2553 if (!ctx->nr_active || !(is_active & EVENT_ALL)) 2554 return; 2555 2556 perf_pmu_disable(ctx->pmu); 2557 if (is_active & EVENT_PINNED) { 2558 list_for_each_entry(event, &ctx->pinned_groups, group_entry) 2559 group_sched_out(event, cpuctx, ctx); 2560 } 2561 2562 if (is_active & EVENT_FLEXIBLE) { 2563 list_for_each_entry(event, &ctx->flexible_groups, group_entry) 2564 group_sched_out(event, cpuctx, ctx); 2565 } 2566 perf_pmu_enable(ctx->pmu); 2567 } 2568 2569 /* 2570 * Test whether two contexts are equivalent, i.e. whether they have both been 2571 * cloned from the same version of the same context. 2572 * 2573 * Equivalence is measured using a generation number in the context that is 2574 * incremented on each modification to it; see unclone_ctx(), list_add_event() 2575 * and list_del_event(). 2576 */ 2577 static int context_equiv(struct perf_event_context *ctx1, 2578 struct perf_event_context *ctx2) 2579 { 2580 lockdep_assert_held(&ctx1->lock); 2581 lockdep_assert_held(&ctx2->lock); 2582 2583 /* Pinning disables the swap optimization */ 2584 if (ctx1->pin_count || ctx2->pin_count) 2585 return 0; 2586 2587 /* If ctx1 is the parent of ctx2 */ 2588 if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen) 2589 return 1; 2590 2591 /* If ctx2 is the parent of ctx1 */ 2592 if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation) 2593 return 1; 2594 2595 /* 2596 * If ctx1 and ctx2 have the same parent; we flatten the parent 2597 * hierarchy, see perf_event_init_context(). 2598 */ 2599 if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx && 2600 ctx1->parent_gen == ctx2->parent_gen) 2601 return 1; 2602 2603 /* Unmatched */ 2604 return 0; 2605 } 2606 2607 static void __perf_event_sync_stat(struct perf_event *event, 2608 struct perf_event *next_event) 2609 { 2610 u64 value; 2611 2612 if (!event->attr.inherit_stat) 2613 return; 2614 2615 /* 2616 * Update the event value, we cannot use perf_event_read() 2617 * because we're in the middle of a context switch and have IRQs 2618 * disabled, which upsets smp_call_function_single(), however 2619 * we know the event must be on the current CPU, therefore we 2620 * don't need to use it. 2621 */ 2622 switch (event->state) { 2623 case PERF_EVENT_STATE_ACTIVE: 2624 event->pmu->read(event); 2625 /* fall-through */ 2626 2627 case PERF_EVENT_STATE_INACTIVE: 2628 update_event_times(event); 2629 break; 2630 2631 default: 2632 break; 2633 } 2634 2635 /* 2636 * In order to keep per-task stats reliable we need to flip the event 2637 * values when we flip the contexts. 2638 */ 2639 value = local64_read(&next_event->count); 2640 value = local64_xchg(&event->count, value); 2641 local64_set(&next_event->count, value); 2642 2643 swap(event->total_time_enabled, next_event->total_time_enabled); 2644 swap(event->total_time_running, next_event->total_time_running); 2645 2646 /* 2647 * Since we swizzled the values, update the user visible data too. 2648 */ 2649 perf_event_update_userpage(event); 2650 perf_event_update_userpage(next_event); 2651 } 2652 2653 static void perf_event_sync_stat(struct perf_event_context *ctx, 2654 struct perf_event_context *next_ctx) 2655 { 2656 struct perf_event *event, *next_event; 2657 2658 if (!ctx->nr_stat) 2659 return; 2660 2661 update_context_time(ctx); 2662 2663 event = list_first_entry(&ctx->event_list, 2664 struct perf_event, event_entry); 2665 2666 next_event = list_first_entry(&next_ctx->event_list, 2667 struct perf_event, event_entry); 2668 2669 while (&event->event_entry != &ctx->event_list && 2670 &next_event->event_entry != &next_ctx->event_list) { 2671 2672 __perf_event_sync_stat(event, next_event); 2673 2674 event = list_next_entry(event, event_entry); 2675 next_event = list_next_entry(next_event, event_entry); 2676 } 2677 } 2678 2679 static void perf_event_context_sched_out(struct task_struct *task, int ctxn, 2680 struct task_struct *next) 2681 { 2682 struct perf_event_context *ctx = task->perf_event_ctxp[ctxn]; 2683 struct perf_event_context *next_ctx; 2684 struct perf_event_context *parent, *next_parent; 2685 struct perf_cpu_context *cpuctx; 2686 int do_switch = 1; 2687 2688 if (likely(!ctx)) 2689 return; 2690 2691 cpuctx = __get_cpu_context(ctx); 2692 if (!cpuctx->task_ctx) 2693 return; 2694 2695 rcu_read_lock(); 2696 next_ctx = next->perf_event_ctxp[ctxn]; 2697 if (!next_ctx) 2698 goto unlock; 2699 2700 parent = rcu_dereference(ctx->parent_ctx); 2701 next_parent = rcu_dereference(next_ctx->parent_ctx); 2702 2703 /* If neither context have a parent context; they cannot be clones. */ 2704 if (!parent && !next_parent) 2705 goto unlock; 2706 2707 if (next_parent == ctx || next_ctx == parent || next_parent == parent) { 2708 /* 2709 * Looks like the two contexts are clones, so we might be 2710 * able to optimize the context switch. We lock both 2711 * contexts and check that they are clones under the 2712 * lock (including re-checking that neither has been 2713 * uncloned in the meantime). It doesn't matter which 2714 * order we take the locks because no other cpu could 2715 * be trying to lock both of these tasks. 2716 */ 2717 raw_spin_lock(&ctx->lock); 2718 raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING); 2719 if (context_equiv(ctx, next_ctx)) { 2720 WRITE_ONCE(ctx->task, next); 2721 WRITE_ONCE(next_ctx->task, task); 2722 2723 swap(ctx->task_ctx_data, next_ctx->task_ctx_data); 2724 2725 /* 2726 * RCU_INIT_POINTER here is safe because we've not 2727 * modified the ctx and the above modification of 2728 * ctx->task and ctx->task_ctx_data are immaterial 2729 * since those values are always verified under 2730 * ctx->lock which we're now holding. 2731 */ 2732 RCU_INIT_POINTER(task->perf_event_ctxp[ctxn], next_ctx); 2733 RCU_INIT_POINTER(next->perf_event_ctxp[ctxn], ctx); 2734 2735 do_switch = 0; 2736 2737 perf_event_sync_stat(ctx, next_ctx); 2738 } 2739 raw_spin_unlock(&next_ctx->lock); 2740 raw_spin_unlock(&ctx->lock); 2741 } 2742 unlock: 2743 rcu_read_unlock(); 2744 2745 if (do_switch) { 2746 raw_spin_lock(&ctx->lock); 2747 task_ctx_sched_out(cpuctx, ctx); 2748 raw_spin_unlock(&ctx->lock); 2749 } 2750 } 2751 2752 void perf_sched_cb_dec(struct pmu *pmu) 2753 { 2754 this_cpu_dec(perf_sched_cb_usages); 2755 } 2756 2757 void perf_sched_cb_inc(struct pmu *pmu) 2758 { 2759 this_cpu_inc(perf_sched_cb_usages); 2760 } 2761 2762 /* 2763 * This function provides the context switch callback to the lower code 2764 * layer. It is invoked ONLY when the context switch callback is enabled. 2765 */ 2766 static void perf_pmu_sched_task(struct task_struct *prev, 2767 struct task_struct *next, 2768 bool sched_in) 2769 { 2770 struct perf_cpu_context *cpuctx; 2771 struct pmu *pmu; 2772 unsigned long flags; 2773 2774 if (prev == next) 2775 return; 2776 2777 local_irq_save(flags); 2778 2779 rcu_read_lock(); 2780 2781 list_for_each_entry_rcu(pmu, &pmus, entry) { 2782 if (pmu->sched_task) { 2783 cpuctx = this_cpu_ptr(pmu->pmu_cpu_context); 2784 2785 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 2786 2787 perf_pmu_disable(pmu); 2788 2789 pmu->sched_task(cpuctx->task_ctx, sched_in); 2790 2791 perf_pmu_enable(pmu); 2792 2793 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 2794 } 2795 } 2796 2797 rcu_read_unlock(); 2798 2799 local_irq_restore(flags); 2800 } 2801 2802 static void perf_event_switch(struct task_struct *task, 2803 struct task_struct *next_prev, bool sched_in); 2804 2805 #define for_each_task_context_nr(ctxn) \ 2806 for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++) 2807 2808 /* 2809 * Called from scheduler to remove the events of the current task, 2810 * with interrupts disabled. 2811 * 2812 * We stop each event and update the event value in event->count. 2813 * 2814 * This does not protect us against NMI, but disable() 2815 * sets the disabled bit in the control field of event _before_ 2816 * accessing the event control register. If a NMI hits, then it will 2817 * not restart the event. 2818 */ 2819 void __perf_event_task_sched_out(struct task_struct *task, 2820 struct task_struct *next) 2821 { 2822 int ctxn; 2823 2824 if (__this_cpu_read(perf_sched_cb_usages)) 2825 perf_pmu_sched_task(task, next, false); 2826 2827 if (atomic_read(&nr_switch_events)) 2828 perf_event_switch(task, next, false); 2829 2830 for_each_task_context_nr(ctxn) 2831 perf_event_context_sched_out(task, ctxn, next); 2832 2833 /* 2834 * if cgroup events exist on this CPU, then we need 2835 * to check if we have to switch out PMU state. 2836 * cgroup event are system-wide mode only 2837 */ 2838 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 2839 perf_cgroup_sched_out(task, next); 2840 } 2841 2842 /* 2843 * Called with IRQs disabled 2844 */ 2845 static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx, 2846 enum event_type_t event_type) 2847 { 2848 ctx_sched_out(&cpuctx->ctx, cpuctx, event_type); 2849 } 2850 2851 static void 2852 ctx_pinned_sched_in(struct perf_event_context *ctx, 2853 struct perf_cpu_context *cpuctx) 2854 { 2855 struct perf_event *event; 2856 2857 list_for_each_entry(event, &ctx->pinned_groups, group_entry) { 2858 if (event->state <= PERF_EVENT_STATE_OFF) 2859 continue; 2860 if (!event_filter_match(event)) 2861 continue; 2862 2863 /* may need to reset tstamp_enabled */ 2864 if (is_cgroup_event(event)) 2865 perf_cgroup_mark_enabled(event, ctx); 2866 2867 if (group_can_go_on(event, cpuctx, 1)) 2868 group_sched_in(event, cpuctx, ctx); 2869 2870 /* 2871 * If this pinned group hasn't been scheduled, 2872 * put it in error state. 2873 */ 2874 if (event->state == PERF_EVENT_STATE_INACTIVE) { 2875 update_group_times(event); 2876 event->state = PERF_EVENT_STATE_ERROR; 2877 } 2878 } 2879 } 2880 2881 static void 2882 ctx_flexible_sched_in(struct perf_event_context *ctx, 2883 struct perf_cpu_context *cpuctx) 2884 { 2885 struct perf_event *event; 2886 int can_add_hw = 1; 2887 2888 list_for_each_entry(event, &ctx->flexible_groups, group_entry) { 2889 /* Ignore events in OFF or ERROR state */ 2890 if (event->state <= PERF_EVENT_STATE_OFF) 2891 continue; 2892 /* 2893 * Listen to the 'cpu' scheduling filter constraint 2894 * of events: 2895 */ 2896 if (!event_filter_match(event)) 2897 continue; 2898 2899 /* may need to reset tstamp_enabled */ 2900 if (is_cgroup_event(event)) 2901 perf_cgroup_mark_enabled(event, ctx); 2902 2903 if (group_can_go_on(event, cpuctx, can_add_hw)) { 2904 if (group_sched_in(event, cpuctx, ctx)) 2905 can_add_hw = 0; 2906 } 2907 } 2908 } 2909 2910 static void 2911 ctx_sched_in(struct perf_event_context *ctx, 2912 struct perf_cpu_context *cpuctx, 2913 enum event_type_t event_type, 2914 struct task_struct *task) 2915 { 2916 int is_active = ctx->is_active; 2917 u64 now; 2918 2919 lockdep_assert_held(&ctx->lock); 2920 2921 if (likely(!ctx->nr_events)) 2922 return; 2923 2924 ctx->is_active |= (event_type | EVENT_TIME); 2925 if (ctx->task) { 2926 if (!is_active) 2927 cpuctx->task_ctx = ctx; 2928 else 2929 WARN_ON_ONCE(cpuctx->task_ctx != ctx); 2930 } 2931 2932 is_active ^= ctx->is_active; /* changed bits */ 2933 2934 if (is_active & EVENT_TIME) { 2935 /* start ctx time */ 2936 now = perf_clock(); 2937 ctx->timestamp = now; 2938 perf_cgroup_set_timestamp(task, ctx); 2939 } 2940 2941 /* 2942 * First go through the list and put on any pinned groups 2943 * in order to give them the best chance of going on. 2944 */ 2945 if (is_active & EVENT_PINNED) 2946 ctx_pinned_sched_in(ctx, cpuctx); 2947 2948 /* Then walk through the lower prio flexible groups */ 2949 if (is_active & EVENT_FLEXIBLE) 2950 ctx_flexible_sched_in(ctx, cpuctx); 2951 } 2952 2953 static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx, 2954 enum event_type_t event_type, 2955 struct task_struct *task) 2956 { 2957 struct perf_event_context *ctx = &cpuctx->ctx; 2958 2959 ctx_sched_in(ctx, cpuctx, event_type, task); 2960 } 2961 2962 static void perf_event_context_sched_in(struct perf_event_context *ctx, 2963 struct task_struct *task) 2964 { 2965 struct perf_cpu_context *cpuctx; 2966 2967 cpuctx = __get_cpu_context(ctx); 2968 if (cpuctx->task_ctx == ctx) 2969 return; 2970 2971 perf_ctx_lock(cpuctx, ctx); 2972 perf_pmu_disable(ctx->pmu); 2973 /* 2974 * We want to keep the following priority order: 2975 * cpu pinned (that don't need to move), task pinned, 2976 * cpu flexible, task flexible. 2977 */ 2978 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 2979 perf_event_sched_in(cpuctx, ctx, task); 2980 perf_pmu_enable(ctx->pmu); 2981 perf_ctx_unlock(cpuctx, ctx); 2982 } 2983 2984 /* 2985 * Called from scheduler to add the events of the current task 2986 * with interrupts disabled. 2987 * 2988 * We restore the event value and then enable it. 2989 * 2990 * This does not protect us against NMI, but enable() 2991 * sets the enabled bit in the control field of event _before_ 2992 * accessing the event control register. If a NMI hits, then it will 2993 * keep the event running. 2994 */ 2995 void __perf_event_task_sched_in(struct task_struct *prev, 2996 struct task_struct *task) 2997 { 2998 struct perf_event_context *ctx; 2999 int ctxn; 3000 3001 /* 3002 * If cgroup events exist on this CPU, then we need to check if we have 3003 * to switch in PMU state; cgroup event are system-wide mode only. 3004 * 3005 * Since cgroup events are CPU events, we must schedule these in before 3006 * we schedule in the task events. 3007 */ 3008 if (atomic_read(this_cpu_ptr(&perf_cgroup_events))) 3009 perf_cgroup_sched_in(prev, task); 3010 3011 for_each_task_context_nr(ctxn) { 3012 ctx = task->perf_event_ctxp[ctxn]; 3013 if (likely(!ctx)) 3014 continue; 3015 3016 perf_event_context_sched_in(ctx, task); 3017 } 3018 3019 if (atomic_read(&nr_switch_events)) 3020 perf_event_switch(task, prev, true); 3021 3022 if (__this_cpu_read(perf_sched_cb_usages)) 3023 perf_pmu_sched_task(prev, task, true); 3024 } 3025 3026 static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count) 3027 { 3028 u64 frequency = event->attr.sample_freq; 3029 u64 sec = NSEC_PER_SEC; 3030 u64 divisor, dividend; 3031 3032 int count_fls, nsec_fls, frequency_fls, sec_fls; 3033 3034 count_fls = fls64(count); 3035 nsec_fls = fls64(nsec); 3036 frequency_fls = fls64(frequency); 3037 sec_fls = 30; 3038 3039 /* 3040 * We got @count in @nsec, with a target of sample_freq HZ 3041 * the target period becomes: 3042 * 3043 * @count * 10^9 3044 * period = ------------------- 3045 * @nsec * sample_freq 3046 * 3047 */ 3048 3049 /* 3050 * Reduce accuracy by one bit such that @a and @b converge 3051 * to a similar magnitude. 3052 */ 3053 #define REDUCE_FLS(a, b) \ 3054 do { \ 3055 if (a##_fls > b##_fls) { \ 3056 a >>= 1; \ 3057 a##_fls--; \ 3058 } else { \ 3059 b >>= 1; \ 3060 b##_fls--; \ 3061 } \ 3062 } while (0) 3063 3064 /* 3065 * Reduce accuracy until either term fits in a u64, then proceed with 3066 * the other, so that finally we can do a u64/u64 division. 3067 */ 3068 while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) { 3069 REDUCE_FLS(nsec, frequency); 3070 REDUCE_FLS(sec, count); 3071 } 3072 3073 if (count_fls + sec_fls > 64) { 3074 divisor = nsec * frequency; 3075 3076 while (count_fls + sec_fls > 64) { 3077 REDUCE_FLS(count, sec); 3078 divisor >>= 1; 3079 } 3080 3081 dividend = count * sec; 3082 } else { 3083 dividend = count * sec; 3084 3085 while (nsec_fls + frequency_fls > 64) { 3086 REDUCE_FLS(nsec, frequency); 3087 dividend >>= 1; 3088 } 3089 3090 divisor = nsec * frequency; 3091 } 3092 3093 if (!divisor) 3094 return dividend; 3095 3096 return div64_u64(dividend, divisor); 3097 } 3098 3099 static DEFINE_PER_CPU(int, perf_throttled_count); 3100 static DEFINE_PER_CPU(u64, perf_throttled_seq); 3101 3102 static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable) 3103 { 3104 struct hw_perf_event *hwc = &event->hw; 3105 s64 period, sample_period; 3106 s64 delta; 3107 3108 period = perf_calculate_period(event, nsec, count); 3109 3110 delta = (s64)(period - hwc->sample_period); 3111 delta = (delta + 7) / 8; /* low pass filter */ 3112 3113 sample_period = hwc->sample_period + delta; 3114 3115 if (!sample_period) 3116 sample_period = 1; 3117 3118 hwc->sample_period = sample_period; 3119 3120 if (local64_read(&hwc->period_left) > 8*sample_period) { 3121 if (disable) 3122 event->pmu->stop(event, PERF_EF_UPDATE); 3123 3124 local64_set(&hwc->period_left, 0); 3125 3126 if (disable) 3127 event->pmu->start(event, PERF_EF_RELOAD); 3128 } 3129 } 3130 3131 /* 3132 * combine freq adjustment with unthrottling to avoid two passes over the 3133 * events. At the same time, make sure, having freq events does not change 3134 * the rate of unthrottling as that would introduce bias. 3135 */ 3136 static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx, 3137 int needs_unthr) 3138 { 3139 struct perf_event *event; 3140 struct hw_perf_event *hwc; 3141 u64 now, period = TICK_NSEC; 3142 s64 delta; 3143 3144 /* 3145 * only need to iterate over all events iff: 3146 * - context have events in frequency mode (needs freq adjust) 3147 * - there are events to unthrottle on this cpu 3148 */ 3149 if (!(ctx->nr_freq || needs_unthr)) 3150 return; 3151 3152 raw_spin_lock(&ctx->lock); 3153 perf_pmu_disable(ctx->pmu); 3154 3155 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 3156 if (event->state != PERF_EVENT_STATE_ACTIVE) 3157 continue; 3158 3159 if (!event_filter_match(event)) 3160 continue; 3161 3162 perf_pmu_disable(event->pmu); 3163 3164 hwc = &event->hw; 3165 3166 if (hwc->interrupts == MAX_INTERRUPTS) { 3167 hwc->interrupts = 0; 3168 perf_log_throttle(event, 1); 3169 event->pmu->start(event, 0); 3170 } 3171 3172 if (!event->attr.freq || !event->attr.sample_freq) 3173 goto next; 3174 3175 /* 3176 * stop the event and update event->count 3177 */ 3178 event->pmu->stop(event, PERF_EF_UPDATE); 3179 3180 now = local64_read(&event->count); 3181 delta = now - hwc->freq_count_stamp; 3182 hwc->freq_count_stamp = now; 3183 3184 /* 3185 * restart the event 3186 * reload only if value has changed 3187 * we have stopped the event so tell that 3188 * to perf_adjust_period() to avoid stopping it 3189 * twice. 3190 */ 3191 if (delta > 0) 3192 perf_adjust_period(event, period, delta, false); 3193 3194 event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0); 3195 next: 3196 perf_pmu_enable(event->pmu); 3197 } 3198 3199 perf_pmu_enable(ctx->pmu); 3200 raw_spin_unlock(&ctx->lock); 3201 } 3202 3203 /* 3204 * Round-robin a context's events: 3205 */ 3206 static void rotate_ctx(struct perf_event_context *ctx) 3207 { 3208 /* 3209 * Rotate the first entry last of non-pinned groups. Rotation might be 3210 * disabled by the inheritance code. 3211 */ 3212 if (!ctx->rotate_disable) 3213 list_rotate_left(&ctx->flexible_groups); 3214 } 3215 3216 static int perf_rotate_context(struct perf_cpu_context *cpuctx) 3217 { 3218 struct perf_event_context *ctx = NULL; 3219 int rotate = 0; 3220 3221 if (cpuctx->ctx.nr_events) { 3222 if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active) 3223 rotate = 1; 3224 } 3225 3226 ctx = cpuctx->task_ctx; 3227 if (ctx && ctx->nr_events) { 3228 if (ctx->nr_events != ctx->nr_active) 3229 rotate = 1; 3230 } 3231 3232 if (!rotate) 3233 goto done; 3234 3235 perf_ctx_lock(cpuctx, cpuctx->task_ctx); 3236 perf_pmu_disable(cpuctx->ctx.pmu); 3237 3238 cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE); 3239 if (ctx) 3240 ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE); 3241 3242 rotate_ctx(&cpuctx->ctx); 3243 if (ctx) 3244 rotate_ctx(ctx); 3245 3246 perf_event_sched_in(cpuctx, ctx, current); 3247 3248 perf_pmu_enable(cpuctx->ctx.pmu); 3249 perf_ctx_unlock(cpuctx, cpuctx->task_ctx); 3250 done: 3251 3252 return rotate; 3253 } 3254 3255 void perf_event_task_tick(void) 3256 { 3257 struct list_head *head = this_cpu_ptr(&active_ctx_list); 3258 struct perf_event_context *ctx, *tmp; 3259 int throttled; 3260 3261 WARN_ON(!irqs_disabled()); 3262 3263 __this_cpu_inc(perf_throttled_seq); 3264 throttled = __this_cpu_xchg(perf_throttled_count, 0); 3265 tick_dep_clear_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); 3266 3267 list_for_each_entry_safe(ctx, tmp, head, active_ctx_list) 3268 perf_adjust_freq_unthr_context(ctx, throttled); 3269 } 3270 3271 static int event_enable_on_exec(struct perf_event *event, 3272 struct perf_event_context *ctx) 3273 { 3274 if (!event->attr.enable_on_exec) 3275 return 0; 3276 3277 event->attr.enable_on_exec = 0; 3278 if (event->state >= PERF_EVENT_STATE_INACTIVE) 3279 return 0; 3280 3281 __perf_event_mark_enabled(event); 3282 3283 return 1; 3284 } 3285 3286 /* 3287 * Enable all of a task's events that have been marked enable-on-exec. 3288 * This expects task == current. 3289 */ 3290 static void perf_event_enable_on_exec(int ctxn) 3291 { 3292 struct perf_event_context *ctx, *clone_ctx = NULL; 3293 struct perf_cpu_context *cpuctx; 3294 struct perf_event *event; 3295 unsigned long flags; 3296 int enabled = 0; 3297 3298 local_irq_save(flags); 3299 ctx = current->perf_event_ctxp[ctxn]; 3300 if (!ctx || !ctx->nr_events) 3301 goto out; 3302 3303 cpuctx = __get_cpu_context(ctx); 3304 perf_ctx_lock(cpuctx, ctx); 3305 ctx_sched_out(ctx, cpuctx, EVENT_TIME); 3306 list_for_each_entry(event, &ctx->event_list, event_entry) 3307 enabled |= event_enable_on_exec(event, ctx); 3308 3309 /* 3310 * Unclone and reschedule this context if we enabled any event. 3311 */ 3312 if (enabled) { 3313 clone_ctx = unclone_ctx(ctx); 3314 ctx_resched(cpuctx, ctx); 3315 } 3316 perf_ctx_unlock(cpuctx, ctx); 3317 3318 out: 3319 local_irq_restore(flags); 3320 3321 if (clone_ctx) 3322 put_ctx(clone_ctx); 3323 } 3324 3325 struct perf_read_data { 3326 struct perf_event *event; 3327 bool group; 3328 int ret; 3329 }; 3330 3331 /* 3332 * Cross CPU call to read the hardware event 3333 */ 3334 static void __perf_event_read(void *info) 3335 { 3336 struct perf_read_data *data = info; 3337 struct perf_event *sub, *event = data->event; 3338 struct perf_event_context *ctx = event->ctx; 3339 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 3340 struct pmu *pmu = event->pmu; 3341 3342 /* 3343 * If this is a task context, we need to check whether it is 3344 * the current task context of this cpu. If not it has been 3345 * scheduled out before the smp call arrived. In that case 3346 * event->count would have been updated to a recent sample 3347 * when the event was scheduled out. 3348 */ 3349 if (ctx->task && cpuctx->task_ctx != ctx) 3350 return; 3351 3352 raw_spin_lock(&ctx->lock); 3353 if (ctx->is_active) { 3354 update_context_time(ctx); 3355 update_cgrp_time_from_event(event); 3356 } 3357 3358 update_event_times(event); 3359 if (event->state != PERF_EVENT_STATE_ACTIVE) 3360 goto unlock; 3361 3362 if (!data->group) { 3363 pmu->read(event); 3364 data->ret = 0; 3365 goto unlock; 3366 } 3367 3368 pmu->start_txn(pmu, PERF_PMU_TXN_READ); 3369 3370 pmu->read(event); 3371 3372 list_for_each_entry(sub, &event->sibling_list, group_entry) { 3373 update_event_times(sub); 3374 if (sub->state == PERF_EVENT_STATE_ACTIVE) { 3375 /* 3376 * Use sibling's PMU rather than @event's since 3377 * sibling could be on different (eg: software) PMU. 3378 */ 3379 sub->pmu->read(sub); 3380 } 3381 } 3382 3383 data->ret = pmu->commit_txn(pmu); 3384 3385 unlock: 3386 raw_spin_unlock(&ctx->lock); 3387 } 3388 3389 static inline u64 perf_event_count(struct perf_event *event) 3390 { 3391 if (event->pmu->count) 3392 return event->pmu->count(event); 3393 3394 return __perf_event_count(event); 3395 } 3396 3397 /* 3398 * NMI-safe method to read a local event, that is an event that 3399 * is: 3400 * - either for the current task, or for this CPU 3401 * - does not have inherit set, for inherited task events 3402 * will not be local and we cannot read them atomically 3403 * - must not have a pmu::count method 3404 */ 3405 u64 perf_event_read_local(struct perf_event *event) 3406 { 3407 unsigned long flags; 3408 u64 val; 3409 3410 /* 3411 * Disabling interrupts avoids all counter scheduling (context 3412 * switches, timer based rotation and IPIs). 3413 */ 3414 local_irq_save(flags); 3415 3416 /* If this is a per-task event, it must be for current */ 3417 WARN_ON_ONCE((event->attach_state & PERF_ATTACH_TASK) && 3418 event->hw.target != current); 3419 3420 /* If this is a per-CPU event, it must be for this CPU */ 3421 WARN_ON_ONCE(!(event->attach_state & PERF_ATTACH_TASK) && 3422 event->cpu != smp_processor_id()); 3423 3424 /* 3425 * It must not be an event with inherit set, we cannot read 3426 * all child counters from atomic context. 3427 */ 3428 WARN_ON_ONCE(event->attr.inherit); 3429 3430 /* 3431 * It must not have a pmu::count method, those are not 3432 * NMI safe. 3433 */ 3434 WARN_ON_ONCE(event->pmu->count); 3435 3436 /* 3437 * If the event is currently on this CPU, its either a per-task event, 3438 * or local to this CPU. Furthermore it means its ACTIVE (otherwise 3439 * oncpu == -1). 3440 */ 3441 if (event->oncpu == smp_processor_id()) 3442 event->pmu->read(event); 3443 3444 val = local64_read(&event->count); 3445 local_irq_restore(flags); 3446 3447 return val; 3448 } 3449 3450 static int perf_event_read(struct perf_event *event, bool group) 3451 { 3452 int ret = 0; 3453 3454 /* 3455 * If event is enabled and currently active on a CPU, update the 3456 * value in the event structure: 3457 */ 3458 if (event->state == PERF_EVENT_STATE_ACTIVE) { 3459 struct perf_read_data data = { 3460 .event = event, 3461 .group = group, 3462 .ret = 0, 3463 }; 3464 smp_call_function_single(event->oncpu, 3465 __perf_event_read, &data, 1); 3466 ret = data.ret; 3467 } else if (event->state == PERF_EVENT_STATE_INACTIVE) { 3468 struct perf_event_context *ctx = event->ctx; 3469 unsigned long flags; 3470 3471 raw_spin_lock_irqsave(&ctx->lock, flags); 3472 /* 3473 * may read while context is not active 3474 * (e.g., thread is blocked), in that case 3475 * we cannot update context time 3476 */ 3477 if (ctx->is_active) { 3478 update_context_time(ctx); 3479 update_cgrp_time_from_event(event); 3480 } 3481 if (group) 3482 update_group_times(event); 3483 else 3484 update_event_times(event); 3485 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3486 } 3487 3488 return ret; 3489 } 3490 3491 /* 3492 * Initialize the perf_event context in a task_struct: 3493 */ 3494 static void __perf_event_init_context(struct perf_event_context *ctx) 3495 { 3496 raw_spin_lock_init(&ctx->lock); 3497 mutex_init(&ctx->mutex); 3498 INIT_LIST_HEAD(&ctx->active_ctx_list); 3499 INIT_LIST_HEAD(&ctx->pinned_groups); 3500 INIT_LIST_HEAD(&ctx->flexible_groups); 3501 INIT_LIST_HEAD(&ctx->event_list); 3502 atomic_set(&ctx->refcount, 1); 3503 } 3504 3505 static struct perf_event_context * 3506 alloc_perf_context(struct pmu *pmu, struct task_struct *task) 3507 { 3508 struct perf_event_context *ctx; 3509 3510 ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL); 3511 if (!ctx) 3512 return NULL; 3513 3514 __perf_event_init_context(ctx); 3515 if (task) { 3516 ctx->task = task; 3517 get_task_struct(task); 3518 } 3519 ctx->pmu = pmu; 3520 3521 return ctx; 3522 } 3523 3524 static struct task_struct * 3525 find_lively_task_by_vpid(pid_t vpid) 3526 { 3527 struct task_struct *task; 3528 3529 rcu_read_lock(); 3530 if (!vpid) 3531 task = current; 3532 else 3533 task = find_task_by_vpid(vpid); 3534 if (task) 3535 get_task_struct(task); 3536 rcu_read_unlock(); 3537 3538 if (!task) 3539 return ERR_PTR(-ESRCH); 3540 3541 return task; 3542 } 3543 3544 /* 3545 * Returns a matching context with refcount and pincount. 3546 */ 3547 static struct perf_event_context * 3548 find_get_context(struct pmu *pmu, struct task_struct *task, 3549 struct perf_event *event) 3550 { 3551 struct perf_event_context *ctx, *clone_ctx = NULL; 3552 struct perf_cpu_context *cpuctx; 3553 void *task_ctx_data = NULL; 3554 unsigned long flags; 3555 int ctxn, err; 3556 int cpu = event->cpu; 3557 3558 if (!task) { 3559 /* Must be root to operate on a CPU event: */ 3560 if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN)) 3561 return ERR_PTR(-EACCES); 3562 3563 /* 3564 * We could be clever and allow to attach a event to an 3565 * offline CPU and activate it when the CPU comes up, but 3566 * that's for later. 3567 */ 3568 if (!cpu_online(cpu)) 3569 return ERR_PTR(-ENODEV); 3570 3571 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 3572 ctx = &cpuctx->ctx; 3573 get_ctx(ctx); 3574 ++ctx->pin_count; 3575 3576 return ctx; 3577 } 3578 3579 err = -EINVAL; 3580 ctxn = pmu->task_ctx_nr; 3581 if (ctxn < 0) 3582 goto errout; 3583 3584 if (event->attach_state & PERF_ATTACH_TASK_DATA) { 3585 task_ctx_data = kzalloc(pmu->task_ctx_size, GFP_KERNEL); 3586 if (!task_ctx_data) { 3587 err = -ENOMEM; 3588 goto errout; 3589 } 3590 } 3591 3592 retry: 3593 ctx = perf_lock_task_context(task, ctxn, &flags); 3594 if (ctx) { 3595 clone_ctx = unclone_ctx(ctx); 3596 ++ctx->pin_count; 3597 3598 if (task_ctx_data && !ctx->task_ctx_data) { 3599 ctx->task_ctx_data = task_ctx_data; 3600 task_ctx_data = NULL; 3601 } 3602 raw_spin_unlock_irqrestore(&ctx->lock, flags); 3603 3604 if (clone_ctx) 3605 put_ctx(clone_ctx); 3606 } else { 3607 ctx = alloc_perf_context(pmu, task); 3608 err = -ENOMEM; 3609 if (!ctx) 3610 goto errout; 3611 3612 if (task_ctx_data) { 3613 ctx->task_ctx_data = task_ctx_data; 3614 task_ctx_data = NULL; 3615 } 3616 3617 err = 0; 3618 mutex_lock(&task->perf_event_mutex); 3619 /* 3620 * If it has already passed perf_event_exit_task(). 3621 * we must see PF_EXITING, it takes this mutex too. 3622 */ 3623 if (task->flags & PF_EXITING) 3624 err = -ESRCH; 3625 else if (task->perf_event_ctxp[ctxn]) 3626 err = -EAGAIN; 3627 else { 3628 get_ctx(ctx); 3629 ++ctx->pin_count; 3630 rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx); 3631 } 3632 mutex_unlock(&task->perf_event_mutex); 3633 3634 if (unlikely(err)) { 3635 put_ctx(ctx); 3636 3637 if (err == -EAGAIN) 3638 goto retry; 3639 goto errout; 3640 } 3641 } 3642 3643 kfree(task_ctx_data); 3644 return ctx; 3645 3646 errout: 3647 kfree(task_ctx_data); 3648 return ERR_PTR(err); 3649 } 3650 3651 static void perf_event_free_filter(struct perf_event *event); 3652 static void perf_event_free_bpf_prog(struct perf_event *event); 3653 3654 static void free_event_rcu(struct rcu_head *head) 3655 { 3656 struct perf_event *event; 3657 3658 event = container_of(head, struct perf_event, rcu_head); 3659 if (event->ns) 3660 put_pid_ns(event->ns); 3661 perf_event_free_filter(event); 3662 kfree(event); 3663 } 3664 3665 static void ring_buffer_attach(struct perf_event *event, 3666 struct ring_buffer *rb); 3667 3668 static void unaccount_event_cpu(struct perf_event *event, int cpu) 3669 { 3670 if (event->parent) 3671 return; 3672 3673 if (is_cgroup_event(event)) 3674 atomic_dec(&per_cpu(perf_cgroup_events, cpu)); 3675 } 3676 3677 #ifdef CONFIG_NO_HZ_FULL 3678 static DEFINE_SPINLOCK(nr_freq_lock); 3679 #endif 3680 3681 static void unaccount_freq_event_nohz(void) 3682 { 3683 #ifdef CONFIG_NO_HZ_FULL 3684 spin_lock(&nr_freq_lock); 3685 if (atomic_dec_and_test(&nr_freq_events)) 3686 tick_nohz_dep_clear(TICK_DEP_BIT_PERF_EVENTS); 3687 spin_unlock(&nr_freq_lock); 3688 #endif 3689 } 3690 3691 static void unaccount_freq_event(void) 3692 { 3693 if (tick_nohz_full_enabled()) 3694 unaccount_freq_event_nohz(); 3695 else 3696 atomic_dec(&nr_freq_events); 3697 } 3698 3699 static void unaccount_event(struct perf_event *event) 3700 { 3701 bool dec = false; 3702 3703 if (event->parent) 3704 return; 3705 3706 if (event->attach_state & PERF_ATTACH_TASK) 3707 dec = true; 3708 if (event->attr.mmap || event->attr.mmap_data) 3709 atomic_dec(&nr_mmap_events); 3710 if (event->attr.comm) 3711 atomic_dec(&nr_comm_events); 3712 if (event->attr.task) 3713 atomic_dec(&nr_task_events); 3714 if (event->attr.freq) 3715 unaccount_freq_event(); 3716 if (event->attr.context_switch) { 3717 dec = true; 3718 atomic_dec(&nr_switch_events); 3719 } 3720 if (is_cgroup_event(event)) 3721 dec = true; 3722 if (has_branch_stack(event)) 3723 dec = true; 3724 3725 if (dec) { 3726 if (!atomic_add_unless(&perf_sched_count, -1, 1)) 3727 schedule_delayed_work(&perf_sched_work, HZ); 3728 } 3729 3730 unaccount_event_cpu(event, event->cpu); 3731 } 3732 3733 static void perf_sched_delayed(struct work_struct *work) 3734 { 3735 mutex_lock(&perf_sched_mutex); 3736 if (atomic_dec_and_test(&perf_sched_count)) 3737 static_branch_disable(&perf_sched_events); 3738 mutex_unlock(&perf_sched_mutex); 3739 } 3740 3741 /* 3742 * The following implement mutual exclusion of events on "exclusive" pmus 3743 * (PERF_PMU_CAP_EXCLUSIVE). Such pmus can only have one event scheduled 3744 * at a time, so we disallow creating events that might conflict, namely: 3745 * 3746 * 1) cpu-wide events in the presence of per-task events, 3747 * 2) per-task events in the presence of cpu-wide events, 3748 * 3) two matching events on the same context. 3749 * 3750 * The former two cases are handled in the allocation path (perf_event_alloc(), 3751 * _free_event()), the latter -- before the first perf_install_in_context(). 3752 */ 3753 static int exclusive_event_init(struct perf_event *event) 3754 { 3755 struct pmu *pmu = event->pmu; 3756 3757 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 3758 return 0; 3759 3760 /* 3761 * Prevent co-existence of per-task and cpu-wide events on the 3762 * same exclusive pmu. 3763 * 3764 * Negative pmu::exclusive_cnt means there are cpu-wide 3765 * events on this "exclusive" pmu, positive means there are 3766 * per-task events. 3767 * 3768 * Since this is called in perf_event_alloc() path, event::ctx 3769 * doesn't exist yet; it is, however, safe to use PERF_ATTACH_TASK 3770 * to mean "per-task event", because unlike other attach states it 3771 * never gets cleared. 3772 */ 3773 if (event->attach_state & PERF_ATTACH_TASK) { 3774 if (!atomic_inc_unless_negative(&pmu->exclusive_cnt)) 3775 return -EBUSY; 3776 } else { 3777 if (!atomic_dec_unless_positive(&pmu->exclusive_cnt)) 3778 return -EBUSY; 3779 } 3780 3781 return 0; 3782 } 3783 3784 static void exclusive_event_destroy(struct perf_event *event) 3785 { 3786 struct pmu *pmu = event->pmu; 3787 3788 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 3789 return; 3790 3791 /* see comment in exclusive_event_init() */ 3792 if (event->attach_state & PERF_ATTACH_TASK) 3793 atomic_dec(&pmu->exclusive_cnt); 3794 else 3795 atomic_inc(&pmu->exclusive_cnt); 3796 } 3797 3798 static bool exclusive_event_match(struct perf_event *e1, struct perf_event *e2) 3799 { 3800 if ((e1->pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE) && 3801 (e1->cpu == e2->cpu || 3802 e1->cpu == -1 || 3803 e2->cpu == -1)) 3804 return true; 3805 return false; 3806 } 3807 3808 /* Called under the same ctx::mutex as perf_install_in_context() */ 3809 static bool exclusive_event_installable(struct perf_event *event, 3810 struct perf_event_context *ctx) 3811 { 3812 struct perf_event *iter_event; 3813 struct pmu *pmu = event->pmu; 3814 3815 if (!(pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE)) 3816 return true; 3817 3818 list_for_each_entry(iter_event, &ctx->event_list, event_entry) { 3819 if (exclusive_event_match(iter_event, event)) 3820 return false; 3821 } 3822 3823 return true; 3824 } 3825 3826 static void perf_addr_filters_splice(struct perf_event *event, 3827 struct list_head *head); 3828 3829 static void _free_event(struct perf_event *event) 3830 { 3831 irq_work_sync(&event->pending); 3832 3833 unaccount_event(event); 3834 3835 if (event->rb) { 3836 /* 3837 * Can happen when we close an event with re-directed output. 3838 * 3839 * Since we have a 0 refcount, perf_mmap_close() will skip 3840 * over us; possibly making our ring_buffer_put() the last. 3841 */ 3842 mutex_lock(&event->mmap_mutex); 3843 ring_buffer_attach(event, NULL); 3844 mutex_unlock(&event->mmap_mutex); 3845 } 3846 3847 if (is_cgroup_event(event)) 3848 perf_detach_cgroup(event); 3849 3850 if (!event->parent) { 3851 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) 3852 put_callchain_buffers(); 3853 } 3854 3855 perf_event_free_bpf_prog(event); 3856 perf_addr_filters_splice(event, NULL); 3857 kfree(event->addr_filters_offs); 3858 3859 if (event->destroy) 3860 event->destroy(event); 3861 3862 if (event->ctx) 3863 put_ctx(event->ctx); 3864 3865 if (event->pmu) { 3866 exclusive_event_destroy(event); 3867 module_put(event->pmu->module); 3868 } 3869 3870 call_rcu(&event->rcu_head, free_event_rcu); 3871 } 3872 3873 /* 3874 * Used to free events which have a known refcount of 1, such as in error paths 3875 * where the event isn't exposed yet and inherited events. 3876 */ 3877 static void free_event(struct perf_event *event) 3878 { 3879 if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1, 3880 "unexpected event refcount: %ld; ptr=%p\n", 3881 atomic_long_read(&event->refcount), event)) { 3882 /* leak to avoid use-after-free */ 3883 return; 3884 } 3885 3886 _free_event(event); 3887 } 3888 3889 /* 3890 * Remove user event from the owner task. 3891 */ 3892 static void perf_remove_from_owner(struct perf_event *event) 3893 { 3894 struct task_struct *owner; 3895 3896 rcu_read_lock(); 3897 /* 3898 * Matches the smp_store_release() in perf_event_exit_task(). If we 3899 * observe !owner it means the list deletion is complete and we can 3900 * indeed free this event, otherwise we need to serialize on 3901 * owner->perf_event_mutex. 3902 */ 3903 owner = lockless_dereference(event->owner); 3904 if (owner) { 3905 /* 3906 * Since delayed_put_task_struct() also drops the last 3907 * task reference we can safely take a new reference 3908 * while holding the rcu_read_lock(). 3909 */ 3910 get_task_struct(owner); 3911 } 3912 rcu_read_unlock(); 3913 3914 if (owner) { 3915 /* 3916 * If we're here through perf_event_exit_task() we're already 3917 * holding ctx->mutex which would be an inversion wrt. the 3918 * normal lock order. 3919 * 3920 * However we can safely take this lock because its the child 3921 * ctx->mutex. 3922 */ 3923 mutex_lock_nested(&owner->perf_event_mutex, SINGLE_DEPTH_NESTING); 3924 3925 /* 3926 * We have to re-check the event->owner field, if it is cleared 3927 * we raced with perf_event_exit_task(), acquiring the mutex 3928 * ensured they're done, and we can proceed with freeing the 3929 * event. 3930 */ 3931 if (event->owner) { 3932 list_del_init(&event->owner_entry); 3933 smp_store_release(&event->owner, NULL); 3934 } 3935 mutex_unlock(&owner->perf_event_mutex); 3936 put_task_struct(owner); 3937 } 3938 } 3939 3940 static void put_event(struct perf_event *event) 3941 { 3942 if (!atomic_long_dec_and_test(&event->refcount)) 3943 return; 3944 3945 _free_event(event); 3946 } 3947 3948 /* 3949 * Kill an event dead; while event:refcount will preserve the event 3950 * object, it will not preserve its functionality. Once the last 'user' 3951 * gives up the object, we'll destroy the thing. 3952 */ 3953 int perf_event_release_kernel(struct perf_event *event) 3954 { 3955 struct perf_event_context *ctx = event->ctx; 3956 struct perf_event *child, *tmp; 3957 3958 /* 3959 * If we got here through err_file: fput(event_file); we will not have 3960 * attached to a context yet. 3961 */ 3962 if (!ctx) { 3963 WARN_ON_ONCE(event->attach_state & 3964 (PERF_ATTACH_CONTEXT|PERF_ATTACH_GROUP)); 3965 goto no_ctx; 3966 } 3967 3968 if (!is_kernel_event(event)) 3969 perf_remove_from_owner(event); 3970 3971 ctx = perf_event_ctx_lock(event); 3972 WARN_ON_ONCE(ctx->parent_ctx); 3973 perf_remove_from_context(event, DETACH_GROUP); 3974 3975 raw_spin_lock_irq(&ctx->lock); 3976 /* 3977 * Mark this even as STATE_DEAD, there is no external reference to it 3978 * anymore. 3979 * 3980 * Anybody acquiring event->child_mutex after the below loop _must_ 3981 * also see this, most importantly inherit_event() which will avoid 3982 * placing more children on the list. 3983 * 3984 * Thus this guarantees that we will in fact observe and kill _ALL_ 3985 * child events. 3986 */ 3987 event->state = PERF_EVENT_STATE_DEAD; 3988 raw_spin_unlock_irq(&ctx->lock); 3989 3990 perf_event_ctx_unlock(event, ctx); 3991 3992 again: 3993 mutex_lock(&event->child_mutex); 3994 list_for_each_entry(child, &event->child_list, child_list) { 3995 3996 /* 3997 * Cannot change, child events are not migrated, see the 3998 * comment with perf_event_ctx_lock_nested(). 3999 */ 4000 ctx = lockless_dereference(child->ctx); 4001 /* 4002 * Since child_mutex nests inside ctx::mutex, we must jump 4003 * through hoops. We start by grabbing a reference on the ctx. 4004 * 4005 * Since the event cannot get freed while we hold the 4006 * child_mutex, the context must also exist and have a !0 4007 * reference count. 4008 */ 4009 get_ctx(ctx); 4010 4011 /* 4012 * Now that we have a ctx ref, we can drop child_mutex, and 4013 * acquire ctx::mutex without fear of it going away. Then we 4014 * can re-acquire child_mutex. 4015 */ 4016 mutex_unlock(&event->child_mutex); 4017 mutex_lock(&ctx->mutex); 4018 mutex_lock(&event->child_mutex); 4019 4020 /* 4021 * Now that we hold ctx::mutex and child_mutex, revalidate our 4022 * state, if child is still the first entry, it didn't get freed 4023 * and we can continue doing so. 4024 */ 4025 tmp = list_first_entry_or_null(&event->child_list, 4026 struct perf_event, child_list); 4027 if (tmp == child) { 4028 perf_remove_from_context(child, DETACH_GROUP); 4029 list_del(&child->child_list); 4030 free_event(child); 4031 /* 4032 * This matches the refcount bump in inherit_event(); 4033 * this can't be the last reference. 4034 */ 4035 put_event(event); 4036 } 4037 4038 mutex_unlock(&event->child_mutex); 4039 mutex_unlock(&ctx->mutex); 4040 put_ctx(ctx); 4041 goto again; 4042 } 4043 mutex_unlock(&event->child_mutex); 4044 4045 no_ctx: 4046 put_event(event); /* Must be the 'last' reference */ 4047 return 0; 4048 } 4049 EXPORT_SYMBOL_GPL(perf_event_release_kernel); 4050 4051 /* 4052 * Called when the last reference to the file is gone. 4053 */ 4054 static int perf_release(struct inode *inode, struct file *file) 4055 { 4056 perf_event_release_kernel(file->private_data); 4057 return 0; 4058 } 4059 4060 u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running) 4061 { 4062 struct perf_event *child; 4063 u64 total = 0; 4064 4065 *enabled = 0; 4066 *running = 0; 4067 4068 mutex_lock(&event->child_mutex); 4069 4070 (void)perf_event_read(event, false); 4071 total += perf_event_count(event); 4072 4073 *enabled += event->total_time_enabled + 4074 atomic64_read(&event->child_total_time_enabled); 4075 *running += event->total_time_running + 4076 atomic64_read(&event->child_total_time_running); 4077 4078 list_for_each_entry(child, &event->child_list, child_list) { 4079 (void)perf_event_read(child, false); 4080 total += perf_event_count(child); 4081 *enabled += child->total_time_enabled; 4082 *running += child->total_time_running; 4083 } 4084 mutex_unlock(&event->child_mutex); 4085 4086 return total; 4087 } 4088 EXPORT_SYMBOL_GPL(perf_event_read_value); 4089 4090 static int __perf_read_group_add(struct perf_event *leader, 4091 u64 read_format, u64 *values) 4092 { 4093 struct perf_event *sub; 4094 int n = 1; /* skip @nr */ 4095 int ret; 4096 4097 ret = perf_event_read(leader, true); 4098 if (ret) 4099 return ret; 4100 4101 /* 4102 * Since we co-schedule groups, {enabled,running} times of siblings 4103 * will be identical to those of the leader, so we only publish one 4104 * set. 4105 */ 4106 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 4107 values[n++] += leader->total_time_enabled + 4108 atomic64_read(&leader->child_total_time_enabled); 4109 } 4110 4111 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 4112 values[n++] += leader->total_time_running + 4113 atomic64_read(&leader->child_total_time_running); 4114 } 4115 4116 /* 4117 * Write {count,id} tuples for every sibling. 4118 */ 4119 values[n++] += perf_event_count(leader); 4120 if (read_format & PERF_FORMAT_ID) 4121 values[n++] = primary_event_id(leader); 4122 4123 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 4124 values[n++] += perf_event_count(sub); 4125 if (read_format & PERF_FORMAT_ID) 4126 values[n++] = primary_event_id(sub); 4127 } 4128 4129 return 0; 4130 } 4131 4132 static int perf_read_group(struct perf_event *event, 4133 u64 read_format, char __user *buf) 4134 { 4135 struct perf_event *leader = event->group_leader, *child; 4136 struct perf_event_context *ctx = leader->ctx; 4137 int ret; 4138 u64 *values; 4139 4140 lockdep_assert_held(&ctx->mutex); 4141 4142 values = kzalloc(event->read_size, GFP_KERNEL); 4143 if (!values) 4144 return -ENOMEM; 4145 4146 values[0] = 1 + leader->nr_siblings; 4147 4148 /* 4149 * By locking the child_mutex of the leader we effectively 4150 * lock the child list of all siblings.. XXX explain how. 4151 */ 4152 mutex_lock(&leader->child_mutex); 4153 4154 ret = __perf_read_group_add(leader, read_format, values); 4155 if (ret) 4156 goto unlock; 4157 4158 list_for_each_entry(child, &leader->child_list, child_list) { 4159 ret = __perf_read_group_add(child, read_format, values); 4160 if (ret) 4161 goto unlock; 4162 } 4163 4164 mutex_unlock(&leader->child_mutex); 4165 4166 ret = event->read_size; 4167 if (copy_to_user(buf, values, event->read_size)) 4168 ret = -EFAULT; 4169 goto out; 4170 4171 unlock: 4172 mutex_unlock(&leader->child_mutex); 4173 out: 4174 kfree(values); 4175 return ret; 4176 } 4177 4178 static int perf_read_one(struct perf_event *event, 4179 u64 read_format, char __user *buf) 4180 { 4181 u64 enabled, running; 4182 u64 values[4]; 4183 int n = 0; 4184 4185 values[n++] = perf_event_read_value(event, &enabled, &running); 4186 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 4187 values[n++] = enabled; 4188 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 4189 values[n++] = running; 4190 if (read_format & PERF_FORMAT_ID) 4191 values[n++] = primary_event_id(event); 4192 4193 if (copy_to_user(buf, values, n * sizeof(u64))) 4194 return -EFAULT; 4195 4196 return n * sizeof(u64); 4197 } 4198 4199 static bool is_event_hup(struct perf_event *event) 4200 { 4201 bool no_children; 4202 4203 if (event->state > PERF_EVENT_STATE_EXIT) 4204 return false; 4205 4206 mutex_lock(&event->child_mutex); 4207 no_children = list_empty(&event->child_list); 4208 mutex_unlock(&event->child_mutex); 4209 return no_children; 4210 } 4211 4212 /* 4213 * Read the performance event - simple non blocking version for now 4214 */ 4215 static ssize_t 4216 __perf_read(struct perf_event *event, char __user *buf, size_t count) 4217 { 4218 u64 read_format = event->attr.read_format; 4219 int ret; 4220 4221 /* 4222 * Return end-of-file for a read on a event that is in 4223 * error state (i.e. because it was pinned but it couldn't be 4224 * scheduled on to the CPU at some point). 4225 */ 4226 if (event->state == PERF_EVENT_STATE_ERROR) 4227 return 0; 4228 4229 if (count < event->read_size) 4230 return -ENOSPC; 4231 4232 WARN_ON_ONCE(event->ctx->parent_ctx); 4233 if (read_format & PERF_FORMAT_GROUP) 4234 ret = perf_read_group(event, read_format, buf); 4235 else 4236 ret = perf_read_one(event, read_format, buf); 4237 4238 return ret; 4239 } 4240 4241 static ssize_t 4242 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos) 4243 { 4244 struct perf_event *event = file->private_data; 4245 struct perf_event_context *ctx; 4246 int ret; 4247 4248 ctx = perf_event_ctx_lock(event); 4249 ret = __perf_read(event, buf, count); 4250 perf_event_ctx_unlock(event, ctx); 4251 4252 return ret; 4253 } 4254 4255 static unsigned int perf_poll(struct file *file, poll_table *wait) 4256 { 4257 struct perf_event *event = file->private_data; 4258 struct ring_buffer *rb; 4259 unsigned int events = POLLHUP; 4260 4261 poll_wait(file, &event->waitq, wait); 4262 4263 if (is_event_hup(event)) 4264 return events; 4265 4266 /* 4267 * Pin the event->rb by taking event->mmap_mutex; otherwise 4268 * perf_event_set_output() can swizzle our rb and make us miss wakeups. 4269 */ 4270 mutex_lock(&event->mmap_mutex); 4271 rb = event->rb; 4272 if (rb) 4273 events = atomic_xchg(&rb->poll, 0); 4274 mutex_unlock(&event->mmap_mutex); 4275 return events; 4276 } 4277 4278 static void _perf_event_reset(struct perf_event *event) 4279 { 4280 (void)perf_event_read(event, false); 4281 local64_set(&event->count, 0); 4282 perf_event_update_userpage(event); 4283 } 4284 4285 /* 4286 * Holding the top-level event's child_mutex means that any 4287 * descendant process that has inherited this event will block 4288 * in perf_event_exit_event() if it goes to exit, thus satisfying the 4289 * task existence requirements of perf_event_enable/disable. 4290 */ 4291 static void perf_event_for_each_child(struct perf_event *event, 4292 void (*func)(struct perf_event *)) 4293 { 4294 struct perf_event *child; 4295 4296 WARN_ON_ONCE(event->ctx->parent_ctx); 4297 4298 mutex_lock(&event->child_mutex); 4299 func(event); 4300 list_for_each_entry(child, &event->child_list, child_list) 4301 func(child); 4302 mutex_unlock(&event->child_mutex); 4303 } 4304 4305 static void perf_event_for_each(struct perf_event *event, 4306 void (*func)(struct perf_event *)) 4307 { 4308 struct perf_event_context *ctx = event->ctx; 4309 struct perf_event *sibling; 4310 4311 lockdep_assert_held(&ctx->mutex); 4312 4313 event = event->group_leader; 4314 4315 perf_event_for_each_child(event, func); 4316 list_for_each_entry(sibling, &event->sibling_list, group_entry) 4317 perf_event_for_each_child(sibling, func); 4318 } 4319 4320 static void __perf_event_period(struct perf_event *event, 4321 struct perf_cpu_context *cpuctx, 4322 struct perf_event_context *ctx, 4323 void *info) 4324 { 4325 u64 value = *((u64 *)info); 4326 bool active; 4327 4328 if (event->attr.freq) { 4329 event->attr.sample_freq = value; 4330 } else { 4331 event->attr.sample_period = value; 4332 event->hw.sample_period = value; 4333 } 4334 4335 active = (event->state == PERF_EVENT_STATE_ACTIVE); 4336 if (active) { 4337 perf_pmu_disable(ctx->pmu); 4338 /* 4339 * We could be throttled; unthrottle now to avoid the tick 4340 * trying to unthrottle while we already re-started the event. 4341 */ 4342 if (event->hw.interrupts == MAX_INTERRUPTS) { 4343 event->hw.interrupts = 0; 4344 perf_log_throttle(event, 1); 4345 } 4346 event->pmu->stop(event, PERF_EF_UPDATE); 4347 } 4348 4349 local64_set(&event->hw.period_left, 0); 4350 4351 if (active) { 4352 event->pmu->start(event, PERF_EF_RELOAD); 4353 perf_pmu_enable(ctx->pmu); 4354 } 4355 } 4356 4357 static int perf_event_period(struct perf_event *event, u64 __user *arg) 4358 { 4359 u64 value; 4360 4361 if (!is_sampling_event(event)) 4362 return -EINVAL; 4363 4364 if (copy_from_user(&value, arg, sizeof(value))) 4365 return -EFAULT; 4366 4367 if (!value) 4368 return -EINVAL; 4369 4370 if (event->attr.freq && value > sysctl_perf_event_sample_rate) 4371 return -EINVAL; 4372 4373 event_function_call(event, __perf_event_period, &value); 4374 4375 return 0; 4376 } 4377 4378 static const struct file_operations perf_fops; 4379 4380 static inline int perf_fget_light(int fd, struct fd *p) 4381 { 4382 struct fd f = fdget(fd); 4383 if (!f.file) 4384 return -EBADF; 4385 4386 if (f.file->f_op != &perf_fops) { 4387 fdput(f); 4388 return -EBADF; 4389 } 4390 *p = f; 4391 return 0; 4392 } 4393 4394 static int perf_event_set_output(struct perf_event *event, 4395 struct perf_event *output_event); 4396 static int perf_event_set_filter(struct perf_event *event, void __user *arg); 4397 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd); 4398 4399 static long _perf_ioctl(struct perf_event *event, unsigned int cmd, unsigned long arg) 4400 { 4401 void (*func)(struct perf_event *); 4402 u32 flags = arg; 4403 4404 switch (cmd) { 4405 case PERF_EVENT_IOC_ENABLE: 4406 func = _perf_event_enable; 4407 break; 4408 case PERF_EVENT_IOC_DISABLE: 4409 func = _perf_event_disable; 4410 break; 4411 case PERF_EVENT_IOC_RESET: 4412 func = _perf_event_reset; 4413 break; 4414 4415 case PERF_EVENT_IOC_REFRESH: 4416 return _perf_event_refresh(event, arg); 4417 4418 case PERF_EVENT_IOC_PERIOD: 4419 return perf_event_period(event, (u64 __user *)arg); 4420 4421 case PERF_EVENT_IOC_ID: 4422 { 4423 u64 id = primary_event_id(event); 4424 4425 if (copy_to_user((void __user *)arg, &id, sizeof(id))) 4426 return -EFAULT; 4427 return 0; 4428 } 4429 4430 case PERF_EVENT_IOC_SET_OUTPUT: 4431 { 4432 int ret; 4433 if (arg != -1) { 4434 struct perf_event *output_event; 4435 struct fd output; 4436 ret = perf_fget_light(arg, &output); 4437 if (ret) 4438 return ret; 4439 output_event = output.file->private_data; 4440 ret = perf_event_set_output(event, output_event); 4441 fdput(output); 4442 } else { 4443 ret = perf_event_set_output(event, NULL); 4444 } 4445 return ret; 4446 } 4447 4448 case PERF_EVENT_IOC_SET_FILTER: 4449 return perf_event_set_filter(event, (void __user *)arg); 4450 4451 case PERF_EVENT_IOC_SET_BPF: 4452 return perf_event_set_bpf_prog(event, arg); 4453 4454 case PERF_EVENT_IOC_PAUSE_OUTPUT: { 4455 struct ring_buffer *rb; 4456 4457 rcu_read_lock(); 4458 rb = rcu_dereference(event->rb); 4459 if (!rb || !rb->nr_pages) { 4460 rcu_read_unlock(); 4461 return -EINVAL; 4462 } 4463 rb_toggle_paused(rb, !!arg); 4464 rcu_read_unlock(); 4465 return 0; 4466 } 4467 default: 4468 return -ENOTTY; 4469 } 4470 4471 if (flags & PERF_IOC_FLAG_GROUP) 4472 perf_event_for_each(event, func); 4473 else 4474 perf_event_for_each_child(event, func); 4475 4476 return 0; 4477 } 4478 4479 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 4480 { 4481 struct perf_event *event = file->private_data; 4482 struct perf_event_context *ctx; 4483 long ret; 4484 4485 ctx = perf_event_ctx_lock(event); 4486 ret = _perf_ioctl(event, cmd, arg); 4487 perf_event_ctx_unlock(event, ctx); 4488 4489 return ret; 4490 } 4491 4492 #ifdef CONFIG_COMPAT 4493 static long perf_compat_ioctl(struct file *file, unsigned int cmd, 4494 unsigned long arg) 4495 { 4496 switch (_IOC_NR(cmd)) { 4497 case _IOC_NR(PERF_EVENT_IOC_SET_FILTER): 4498 case _IOC_NR(PERF_EVENT_IOC_ID): 4499 /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */ 4500 if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) { 4501 cmd &= ~IOCSIZE_MASK; 4502 cmd |= sizeof(void *) << IOCSIZE_SHIFT; 4503 } 4504 break; 4505 } 4506 return perf_ioctl(file, cmd, arg); 4507 } 4508 #else 4509 # define perf_compat_ioctl NULL 4510 #endif 4511 4512 int perf_event_task_enable(void) 4513 { 4514 struct perf_event_context *ctx; 4515 struct perf_event *event; 4516 4517 mutex_lock(¤t->perf_event_mutex); 4518 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 4519 ctx = perf_event_ctx_lock(event); 4520 perf_event_for_each_child(event, _perf_event_enable); 4521 perf_event_ctx_unlock(event, ctx); 4522 } 4523 mutex_unlock(¤t->perf_event_mutex); 4524 4525 return 0; 4526 } 4527 4528 int perf_event_task_disable(void) 4529 { 4530 struct perf_event_context *ctx; 4531 struct perf_event *event; 4532 4533 mutex_lock(¤t->perf_event_mutex); 4534 list_for_each_entry(event, ¤t->perf_event_list, owner_entry) { 4535 ctx = perf_event_ctx_lock(event); 4536 perf_event_for_each_child(event, _perf_event_disable); 4537 perf_event_ctx_unlock(event, ctx); 4538 } 4539 mutex_unlock(¤t->perf_event_mutex); 4540 4541 return 0; 4542 } 4543 4544 static int perf_event_index(struct perf_event *event) 4545 { 4546 if (event->hw.state & PERF_HES_STOPPED) 4547 return 0; 4548 4549 if (event->state != PERF_EVENT_STATE_ACTIVE) 4550 return 0; 4551 4552 return event->pmu->event_idx(event); 4553 } 4554 4555 static void calc_timer_values(struct perf_event *event, 4556 u64 *now, 4557 u64 *enabled, 4558 u64 *running) 4559 { 4560 u64 ctx_time; 4561 4562 *now = perf_clock(); 4563 ctx_time = event->shadow_ctx_time + *now; 4564 *enabled = ctx_time - event->tstamp_enabled; 4565 *running = ctx_time - event->tstamp_running; 4566 } 4567 4568 static void perf_event_init_userpage(struct perf_event *event) 4569 { 4570 struct perf_event_mmap_page *userpg; 4571 struct ring_buffer *rb; 4572 4573 rcu_read_lock(); 4574 rb = rcu_dereference(event->rb); 4575 if (!rb) 4576 goto unlock; 4577 4578 userpg = rb->user_page; 4579 4580 /* Allow new userspace to detect that bit 0 is deprecated */ 4581 userpg->cap_bit0_is_deprecated = 1; 4582 userpg->size = offsetof(struct perf_event_mmap_page, __reserved); 4583 userpg->data_offset = PAGE_SIZE; 4584 userpg->data_size = perf_data_size(rb); 4585 4586 unlock: 4587 rcu_read_unlock(); 4588 } 4589 4590 void __weak arch_perf_update_userpage( 4591 struct perf_event *event, struct perf_event_mmap_page *userpg, u64 now) 4592 { 4593 } 4594 4595 /* 4596 * Callers need to ensure there can be no nesting of this function, otherwise 4597 * the seqlock logic goes bad. We can not serialize this because the arch 4598 * code calls this from NMI context. 4599 */ 4600 void perf_event_update_userpage(struct perf_event *event) 4601 { 4602 struct perf_event_mmap_page *userpg; 4603 struct ring_buffer *rb; 4604 u64 enabled, running, now; 4605 4606 rcu_read_lock(); 4607 rb = rcu_dereference(event->rb); 4608 if (!rb) 4609 goto unlock; 4610 4611 /* 4612 * compute total_time_enabled, total_time_running 4613 * based on snapshot values taken when the event 4614 * was last scheduled in. 4615 * 4616 * we cannot simply called update_context_time() 4617 * because of locking issue as we can be called in 4618 * NMI context 4619 */ 4620 calc_timer_values(event, &now, &enabled, &running); 4621 4622 userpg = rb->user_page; 4623 /* 4624 * Disable preemption so as to not let the corresponding user-space 4625 * spin too long if we get preempted. 4626 */ 4627 preempt_disable(); 4628 ++userpg->lock; 4629 barrier(); 4630 userpg->index = perf_event_index(event); 4631 userpg->offset = perf_event_count(event); 4632 if (userpg->index) 4633 userpg->offset -= local64_read(&event->hw.prev_count); 4634 4635 userpg->time_enabled = enabled + 4636 atomic64_read(&event->child_total_time_enabled); 4637 4638 userpg->time_running = running + 4639 atomic64_read(&event->child_total_time_running); 4640 4641 arch_perf_update_userpage(event, userpg, now); 4642 4643 barrier(); 4644 ++userpg->lock; 4645 preempt_enable(); 4646 unlock: 4647 rcu_read_unlock(); 4648 } 4649 4650 static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf) 4651 { 4652 struct perf_event *event = vma->vm_file->private_data; 4653 struct ring_buffer *rb; 4654 int ret = VM_FAULT_SIGBUS; 4655 4656 if (vmf->flags & FAULT_FLAG_MKWRITE) { 4657 if (vmf->pgoff == 0) 4658 ret = 0; 4659 return ret; 4660 } 4661 4662 rcu_read_lock(); 4663 rb = rcu_dereference(event->rb); 4664 if (!rb) 4665 goto unlock; 4666 4667 if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE)) 4668 goto unlock; 4669 4670 vmf->page = perf_mmap_to_page(rb, vmf->pgoff); 4671 if (!vmf->page) 4672 goto unlock; 4673 4674 get_page(vmf->page); 4675 vmf->page->mapping = vma->vm_file->f_mapping; 4676 vmf->page->index = vmf->pgoff; 4677 4678 ret = 0; 4679 unlock: 4680 rcu_read_unlock(); 4681 4682 return ret; 4683 } 4684 4685 static void ring_buffer_attach(struct perf_event *event, 4686 struct ring_buffer *rb) 4687 { 4688 struct ring_buffer *old_rb = NULL; 4689 unsigned long flags; 4690 4691 if (event->rb) { 4692 /* 4693 * Should be impossible, we set this when removing 4694 * event->rb_entry and wait/clear when adding event->rb_entry. 4695 */ 4696 WARN_ON_ONCE(event->rcu_pending); 4697 4698 old_rb = event->rb; 4699 spin_lock_irqsave(&old_rb->event_lock, flags); 4700 list_del_rcu(&event->rb_entry); 4701 spin_unlock_irqrestore(&old_rb->event_lock, flags); 4702 4703 event->rcu_batches = get_state_synchronize_rcu(); 4704 event->rcu_pending = 1; 4705 } 4706 4707 if (rb) { 4708 if (event->rcu_pending) { 4709 cond_synchronize_rcu(event->rcu_batches); 4710 event->rcu_pending = 0; 4711 } 4712 4713 spin_lock_irqsave(&rb->event_lock, flags); 4714 list_add_rcu(&event->rb_entry, &rb->event_list); 4715 spin_unlock_irqrestore(&rb->event_lock, flags); 4716 } 4717 4718 rcu_assign_pointer(event->rb, rb); 4719 4720 if (old_rb) { 4721 ring_buffer_put(old_rb); 4722 /* 4723 * Since we detached before setting the new rb, so that we 4724 * could attach the new rb, we could have missed a wakeup. 4725 * Provide it now. 4726 */ 4727 wake_up_all(&event->waitq); 4728 } 4729 } 4730 4731 static void ring_buffer_wakeup(struct perf_event *event) 4732 { 4733 struct ring_buffer *rb; 4734 4735 rcu_read_lock(); 4736 rb = rcu_dereference(event->rb); 4737 if (rb) { 4738 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) 4739 wake_up_all(&event->waitq); 4740 } 4741 rcu_read_unlock(); 4742 } 4743 4744 struct ring_buffer *ring_buffer_get(struct perf_event *event) 4745 { 4746 struct ring_buffer *rb; 4747 4748 rcu_read_lock(); 4749 rb = rcu_dereference(event->rb); 4750 if (rb) { 4751 if (!atomic_inc_not_zero(&rb->refcount)) 4752 rb = NULL; 4753 } 4754 rcu_read_unlock(); 4755 4756 return rb; 4757 } 4758 4759 void ring_buffer_put(struct ring_buffer *rb) 4760 { 4761 if (!atomic_dec_and_test(&rb->refcount)) 4762 return; 4763 4764 WARN_ON_ONCE(!list_empty(&rb->event_list)); 4765 4766 call_rcu(&rb->rcu_head, rb_free_rcu); 4767 } 4768 4769 static void perf_mmap_open(struct vm_area_struct *vma) 4770 { 4771 struct perf_event *event = vma->vm_file->private_data; 4772 4773 atomic_inc(&event->mmap_count); 4774 atomic_inc(&event->rb->mmap_count); 4775 4776 if (vma->vm_pgoff) 4777 atomic_inc(&event->rb->aux_mmap_count); 4778 4779 if (event->pmu->event_mapped) 4780 event->pmu->event_mapped(event); 4781 } 4782 4783 static void perf_pmu_output_stop(struct perf_event *event); 4784 4785 /* 4786 * A buffer can be mmap()ed multiple times; either directly through the same 4787 * event, or through other events by use of perf_event_set_output(). 4788 * 4789 * In order to undo the VM accounting done by perf_mmap() we need to destroy 4790 * the buffer here, where we still have a VM context. This means we need 4791 * to detach all events redirecting to us. 4792 */ 4793 static void perf_mmap_close(struct vm_area_struct *vma) 4794 { 4795 struct perf_event *event = vma->vm_file->private_data; 4796 4797 struct ring_buffer *rb = ring_buffer_get(event); 4798 struct user_struct *mmap_user = rb->mmap_user; 4799 int mmap_locked = rb->mmap_locked; 4800 unsigned long size = perf_data_size(rb); 4801 4802 if (event->pmu->event_unmapped) 4803 event->pmu->event_unmapped(event); 4804 4805 /* 4806 * rb->aux_mmap_count will always drop before rb->mmap_count and 4807 * event->mmap_count, so it is ok to use event->mmap_mutex to 4808 * serialize with perf_mmap here. 4809 */ 4810 if (rb_has_aux(rb) && vma->vm_pgoff == rb->aux_pgoff && 4811 atomic_dec_and_mutex_lock(&rb->aux_mmap_count, &event->mmap_mutex)) { 4812 /* 4813 * Stop all AUX events that are writing to this buffer, 4814 * so that we can free its AUX pages and corresponding PMU 4815 * data. Note that after rb::aux_mmap_count dropped to zero, 4816 * they won't start any more (see perf_aux_output_begin()). 4817 */ 4818 perf_pmu_output_stop(event); 4819 4820 /* now it's safe to free the pages */ 4821 atomic_long_sub(rb->aux_nr_pages, &mmap_user->locked_vm); 4822 vma->vm_mm->pinned_vm -= rb->aux_mmap_locked; 4823 4824 /* this has to be the last one */ 4825 rb_free_aux(rb); 4826 WARN_ON_ONCE(atomic_read(&rb->aux_refcount)); 4827 4828 mutex_unlock(&event->mmap_mutex); 4829 } 4830 4831 atomic_dec(&rb->mmap_count); 4832 4833 if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) 4834 goto out_put; 4835 4836 ring_buffer_attach(event, NULL); 4837 mutex_unlock(&event->mmap_mutex); 4838 4839 /* If there's still other mmap()s of this buffer, we're done. */ 4840 if (atomic_read(&rb->mmap_count)) 4841 goto out_put; 4842 4843 /* 4844 * No other mmap()s, detach from all other events that might redirect 4845 * into the now unreachable buffer. Somewhat complicated by the 4846 * fact that rb::event_lock otherwise nests inside mmap_mutex. 4847 */ 4848 again: 4849 rcu_read_lock(); 4850 list_for_each_entry_rcu(event, &rb->event_list, rb_entry) { 4851 if (!atomic_long_inc_not_zero(&event->refcount)) { 4852 /* 4853 * This event is en-route to free_event() which will 4854 * detach it and remove it from the list. 4855 */ 4856 continue; 4857 } 4858 rcu_read_unlock(); 4859 4860 mutex_lock(&event->mmap_mutex); 4861 /* 4862 * Check we didn't race with perf_event_set_output() which can 4863 * swizzle the rb from under us while we were waiting to 4864 * acquire mmap_mutex. 4865 * 4866 * If we find a different rb; ignore this event, a next 4867 * iteration will no longer find it on the list. We have to 4868 * still restart the iteration to make sure we're not now 4869 * iterating the wrong list. 4870 */ 4871 if (event->rb == rb) 4872 ring_buffer_attach(event, NULL); 4873 4874 mutex_unlock(&event->mmap_mutex); 4875 put_event(event); 4876 4877 /* 4878 * Restart the iteration; either we're on the wrong list or 4879 * destroyed its integrity by doing a deletion. 4880 */ 4881 goto again; 4882 } 4883 rcu_read_unlock(); 4884 4885 /* 4886 * It could be there's still a few 0-ref events on the list; they'll 4887 * get cleaned up by free_event() -- they'll also still have their 4888 * ref on the rb and will free it whenever they are done with it. 4889 * 4890 * Aside from that, this buffer is 'fully' detached and unmapped, 4891 * undo the VM accounting. 4892 */ 4893 4894 atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm); 4895 vma->vm_mm->pinned_vm -= mmap_locked; 4896 free_uid(mmap_user); 4897 4898 out_put: 4899 ring_buffer_put(rb); /* could be last */ 4900 } 4901 4902 static const struct vm_operations_struct perf_mmap_vmops = { 4903 .open = perf_mmap_open, 4904 .close = perf_mmap_close, /* non mergable */ 4905 .fault = perf_mmap_fault, 4906 .page_mkwrite = perf_mmap_fault, 4907 }; 4908 4909 static int perf_mmap(struct file *file, struct vm_area_struct *vma) 4910 { 4911 struct perf_event *event = file->private_data; 4912 unsigned long user_locked, user_lock_limit; 4913 struct user_struct *user = current_user(); 4914 unsigned long locked, lock_limit; 4915 struct ring_buffer *rb = NULL; 4916 unsigned long vma_size; 4917 unsigned long nr_pages; 4918 long user_extra = 0, extra = 0; 4919 int ret = 0, flags = 0; 4920 4921 /* 4922 * Don't allow mmap() of inherited per-task counters. This would 4923 * create a performance issue due to all children writing to the 4924 * same rb. 4925 */ 4926 if (event->cpu == -1 && event->attr.inherit) 4927 return -EINVAL; 4928 4929 if (!(vma->vm_flags & VM_SHARED)) 4930 return -EINVAL; 4931 4932 vma_size = vma->vm_end - vma->vm_start; 4933 4934 if (vma->vm_pgoff == 0) { 4935 nr_pages = (vma_size / PAGE_SIZE) - 1; 4936 } else { 4937 /* 4938 * AUX area mapping: if rb->aux_nr_pages != 0, it's already 4939 * mapped, all subsequent mappings should have the same size 4940 * and offset. Must be above the normal perf buffer. 4941 */ 4942 u64 aux_offset, aux_size; 4943 4944 if (!event->rb) 4945 return -EINVAL; 4946 4947 nr_pages = vma_size / PAGE_SIZE; 4948 4949 mutex_lock(&event->mmap_mutex); 4950 ret = -EINVAL; 4951 4952 rb = event->rb; 4953 if (!rb) 4954 goto aux_unlock; 4955 4956 aux_offset = ACCESS_ONCE(rb->user_page->aux_offset); 4957 aux_size = ACCESS_ONCE(rb->user_page->aux_size); 4958 4959 if (aux_offset < perf_data_size(rb) + PAGE_SIZE) 4960 goto aux_unlock; 4961 4962 if (aux_offset != vma->vm_pgoff << PAGE_SHIFT) 4963 goto aux_unlock; 4964 4965 /* already mapped with a different offset */ 4966 if (rb_has_aux(rb) && rb->aux_pgoff != vma->vm_pgoff) 4967 goto aux_unlock; 4968 4969 if (aux_size != vma_size || aux_size != nr_pages * PAGE_SIZE) 4970 goto aux_unlock; 4971 4972 /* already mapped with a different size */ 4973 if (rb_has_aux(rb) && rb->aux_nr_pages != nr_pages) 4974 goto aux_unlock; 4975 4976 if (!is_power_of_2(nr_pages)) 4977 goto aux_unlock; 4978 4979 if (!atomic_inc_not_zero(&rb->mmap_count)) 4980 goto aux_unlock; 4981 4982 if (rb_has_aux(rb)) { 4983 atomic_inc(&rb->aux_mmap_count); 4984 ret = 0; 4985 goto unlock; 4986 } 4987 4988 atomic_set(&rb->aux_mmap_count, 1); 4989 user_extra = nr_pages; 4990 4991 goto accounting; 4992 } 4993 4994 /* 4995 * If we have rb pages ensure they're a power-of-two number, so we 4996 * can do bitmasks instead of modulo. 4997 */ 4998 if (nr_pages != 0 && !is_power_of_2(nr_pages)) 4999 return -EINVAL; 5000 5001 if (vma_size != PAGE_SIZE * (1 + nr_pages)) 5002 return -EINVAL; 5003 5004 WARN_ON_ONCE(event->ctx->parent_ctx); 5005 again: 5006 mutex_lock(&event->mmap_mutex); 5007 if (event->rb) { 5008 if (event->rb->nr_pages != nr_pages) { 5009 ret = -EINVAL; 5010 goto unlock; 5011 } 5012 5013 if (!atomic_inc_not_zero(&event->rb->mmap_count)) { 5014 /* 5015 * Raced against perf_mmap_close() through 5016 * perf_event_set_output(). Try again, hope for better 5017 * luck. 5018 */ 5019 mutex_unlock(&event->mmap_mutex); 5020 goto again; 5021 } 5022 5023 goto unlock; 5024 } 5025 5026 user_extra = nr_pages + 1; 5027 5028 accounting: 5029 user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10); 5030 5031 /* 5032 * Increase the limit linearly with more CPUs: 5033 */ 5034 user_lock_limit *= num_online_cpus(); 5035 5036 user_locked = atomic_long_read(&user->locked_vm) + user_extra; 5037 5038 if (user_locked > user_lock_limit) 5039 extra = user_locked - user_lock_limit; 5040 5041 lock_limit = rlimit(RLIMIT_MEMLOCK); 5042 lock_limit >>= PAGE_SHIFT; 5043 locked = vma->vm_mm->pinned_vm + extra; 5044 5045 if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() && 5046 !capable(CAP_IPC_LOCK)) { 5047 ret = -EPERM; 5048 goto unlock; 5049 } 5050 5051 WARN_ON(!rb && event->rb); 5052 5053 if (vma->vm_flags & VM_WRITE) 5054 flags |= RING_BUFFER_WRITABLE; 5055 5056 if (!rb) { 5057 rb = rb_alloc(nr_pages, 5058 event->attr.watermark ? event->attr.wakeup_watermark : 0, 5059 event->cpu, flags); 5060 5061 if (!rb) { 5062 ret = -ENOMEM; 5063 goto unlock; 5064 } 5065 5066 atomic_set(&rb->mmap_count, 1); 5067 rb->mmap_user = get_current_user(); 5068 rb->mmap_locked = extra; 5069 5070 ring_buffer_attach(event, rb); 5071 5072 perf_event_init_userpage(event); 5073 perf_event_update_userpage(event); 5074 } else { 5075 ret = rb_alloc_aux(rb, event, vma->vm_pgoff, nr_pages, 5076 event->attr.aux_watermark, flags); 5077 if (!ret) 5078 rb->aux_mmap_locked = extra; 5079 } 5080 5081 unlock: 5082 if (!ret) { 5083 atomic_long_add(user_extra, &user->locked_vm); 5084 vma->vm_mm->pinned_vm += extra; 5085 5086 atomic_inc(&event->mmap_count); 5087 } else if (rb) { 5088 atomic_dec(&rb->mmap_count); 5089 } 5090 aux_unlock: 5091 mutex_unlock(&event->mmap_mutex); 5092 5093 /* 5094 * Since pinned accounting is per vm we cannot allow fork() to copy our 5095 * vma. 5096 */ 5097 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP; 5098 vma->vm_ops = &perf_mmap_vmops; 5099 5100 if (event->pmu->event_mapped) 5101 event->pmu->event_mapped(event); 5102 5103 return ret; 5104 } 5105 5106 static int perf_fasync(int fd, struct file *filp, int on) 5107 { 5108 struct inode *inode = file_inode(filp); 5109 struct perf_event *event = filp->private_data; 5110 int retval; 5111 5112 inode_lock(inode); 5113 retval = fasync_helper(fd, filp, on, &event->fasync); 5114 inode_unlock(inode); 5115 5116 if (retval < 0) 5117 return retval; 5118 5119 return 0; 5120 } 5121 5122 static const struct file_operations perf_fops = { 5123 .llseek = no_llseek, 5124 .release = perf_release, 5125 .read = perf_read, 5126 .poll = perf_poll, 5127 .unlocked_ioctl = perf_ioctl, 5128 .compat_ioctl = perf_compat_ioctl, 5129 .mmap = perf_mmap, 5130 .fasync = perf_fasync, 5131 }; 5132 5133 /* 5134 * Perf event wakeup 5135 * 5136 * If there's data, ensure we set the poll() state and publish everything 5137 * to user-space before waking everybody up. 5138 */ 5139 5140 static inline struct fasync_struct **perf_event_fasync(struct perf_event *event) 5141 { 5142 /* only the parent has fasync state */ 5143 if (event->parent) 5144 event = event->parent; 5145 return &event->fasync; 5146 } 5147 5148 void perf_event_wakeup(struct perf_event *event) 5149 { 5150 ring_buffer_wakeup(event); 5151 5152 if (event->pending_kill) { 5153 kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill); 5154 event->pending_kill = 0; 5155 } 5156 } 5157 5158 static void perf_pending_event(struct irq_work *entry) 5159 { 5160 struct perf_event *event = container_of(entry, 5161 struct perf_event, pending); 5162 int rctx; 5163 5164 rctx = perf_swevent_get_recursion_context(); 5165 /* 5166 * If we 'fail' here, that's OK, it means recursion is already disabled 5167 * and we won't recurse 'further'. 5168 */ 5169 5170 if (event->pending_disable) { 5171 event->pending_disable = 0; 5172 perf_event_disable_local(event); 5173 } 5174 5175 if (event->pending_wakeup) { 5176 event->pending_wakeup = 0; 5177 perf_event_wakeup(event); 5178 } 5179 5180 if (rctx >= 0) 5181 perf_swevent_put_recursion_context(rctx); 5182 } 5183 5184 /* 5185 * We assume there is only KVM supporting the callbacks. 5186 * Later on, we might change it to a list if there is 5187 * another virtualization implementation supporting the callbacks. 5188 */ 5189 struct perf_guest_info_callbacks *perf_guest_cbs; 5190 5191 int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 5192 { 5193 perf_guest_cbs = cbs; 5194 return 0; 5195 } 5196 EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks); 5197 5198 int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs) 5199 { 5200 perf_guest_cbs = NULL; 5201 return 0; 5202 } 5203 EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks); 5204 5205 static void 5206 perf_output_sample_regs(struct perf_output_handle *handle, 5207 struct pt_regs *regs, u64 mask) 5208 { 5209 int bit; 5210 5211 for_each_set_bit(bit, (const unsigned long *) &mask, 5212 sizeof(mask) * BITS_PER_BYTE) { 5213 u64 val; 5214 5215 val = perf_reg_value(regs, bit); 5216 perf_output_put(handle, val); 5217 } 5218 } 5219 5220 static void perf_sample_regs_user(struct perf_regs *regs_user, 5221 struct pt_regs *regs, 5222 struct pt_regs *regs_user_copy) 5223 { 5224 if (user_mode(regs)) { 5225 regs_user->abi = perf_reg_abi(current); 5226 regs_user->regs = regs; 5227 } else if (current->mm) { 5228 perf_get_regs_user(regs_user, regs, regs_user_copy); 5229 } else { 5230 regs_user->abi = PERF_SAMPLE_REGS_ABI_NONE; 5231 regs_user->regs = NULL; 5232 } 5233 } 5234 5235 static void perf_sample_regs_intr(struct perf_regs *regs_intr, 5236 struct pt_regs *regs) 5237 { 5238 regs_intr->regs = regs; 5239 regs_intr->abi = perf_reg_abi(current); 5240 } 5241 5242 5243 /* 5244 * Get remaining task size from user stack pointer. 5245 * 5246 * It'd be better to take stack vma map and limit this more 5247 * precisly, but there's no way to get it safely under interrupt, 5248 * so using TASK_SIZE as limit. 5249 */ 5250 static u64 perf_ustack_task_size(struct pt_regs *regs) 5251 { 5252 unsigned long addr = perf_user_stack_pointer(regs); 5253 5254 if (!addr || addr >= TASK_SIZE) 5255 return 0; 5256 5257 return TASK_SIZE - addr; 5258 } 5259 5260 static u16 5261 perf_sample_ustack_size(u16 stack_size, u16 header_size, 5262 struct pt_regs *regs) 5263 { 5264 u64 task_size; 5265 5266 /* No regs, no stack pointer, no dump. */ 5267 if (!regs) 5268 return 0; 5269 5270 /* 5271 * Check if we fit in with the requested stack size into the: 5272 * - TASK_SIZE 5273 * If we don't, we limit the size to the TASK_SIZE. 5274 * 5275 * - remaining sample size 5276 * If we don't, we customize the stack size to 5277 * fit in to the remaining sample size. 5278 */ 5279 5280 task_size = min((u64) USHRT_MAX, perf_ustack_task_size(regs)); 5281 stack_size = min(stack_size, (u16) task_size); 5282 5283 /* Current header size plus static size and dynamic size. */ 5284 header_size += 2 * sizeof(u64); 5285 5286 /* Do we fit in with the current stack dump size? */ 5287 if ((u16) (header_size + stack_size) < header_size) { 5288 /* 5289 * If we overflow the maximum size for the sample, 5290 * we customize the stack dump size to fit in. 5291 */ 5292 stack_size = USHRT_MAX - header_size - sizeof(u64); 5293 stack_size = round_up(stack_size, sizeof(u64)); 5294 } 5295 5296 return stack_size; 5297 } 5298 5299 static void 5300 perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size, 5301 struct pt_regs *regs) 5302 { 5303 /* Case of a kernel thread, nothing to dump */ 5304 if (!regs) { 5305 u64 size = 0; 5306 perf_output_put(handle, size); 5307 } else { 5308 unsigned long sp; 5309 unsigned int rem; 5310 u64 dyn_size; 5311 5312 /* 5313 * We dump: 5314 * static size 5315 * - the size requested by user or the best one we can fit 5316 * in to the sample max size 5317 * data 5318 * - user stack dump data 5319 * dynamic size 5320 * - the actual dumped size 5321 */ 5322 5323 /* Static size. */ 5324 perf_output_put(handle, dump_size); 5325 5326 /* Data. */ 5327 sp = perf_user_stack_pointer(regs); 5328 rem = __output_copy_user(handle, (void *) sp, dump_size); 5329 dyn_size = dump_size - rem; 5330 5331 perf_output_skip(handle, rem); 5332 5333 /* Dynamic size. */ 5334 perf_output_put(handle, dyn_size); 5335 } 5336 } 5337 5338 static void __perf_event_header__init_id(struct perf_event_header *header, 5339 struct perf_sample_data *data, 5340 struct perf_event *event) 5341 { 5342 u64 sample_type = event->attr.sample_type; 5343 5344 data->type = sample_type; 5345 header->size += event->id_header_size; 5346 5347 if (sample_type & PERF_SAMPLE_TID) { 5348 /* namespace issues */ 5349 data->tid_entry.pid = perf_event_pid(event, current); 5350 data->tid_entry.tid = perf_event_tid(event, current); 5351 } 5352 5353 if (sample_type & PERF_SAMPLE_TIME) 5354 data->time = perf_event_clock(event); 5355 5356 if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) 5357 data->id = primary_event_id(event); 5358 5359 if (sample_type & PERF_SAMPLE_STREAM_ID) 5360 data->stream_id = event->id; 5361 5362 if (sample_type & PERF_SAMPLE_CPU) { 5363 data->cpu_entry.cpu = raw_smp_processor_id(); 5364 data->cpu_entry.reserved = 0; 5365 } 5366 } 5367 5368 void perf_event_header__init_id(struct perf_event_header *header, 5369 struct perf_sample_data *data, 5370 struct perf_event *event) 5371 { 5372 if (event->attr.sample_id_all) 5373 __perf_event_header__init_id(header, data, event); 5374 } 5375 5376 static void __perf_event__output_id_sample(struct perf_output_handle *handle, 5377 struct perf_sample_data *data) 5378 { 5379 u64 sample_type = data->type; 5380 5381 if (sample_type & PERF_SAMPLE_TID) 5382 perf_output_put(handle, data->tid_entry); 5383 5384 if (sample_type & PERF_SAMPLE_TIME) 5385 perf_output_put(handle, data->time); 5386 5387 if (sample_type & PERF_SAMPLE_ID) 5388 perf_output_put(handle, data->id); 5389 5390 if (sample_type & PERF_SAMPLE_STREAM_ID) 5391 perf_output_put(handle, data->stream_id); 5392 5393 if (sample_type & PERF_SAMPLE_CPU) 5394 perf_output_put(handle, data->cpu_entry); 5395 5396 if (sample_type & PERF_SAMPLE_IDENTIFIER) 5397 perf_output_put(handle, data->id); 5398 } 5399 5400 void perf_event__output_id_sample(struct perf_event *event, 5401 struct perf_output_handle *handle, 5402 struct perf_sample_data *sample) 5403 { 5404 if (event->attr.sample_id_all) 5405 __perf_event__output_id_sample(handle, sample); 5406 } 5407 5408 static void perf_output_read_one(struct perf_output_handle *handle, 5409 struct perf_event *event, 5410 u64 enabled, u64 running) 5411 { 5412 u64 read_format = event->attr.read_format; 5413 u64 values[4]; 5414 int n = 0; 5415 5416 values[n++] = perf_event_count(event); 5417 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { 5418 values[n++] = enabled + 5419 atomic64_read(&event->child_total_time_enabled); 5420 } 5421 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { 5422 values[n++] = running + 5423 atomic64_read(&event->child_total_time_running); 5424 } 5425 if (read_format & PERF_FORMAT_ID) 5426 values[n++] = primary_event_id(event); 5427 5428 __output_copy(handle, values, n * sizeof(u64)); 5429 } 5430 5431 /* 5432 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult. 5433 */ 5434 static void perf_output_read_group(struct perf_output_handle *handle, 5435 struct perf_event *event, 5436 u64 enabled, u64 running) 5437 { 5438 struct perf_event *leader = event->group_leader, *sub; 5439 u64 read_format = event->attr.read_format; 5440 u64 values[5]; 5441 int n = 0; 5442 5443 values[n++] = 1 + leader->nr_siblings; 5444 5445 if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) 5446 values[n++] = enabled; 5447 5448 if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) 5449 values[n++] = running; 5450 5451 if (leader != event) 5452 leader->pmu->read(leader); 5453 5454 values[n++] = perf_event_count(leader); 5455 if (read_format & PERF_FORMAT_ID) 5456 values[n++] = primary_event_id(leader); 5457 5458 __output_copy(handle, values, n * sizeof(u64)); 5459 5460 list_for_each_entry(sub, &leader->sibling_list, group_entry) { 5461 n = 0; 5462 5463 if ((sub != event) && 5464 (sub->state == PERF_EVENT_STATE_ACTIVE)) 5465 sub->pmu->read(sub); 5466 5467 values[n++] = perf_event_count(sub); 5468 if (read_format & PERF_FORMAT_ID) 5469 values[n++] = primary_event_id(sub); 5470 5471 __output_copy(handle, values, n * sizeof(u64)); 5472 } 5473 } 5474 5475 #define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\ 5476 PERF_FORMAT_TOTAL_TIME_RUNNING) 5477 5478 static void perf_output_read(struct perf_output_handle *handle, 5479 struct perf_event *event) 5480 { 5481 u64 enabled = 0, running = 0, now; 5482 u64 read_format = event->attr.read_format; 5483 5484 /* 5485 * compute total_time_enabled, total_time_running 5486 * based on snapshot values taken when the event 5487 * was last scheduled in. 5488 * 5489 * we cannot simply called update_context_time() 5490 * because of locking issue as we are called in 5491 * NMI context 5492 */ 5493 if (read_format & PERF_FORMAT_TOTAL_TIMES) 5494 calc_timer_values(event, &now, &enabled, &running); 5495 5496 if (event->attr.read_format & PERF_FORMAT_GROUP) 5497 perf_output_read_group(handle, event, enabled, running); 5498 else 5499 perf_output_read_one(handle, event, enabled, running); 5500 } 5501 5502 void perf_output_sample(struct perf_output_handle *handle, 5503 struct perf_event_header *header, 5504 struct perf_sample_data *data, 5505 struct perf_event *event) 5506 { 5507 u64 sample_type = data->type; 5508 5509 perf_output_put(handle, *header); 5510 5511 if (sample_type & PERF_SAMPLE_IDENTIFIER) 5512 perf_output_put(handle, data->id); 5513 5514 if (sample_type & PERF_SAMPLE_IP) 5515 perf_output_put(handle, data->ip); 5516 5517 if (sample_type & PERF_SAMPLE_TID) 5518 perf_output_put(handle, data->tid_entry); 5519 5520 if (sample_type & PERF_SAMPLE_TIME) 5521 perf_output_put(handle, data->time); 5522 5523 if (sample_type & PERF_SAMPLE_ADDR) 5524 perf_output_put(handle, data->addr); 5525 5526 if (sample_type & PERF_SAMPLE_ID) 5527 perf_output_put(handle, data->id); 5528 5529 if (sample_type & PERF_SAMPLE_STREAM_ID) 5530 perf_output_put(handle, data->stream_id); 5531 5532 if (sample_type & PERF_SAMPLE_CPU) 5533 perf_output_put(handle, data->cpu_entry); 5534 5535 if (sample_type & PERF_SAMPLE_PERIOD) 5536 perf_output_put(handle, data->period); 5537 5538 if (sample_type & PERF_SAMPLE_READ) 5539 perf_output_read(handle, event); 5540 5541 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 5542 if (data->callchain) { 5543 int size = 1; 5544 5545 if (data->callchain) 5546 size += data->callchain->nr; 5547 5548 size *= sizeof(u64); 5549 5550 __output_copy(handle, data->callchain, size); 5551 } else { 5552 u64 nr = 0; 5553 perf_output_put(handle, nr); 5554 } 5555 } 5556 5557 if (sample_type & PERF_SAMPLE_RAW) { 5558 if (data->raw) { 5559 u32 raw_size = data->raw->size; 5560 u32 real_size = round_up(raw_size + sizeof(u32), 5561 sizeof(u64)) - sizeof(u32); 5562 u64 zero = 0; 5563 5564 perf_output_put(handle, real_size); 5565 __output_copy(handle, data->raw->data, raw_size); 5566 if (real_size - raw_size) 5567 __output_copy(handle, &zero, real_size - raw_size); 5568 } else { 5569 struct { 5570 u32 size; 5571 u32 data; 5572 } raw = { 5573 .size = sizeof(u32), 5574 .data = 0, 5575 }; 5576 perf_output_put(handle, raw); 5577 } 5578 } 5579 5580 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 5581 if (data->br_stack) { 5582 size_t size; 5583 5584 size = data->br_stack->nr 5585 * sizeof(struct perf_branch_entry); 5586 5587 perf_output_put(handle, data->br_stack->nr); 5588 perf_output_copy(handle, data->br_stack->entries, size); 5589 } else { 5590 /* 5591 * we always store at least the value of nr 5592 */ 5593 u64 nr = 0; 5594 perf_output_put(handle, nr); 5595 } 5596 } 5597 5598 if (sample_type & PERF_SAMPLE_REGS_USER) { 5599 u64 abi = data->regs_user.abi; 5600 5601 /* 5602 * If there are no regs to dump, notice it through 5603 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 5604 */ 5605 perf_output_put(handle, abi); 5606 5607 if (abi) { 5608 u64 mask = event->attr.sample_regs_user; 5609 perf_output_sample_regs(handle, 5610 data->regs_user.regs, 5611 mask); 5612 } 5613 } 5614 5615 if (sample_type & PERF_SAMPLE_STACK_USER) { 5616 perf_output_sample_ustack(handle, 5617 data->stack_user_size, 5618 data->regs_user.regs); 5619 } 5620 5621 if (sample_type & PERF_SAMPLE_WEIGHT) 5622 perf_output_put(handle, data->weight); 5623 5624 if (sample_type & PERF_SAMPLE_DATA_SRC) 5625 perf_output_put(handle, data->data_src.val); 5626 5627 if (sample_type & PERF_SAMPLE_TRANSACTION) 5628 perf_output_put(handle, data->txn); 5629 5630 if (sample_type & PERF_SAMPLE_REGS_INTR) { 5631 u64 abi = data->regs_intr.abi; 5632 /* 5633 * If there are no regs to dump, notice it through 5634 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE). 5635 */ 5636 perf_output_put(handle, abi); 5637 5638 if (abi) { 5639 u64 mask = event->attr.sample_regs_intr; 5640 5641 perf_output_sample_regs(handle, 5642 data->regs_intr.regs, 5643 mask); 5644 } 5645 } 5646 5647 if (!event->attr.watermark) { 5648 int wakeup_events = event->attr.wakeup_events; 5649 5650 if (wakeup_events) { 5651 struct ring_buffer *rb = handle->rb; 5652 int events = local_inc_return(&rb->events); 5653 5654 if (events >= wakeup_events) { 5655 local_sub(wakeup_events, &rb->events); 5656 local_inc(&rb->wakeup); 5657 } 5658 } 5659 } 5660 } 5661 5662 void perf_prepare_sample(struct perf_event_header *header, 5663 struct perf_sample_data *data, 5664 struct perf_event *event, 5665 struct pt_regs *regs) 5666 { 5667 u64 sample_type = event->attr.sample_type; 5668 5669 header->type = PERF_RECORD_SAMPLE; 5670 header->size = sizeof(*header) + event->header_size; 5671 5672 header->misc = 0; 5673 header->misc |= perf_misc_flags(regs); 5674 5675 __perf_event_header__init_id(header, data, event); 5676 5677 if (sample_type & PERF_SAMPLE_IP) 5678 data->ip = perf_instruction_pointer(regs); 5679 5680 if (sample_type & PERF_SAMPLE_CALLCHAIN) { 5681 int size = 1; 5682 5683 data->callchain = perf_callchain(event, regs); 5684 5685 if (data->callchain) 5686 size += data->callchain->nr; 5687 5688 header->size += size * sizeof(u64); 5689 } 5690 5691 if (sample_type & PERF_SAMPLE_RAW) { 5692 int size = sizeof(u32); 5693 5694 if (data->raw) 5695 size += data->raw->size; 5696 else 5697 size += sizeof(u32); 5698 5699 header->size += round_up(size, sizeof(u64)); 5700 } 5701 5702 if (sample_type & PERF_SAMPLE_BRANCH_STACK) { 5703 int size = sizeof(u64); /* nr */ 5704 if (data->br_stack) { 5705 size += data->br_stack->nr 5706 * sizeof(struct perf_branch_entry); 5707 } 5708 header->size += size; 5709 } 5710 5711 if (sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER)) 5712 perf_sample_regs_user(&data->regs_user, regs, 5713 &data->regs_user_copy); 5714 5715 if (sample_type & PERF_SAMPLE_REGS_USER) { 5716 /* regs dump ABI info */ 5717 int size = sizeof(u64); 5718 5719 if (data->regs_user.regs) { 5720 u64 mask = event->attr.sample_regs_user; 5721 size += hweight64(mask) * sizeof(u64); 5722 } 5723 5724 header->size += size; 5725 } 5726 5727 if (sample_type & PERF_SAMPLE_STACK_USER) { 5728 /* 5729 * Either we need PERF_SAMPLE_STACK_USER bit to be allways 5730 * processed as the last one or have additional check added 5731 * in case new sample type is added, because we could eat 5732 * up the rest of the sample size. 5733 */ 5734 u16 stack_size = event->attr.sample_stack_user; 5735 u16 size = sizeof(u64); 5736 5737 stack_size = perf_sample_ustack_size(stack_size, header->size, 5738 data->regs_user.regs); 5739 5740 /* 5741 * If there is something to dump, add space for the dump 5742 * itself and for the field that tells the dynamic size, 5743 * which is how many have been actually dumped. 5744 */ 5745 if (stack_size) 5746 size += sizeof(u64) + stack_size; 5747 5748 data->stack_user_size = stack_size; 5749 header->size += size; 5750 } 5751 5752 if (sample_type & PERF_SAMPLE_REGS_INTR) { 5753 /* regs dump ABI info */ 5754 int size = sizeof(u64); 5755 5756 perf_sample_regs_intr(&data->regs_intr, regs); 5757 5758 if (data->regs_intr.regs) { 5759 u64 mask = event->attr.sample_regs_intr; 5760 5761 size += hweight64(mask) * sizeof(u64); 5762 } 5763 5764 header->size += size; 5765 } 5766 } 5767 5768 static void __always_inline 5769 __perf_event_output(struct perf_event *event, 5770 struct perf_sample_data *data, 5771 struct pt_regs *regs, 5772 int (*output_begin)(struct perf_output_handle *, 5773 struct perf_event *, 5774 unsigned int)) 5775 { 5776 struct perf_output_handle handle; 5777 struct perf_event_header header; 5778 5779 /* protect the callchain buffers */ 5780 rcu_read_lock(); 5781 5782 perf_prepare_sample(&header, data, event, regs); 5783 5784 if (output_begin(&handle, event, header.size)) 5785 goto exit; 5786 5787 perf_output_sample(&handle, &header, data, event); 5788 5789 perf_output_end(&handle); 5790 5791 exit: 5792 rcu_read_unlock(); 5793 } 5794 5795 void 5796 perf_event_output_forward(struct perf_event *event, 5797 struct perf_sample_data *data, 5798 struct pt_regs *regs) 5799 { 5800 __perf_event_output(event, data, regs, perf_output_begin_forward); 5801 } 5802 5803 void 5804 perf_event_output_backward(struct perf_event *event, 5805 struct perf_sample_data *data, 5806 struct pt_regs *regs) 5807 { 5808 __perf_event_output(event, data, regs, perf_output_begin_backward); 5809 } 5810 5811 void 5812 perf_event_output(struct perf_event *event, 5813 struct perf_sample_data *data, 5814 struct pt_regs *regs) 5815 { 5816 __perf_event_output(event, data, regs, perf_output_begin); 5817 } 5818 5819 /* 5820 * read event_id 5821 */ 5822 5823 struct perf_read_event { 5824 struct perf_event_header header; 5825 5826 u32 pid; 5827 u32 tid; 5828 }; 5829 5830 static void 5831 perf_event_read_event(struct perf_event *event, 5832 struct task_struct *task) 5833 { 5834 struct perf_output_handle handle; 5835 struct perf_sample_data sample; 5836 struct perf_read_event read_event = { 5837 .header = { 5838 .type = PERF_RECORD_READ, 5839 .misc = 0, 5840 .size = sizeof(read_event) + event->read_size, 5841 }, 5842 .pid = perf_event_pid(event, task), 5843 .tid = perf_event_tid(event, task), 5844 }; 5845 int ret; 5846 5847 perf_event_header__init_id(&read_event.header, &sample, event); 5848 ret = perf_output_begin(&handle, event, read_event.header.size); 5849 if (ret) 5850 return; 5851 5852 perf_output_put(&handle, read_event); 5853 perf_output_read(&handle, event); 5854 perf_event__output_id_sample(event, &handle, &sample); 5855 5856 perf_output_end(&handle); 5857 } 5858 5859 typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data); 5860 5861 static void 5862 perf_event_aux_ctx(struct perf_event_context *ctx, 5863 perf_event_aux_output_cb output, 5864 void *data, bool all) 5865 { 5866 struct perf_event *event; 5867 5868 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 5869 if (!all) { 5870 if (event->state < PERF_EVENT_STATE_INACTIVE) 5871 continue; 5872 if (!event_filter_match(event)) 5873 continue; 5874 } 5875 5876 output(event, data); 5877 } 5878 } 5879 5880 static void 5881 perf_event_aux_task_ctx(perf_event_aux_output_cb output, void *data, 5882 struct perf_event_context *task_ctx) 5883 { 5884 rcu_read_lock(); 5885 preempt_disable(); 5886 perf_event_aux_ctx(task_ctx, output, data, false); 5887 preempt_enable(); 5888 rcu_read_unlock(); 5889 } 5890 5891 static void 5892 perf_event_aux(perf_event_aux_output_cb output, void *data, 5893 struct perf_event_context *task_ctx) 5894 { 5895 struct perf_cpu_context *cpuctx; 5896 struct perf_event_context *ctx; 5897 struct pmu *pmu; 5898 int ctxn; 5899 5900 /* 5901 * If we have task_ctx != NULL we only notify 5902 * the task context itself. The task_ctx is set 5903 * only for EXIT events before releasing task 5904 * context. 5905 */ 5906 if (task_ctx) { 5907 perf_event_aux_task_ctx(output, data, task_ctx); 5908 return; 5909 } 5910 5911 rcu_read_lock(); 5912 list_for_each_entry_rcu(pmu, &pmus, entry) { 5913 cpuctx = get_cpu_ptr(pmu->pmu_cpu_context); 5914 if (cpuctx->unique_pmu != pmu) 5915 goto next; 5916 perf_event_aux_ctx(&cpuctx->ctx, output, data, false); 5917 ctxn = pmu->task_ctx_nr; 5918 if (ctxn < 0) 5919 goto next; 5920 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 5921 if (ctx) 5922 perf_event_aux_ctx(ctx, output, data, false); 5923 next: 5924 put_cpu_ptr(pmu->pmu_cpu_context); 5925 } 5926 rcu_read_unlock(); 5927 } 5928 5929 /* 5930 * Clear all file-based filters at exec, they'll have to be 5931 * re-instated when/if these objects are mmapped again. 5932 */ 5933 static void perf_event_addr_filters_exec(struct perf_event *event, void *data) 5934 { 5935 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 5936 struct perf_addr_filter *filter; 5937 unsigned int restart = 0, count = 0; 5938 unsigned long flags; 5939 5940 if (!has_addr_filter(event)) 5941 return; 5942 5943 raw_spin_lock_irqsave(&ifh->lock, flags); 5944 list_for_each_entry(filter, &ifh->list, entry) { 5945 if (filter->inode) { 5946 event->addr_filters_offs[count] = 0; 5947 restart++; 5948 } 5949 5950 count++; 5951 } 5952 5953 if (restart) 5954 event->addr_filters_gen++; 5955 raw_spin_unlock_irqrestore(&ifh->lock, flags); 5956 5957 if (restart) 5958 perf_event_restart(event); 5959 } 5960 5961 void perf_event_exec(void) 5962 { 5963 struct perf_event_context *ctx; 5964 int ctxn; 5965 5966 rcu_read_lock(); 5967 for_each_task_context_nr(ctxn) { 5968 ctx = current->perf_event_ctxp[ctxn]; 5969 if (!ctx) 5970 continue; 5971 5972 perf_event_enable_on_exec(ctxn); 5973 5974 perf_event_aux_ctx(ctx, perf_event_addr_filters_exec, NULL, 5975 true); 5976 } 5977 rcu_read_unlock(); 5978 } 5979 5980 struct remote_output { 5981 struct ring_buffer *rb; 5982 int err; 5983 }; 5984 5985 static void __perf_event_output_stop(struct perf_event *event, void *data) 5986 { 5987 struct perf_event *parent = event->parent; 5988 struct remote_output *ro = data; 5989 struct ring_buffer *rb = ro->rb; 5990 struct stop_event_data sd = { 5991 .event = event, 5992 }; 5993 5994 if (!has_aux(event)) 5995 return; 5996 5997 if (!parent) 5998 parent = event; 5999 6000 /* 6001 * In case of inheritance, it will be the parent that links to the 6002 * ring-buffer, but it will be the child that's actually using it: 6003 */ 6004 if (rcu_dereference(parent->rb) == rb) 6005 ro->err = __perf_event_stop(&sd); 6006 } 6007 6008 static int __perf_pmu_output_stop(void *info) 6009 { 6010 struct perf_event *event = info; 6011 struct pmu *pmu = event->pmu; 6012 struct perf_cpu_context *cpuctx = get_cpu_ptr(pmu->pmu_cpu_context); 6013 struct remote_output ro = { 6014 .rb = event->rb, 6015 }; 6016 6017 rcu_read_lock(); 6018 perf_event_aux_ctx(&cpuctx->ctx, __perf_event_output_stop, &ro, false); 6019 if (cpuctx->task_ctx) 6020 perf_event_aux_ctx(cpuctx->task_ctx, __perf_event_output_stop, 6021 &ro, false); 6022 rcu_read_unlock(); 6023 6024 return ro.err; 6025 } 6026 6027 static void perf_pmu_output_stop(struct perf_event *event) 6028 { 6029 struct perf_event *iter; 6030 int err, cpu; 6031 6032 restart: 6033 rcu_read_lock(); 6034 list_for_each_entry_rcu(iter, &event->rb->event_list, rb_entry) { 6035 /* 6036 * For per-CPU events, we need to make sure that neither they 6037 * nor their children are running; for cpu==-1 events it's 6038 * sufficient to stop the event itself if it's active, since 6039 * it can't have children. 6040 */ 6041 cpu = iter->cpu; 6042 if (cpu == -1) 6043 cpu = READ_ONCE(iter->oncpu); 6044 6045 if (cpu == -1) 6046 continue; 6047 6048 err = cpu_function_call(cpu, __perf_pmu_output_stop, event); 6049 if (err == -EAGAIN) { 6050 rcu_read_unlock(); 6051 goto restart; 6052 } 6053 } 6054 rcu_read_unlock(); 6055 } 6056 6057 /* 6058 * task tracking -- fork/exit 6059 * 6060 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task 6061 */ 6062 6063 struct perf_task_event { 6064 struct task_struct *task; 6065 struct perf_event_context *task_ctx; 6066 6067 struct { 6068 struct perf_event_header header; 6069 6070 u32 pid; 6071 u32 ppid; 6072 u32 tid; 6073 u32 ptid; 6074 u64 time; 6075 } event_id; 6076 }; 6077 6078 static int perf_event_task_match(struct perf_event *event) 6079 { 6080 return event->attr.comm || event->attr.mmap || 6081 event->attr.mmap2 || event->attr.mmap_data || 6082 event->attr.task; 6083 } 6084 6085 static void perf_event_task_output(struct perf_event *event, 6086 void *data) 6087 { 6088 struct perf_task_event *task_event = data; 6089 struct perf_output_handle handle; 6090 struct perf_sample_data sample; 6091 struct task_struct *task = task_event->task; 6092 int ret, size = task_event->event_id.header.size; 6093 6094 if (!perf_event_task_match(event)) 6095 return; 6096 6097 perf_event_header__init_id(&task_event->event_id.header, &sample, event); 6098 6099 ret = perf_output_begin(&handle, event, 6100 task_event->event_id.header.size); 6101 if (ret) 6102 goto out; 6103 6104 task_event->event_id.pid = perf_event_pid(event, task); 6105 task_event->event_id.ppid = perf_event_pid(event, current); 6106 6107 task_event->event_id.tid = perf_event_tid(event, task); 6108 task_event->event_id.ptid = perf_event_tid(event, current); 6109 6110 task_event->event_id.time = perf_event_clock(event); 6111 6112 perf_output_put(&handle, task_event->event_id); 6113 6114 perf_event__output_id_sample(event, &handle, &sample); 6115 6116 perf_output_end(&handle); 6117 out: 6118 task_event->event_id.header.size = size; 6119 } 6120 6121 static void perf_event_task(struct task_struct *task, 6122 struct perf_event_context *task_ctx, 6123 int new) 6124 { 6125 struct perf_task_event task_event; 6126 6127 if (!atomic_read(&nr_comm_events) && 6128 !atomic_read(&nr_mmap_events) && 6129 !atomic_read(&nr_task_events)) 6130 return; 6131 6132 task_event = (struct perf_task_event){ 6133 .task = task, 6134 .task_ctx = task_ctx, 6135 .event_id = { 6136 .header = { 6137 .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT, 6138 .misc = 0, 6139 .size = sizeof(task_event.event_id), 6140 }, 6141 /* .pid */ 6142 /* .ppid */ 6143 /* .tid */ 6144 /* .ptid */ 6145 /* .time */ 6146 }, 6147 }; 6148 6149 perf_event_aux(perf_event_task_output, 6150 &task_event, 6151 task_ctx); 6152 } 6153 6154 void perf_event_fork(struct task_struct *task) 6155 { 6156 perf_event_task(task, NULL, 1); 6157 } 6158 6159 /* 6160 * comm tracking 6161 */ 6162 6163 struct perf_comm_event { 6164 struct task_struct *task; 6165 char *comm; 6166 int comm_size; 6167 6168 struct { 6169 struct perf_event_header header; 6170 6171 u32 pid; 6172 u32 tid; 6173 } event_id; 6174 }; 6175 6176 static int perf_event_comm_match(struct perf_event *event) 6177 { 6178 return event->attr.comm; 6179 } 6180 6181 static void perf_event_comm_output(struct perf_event *event, 6182 void *data) 6183 { 6184 struct perf_comm_event *comm_event = data; 6185 struct perf_output_handle handle; 6186 struct perf_sample_data sample; 6187 int size = comm_event->event_id.header.size; 6188 int ret; 6189 6190 if (!perf_event_comm_match(event)) 6191 return; 6192 6193 perf_event_header__init_id(&comm_event->event_id.header, &sample, event); 6194 ret = perf_output_begin(&handle, event, 6195 comm_event->event_id.header.size); 6196 6197 if (ret) 6198 goto out; 6199 6200 comm_event->event_id.pid = perf_event_pid(event, comm_event->task); 6201 comm_event->event_id.tid = perf_event_tid(event, comm_event->task); 6202 6203 perf_output_put(&handle, comm_event->event_id); 6204 __output_copy(&handle, comm_event->comm, 6205 comm_event->comm_size); 6206 6207 perf_event__output_id_sample(event, &handle, &sample); 6208 6209 perf_output_end(&handle); 6210 out: 6211 comm_event->event_id.header.size = size; 6212 } 6213 6214 static void perf_event_comm_event(struct perf_comm_event *comm_event) 6215 { 6216 char comm[TASK_COMM_LEN]; 6217 unsigned int size; 6218 6219 memset(comm, 0, sizeof(comm)); 6220 strlcpy(comm, comm_event->task->comm, sizeof(comm)); 6221 size = ALIGN(strlen(comm)+1, sizeof(u64)); 6222 6223 comm_event->comm = comm; 6224 comm_event->comm_size = size; 6225 6226 comm_event->event_id.header.size = sizeof(comm_event->event_id) + size; 6227 6228 perf_event_aux(perf_event_comm_output, 6229 comm_event, 6230 NULL); 6231 } 6232 6233 void perf_event_comm(struct task_struct *task, bool exec) 6234 { 6235 struct perf_comm_event comm_event; 6236 6237 if (!atomic_read(&nr_comm_events)) 6238 return; 6239 6240 comm_event = (struct perf_comm_event){ 6241 .task = task, 6242 /* .comm */ 6243 /* .comm_size */ 6244 .event_id = { 6245 .header = { 6246 .type = PERF_RECORD_COMM, 6247 .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0, 6248 /* .size */ 6249 }, 6250 /* .pid */ 6251 /* .tid */ 6252 }, 6253 }; 6254 6255 perf_event_comm_event(&comm_event); 6256 } 6257 6258 /* 6259 * mmap tracking 6260 */ 6261 6262 struct perf_mmap_event { 6263 struct vm_area_struct *vma; 6264 6265 const char *file_name; 6266 int file_size; 6267 int maj, min; 6268 u64 ino; 6269 u64 ino_generation; 6270 u32 prot, flags; 6271 6272 struct { 6273 struct perf_event_header header; 6274 6275 u32 pid; 6276 u32 tid; 6277 u64 start; 6278 u64 len; 6279 u64 pgoff; 6280 } event_id; 6281 }; 6282 6283 static int perf_event_mmap_match(struct perf_event *event, 6284 void *data) 6285 { 6286 struct perf_mmap_event *mmap_event = data; 6287 struct vm_area_struct *vma = mmap_event->vma; 6288 int executable = vma->vm_flags & VM_EXEC; 6289 6290 return (!executable && event->attr.mmap_data) || 6291 (executable && (event->attr.mmap || event->attr.mmap2)); 6292 } 6293 6294 static void perf_event_mmap_output(struct perf_event *event, 6295 void *data) 6296 { 6297 struct perf_mmap_event *mmap_event = data; 6298 struct perf_output_handle handle; 6299 struct perf_sample_data sample; 6300 int size = mmap_event->event_id.header.size; 6301 int ret; 6302 6303 if (!perf_event_mmap_match(event, data)) 6304 return; 6305 6306 if (event->attr.mmap2) { 6307 mmap_event->event_id.header.type = PERF_RECORD_MMAP2; 6308 mmap_event->event_id.header.size += sizeof(mmap_event->maj); 6309 mmap_event->event_id.header.size += sizeof(mmap_event->min); 6310 mmap_event->event_id.header.size += sizeof(mmap_event->ino); 6311 mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation); 6312 mmap_event->event_id.header.size += sizeof(mmap_event->prot); 6313 mmap_event->event_id.header.size += sizeof(mmap_event->flags); 6314 } 6315 6316 perf_event_header__init_id(&mmap_event->event_id.header, &sample, event); 6317 ret = perf_output_begin(&handle, event, 6318 mmap_event->event_id.header.size); 6319 if (ret) 6320 goto out; 6321 6322 mmap_event->event_id.pid = perf_event_pid(event, current); 6323 mmap_event->event_id.tid = perf_event_tid(event, current); 6324 6325 perf_output_put(&handle, mmap_event->event_id); 6326 6327 if (event->attr.mmap2) { 6328 perf_output_put(&handle, mmap_event->maj); 6329 perf_output_put(&handle, mmap_event->min); 6330 perf_output_put(&handle, mmap_event->ino); 6331 perf_output_put(&handle, mmap_event->ino_generation); 6332 perf_output_put(&handle, mmap_event->prot); 6333 perf_output_put(&handle, mmap_event->flags); 6334 } 6335 6336 __output_copy(&handle, mmap_event->file_name, 6337 mmap_event->file_size); 6338 6339 perf_event__output_id_sample(event, &handle, &sample); 6340 6341 perf_output_end(&handle); 6342 out: 6343 mmap_event->event_id.header.size = size; 6344 } 6345 6346 static void perf_event_mmap_event(struct perf_mmap_event *mmap_event) 6347 { 6348 struct vm_area_struct *vma = mmap_event->vma; 6349 struct file *file = vma->vm_file; 6350 int maj = 0, min = 0; 6351 u64 ino = 0, gen = 0; 6352 u32 prot = 0, flags = 0; 6353 unsigned int size; 6354 char tmp[16]; 6355 char *buf = NULL; 6356 char *name; 6357 6358 if (file) { 6359 struct inode *inode; 6360 dev_t dev; 6361 6362 buf = kmalloc(PATH_MAX, GFP_KERNEL); 6363 if (!buf) { 6364 name = "//enomem"; 6365 goto cpy_name; 6366 } 6367 /* 6368 * d_path() works from the end of the rb backwards, so we 6369 * need to add enough zero bytes after the string to handle 6370 * the 64bit alignment we do later. 6371 */ 6372 name = file_path(file, buf, PATH_MAX - sizeof(u64)); 6373 if (IS_ERR(name)) { 6374 name = "//toolong"; 6375 goto cpy_name; 6376 } 6377 inode = file_inode(vma->vm_file); 6378 dev = inode->i_sb->s_dev; 6379 ino = inode->i_ino; 6380 gen = inode->i_generation; 6381 maj = MAJOR(dev); 6382 min = MINOR(dev); 6383 6384 if (vma->vm_flags & VM_READ) 6385 prot |= PROT_READ; 6386 if (vma->vm_flags & VM_WRITE) 6387 prot |= PROT_WRITE; 6388 if (vma->vm_flags & VM_EXEC) 6389 prot |= PROT_EXEC; 6390 6391 if (vma->vm_flags & VM_MAYSHARE) 6392 flags = MAP_SHARED; 6393 else 6394 flags = MAP_PRIVATE; 6395 6396 if (vma->vm_flags & VM_DENYWRITE) 6397 flags |= MAP_DENYWRITE; 6398 if (vma->vm_flags & VM_MAYEXEC) 6399 flags |= MAP_EXECUTABLE; 6400 if (vma->vm_flags & VM_LOCKED) 6401 flags |= MAP_LOCKED; 6402 if (vma->vm_flags & VM_HUGETLB) 6403 flags |= MAP_HUGETLB; 6404 6405 goto got_name; 6406 } else { 6407 if (vma->vm_ops && vma->vm_ops->name) { 6408 name = (char *) vma->vm_ops->name(vma); 6409 if (name) 6410 goto cpy_name; 6411 } 6412 6413 name = (char *)arch_vma_name(vma); 6414 if (name) 6415 goto cpy_name; 6416 6417 if (vma->vm_start <= vma->vm_mm->start_brk && 6418 vma->vm_end >= vma->vm_mm->brk) { 6419 name = "[heap]"; 6420 goto cpy_name; 6421 } 6422 if (vma->vm_start <= vma->vm_mm->start_stack && 6423 vma->vm_end >= vma->vm_mm->start_stack) { 6424 name = "[stack]"; 6425 goto cpy_name; 6426 } 6427 6428 name = "//anon"; 6429 goto cpy_name; 6430 } 6431 6432 cpy_name: 6433 strlcpy(tmp, name, sizeof(tmp)); 6434 name = tmp; 6435 got_name: 6436 /* 6437 * Since our buffer works in 8 byte units we need to align our string 6438 * size to a multiple of 8. However, we must guarantee the tail end is 6439 * zero'd out to avoid leaking random bits to userspace. 6440 */ 6441 size = strlen(name)+1; 6442 while (!IS_ALIGNED(size, sizeof(u64))) 6443 name[size++] = '\0'; 6444 6445 mmap_event->file_name = name; 6446 mmap_event->file_size = size; 6447 mmap_event->maj = maj; 6448 mmap_event->min = min; 6449 mmap_event->ino = ino; 6450 mmap_event->ino_generation = gen; 6451 mmap_event->prot = prot; 6452 mmap_event->flags = flags; 6453 6454 if (!(vma->vm_flags & VM_EXEC)) 6455 mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA; 6456 6457 mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size; 6458 6459 perf_event_aux(perf_event_mmap_output, 6460 mmap_event, 6461 NULL); 6462 6463 kfree(buf); 6464 } 6465 6466 /* 6467 * Whether this @filter depends on a dynamic object which is not loaded 6468 * yet or its load addresses are not known. 6469 */ 6470 static bool perf_addr_filter_needs_mmap(struct perf_addr_filter *filter) 6471 { 6472 return filter->filter && filter->inode; 6473 } 6474 6475 /* 6476 * Check whether inode and address range match filter criteria. 6477 */ 6478 static bool perf_addr_filter_match(struct perf_addr_filter *filter, 6479 struct file *file, unsigned long offset, 6480 unsigned long size) 6481 { 6482 if (filter->inode != file->f_inode) 6483 return false; 6484 6485 if (filter->offset > offset + size) 6486 return false; 6487 6488 if (filter->offset + filter->size < offset) 6489 return false; 6490 6491 return true; 6492 } 6493 6494 static void __perf_addr_filters_adjust(struct perf_event *event, void *data) 6495 { 6496 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 6497 struct vm_area_struct *vma = data; 6498 unsigned long off = vma->vm_pgoff << PAGE_SHIFT, flags; 6499 struct file *file = vma->vm_file; 6500 struct perf_addr_filter *filter; 6501 unsigned int restart = 0, count = 0; 6502 6503 if (!has_addr_filter(event)) 6504 return; 6505 6506 if (!file) 6507 return; 6508 6509 raw_spin_lock_irqsave(&ifh->lock, flags); 6510 list_for_each_entry(filter, &ifh->list, entry) { 6511 if (perf_addr_filter_match(filter, file, off, 6512 vma->vm_end - vma->vm_start)) { 6513 event->addr_filters_offs[count] = vma->vm_start; 6514 restart++; 6515 } 6516 6517 count++; 6518 } 6519 6520 if (restart) 6521 event->addr_filters_gen++; 6522 raw_spin_unlock_irqrestore(&ifh->lock, flags); 6523 6524 if (restart) 6525 perf_event_restart(event); 6526 } 6527 6528 /* 6529 * Adjust all task's events' filters to the new vma 6530 */ 6531 static void perf_addr_filters_adjust(struct vm_area_struct *vma) 6532 { 6533 struct perf_event_context *ctx; 6534 int ctxn; 6535 6536 rcu_read_lock(); 6537 for_each_task_context_nr(ctxn) { 6538 ctx = rcu_dereference(current->perf_event_ctxp[ctxn]); 6539 if (!ctx) 6540 continue; 6541 6542 perf_event_aux_ctx(ctx, __perf_addr_filters_adjust, vma, true); 6543 } 6544 rcu_read_unlock(); 6545 } 6546 6547 void perf_event_mmap(struct vm_area_struct *vma) 6548 { 6549 struct perf_mmap_event mmap_event; 6550 6551 if (!atomic_read(&nr_mmap_events)) 6552 return; 6553 6554 mmap_event = (struct perf_mmap_event){ 6555 .vma = vma, 6556 /* .file_name */ 6557 /* .file_size */ 6558 .event_id = { 6559 .header = { 6560 .type = PERF_RECORD_MMAP, 6561 .misc = PERF_RECORD_MISC_USER, 6562 /* .size */ 6563 }, 6564 /* .pid */ 6565 /* .tid */ 6566 .start = vma->vm_start, 6567 .len = vma->vm_end - vma->vm_start, 6568 .pgoff = (u64)vma->vm_pgoff << PAGE_SHIFT, 6569 }, 6570 /* .maj (attr_mmap2 only) */ 6571 /* .min (attr_mmap2 only) */ 6572 /* .ino (attr_mmap2 only) */ 6573 /* .ino_generation (attr_mmap2 only) */ 6574 /* .prot (attr_mmap2 only) */ 6575 /* .flags (attr_mmap2 only) */ 6576 }; 6577 6578 perf_addr_filters_adjust(vma); 6579 perf_event_mmap_event(&mmap_event); 6580 } 6581 6582 void perf_event_aux_event(struct perf_event *event, unsigned long head, 6583 unsigned long size, u64 flags) 6584 { 6585 struct perf_output_handle handle; 6586 struct perf_sample_data sample; 6587 struct perf_aux_event { 6588 struct perf_event_header header; 6589 u64 offset; 6590 u64 size; 6591 u64 flags; 6592 } rec = { 6593 .header = { 6594 .type = PERF_RECORD_AUX, 6595 .misc = 0, 6596 .size = sizeof(rec), 6597 }, 6598 .offset = head, 6599 .size = size, 6600 .flags = flags, 6601 }; 6602 int ret; 6603 6604 perf_event_header__init_id(&rec.header, &sample, event); 6605 ret = perf_output_begin(&handle, event, rec.header.size); 6606 6607 if (ret) 6608 return; 6609 6610 perf_output_put(&handle, rec); 6611 perf_event__output_id_sample(event, &handle, &sample); 6612 6613 perf_output_end(&handle); 6614 } 6615 6616 /* 6617 * Lost/dropped samples logging 6618 */ 6619 void perf_log_lost_samples(struct perf_event *event, u64 lost) 6620 { 6621 struct perf_output_handle handle; 6622 struct perf_sample_data sample; 6623 int ret; 6624 6625 struct { 6626 struct perf_event_header header; 6627 u64 lost; 6628 } lost_samples_event = { 6629 .header = { 6630 .type = PERF_RECORD_LOST_SAMPLES, 6631 .misc = 0, 6632 .size = sizeof(lost_samples_event), 6633 }, 6634 .lost = lost, 6635 }; 6636 6637 perf_event_header__init_id(&lost_samples_event.header, &sample, event); 6638 6639 ret = perf_output_begin(&handle, event, 6640 lost_samples_event.header.size); 6641 if (ret) 6642 return; 6643 6644 perf_output_put(&handle, lost_samples_event); 6645 perf_event__output_id_sample(event, &handle, &sample); 6646 perf_output_end(&handle); 6647 } 6648 6649 /* 6650 * context_switch tracking 6651 */ 6652 6653 struct perf_switch_event { 6654 struct task_struct *task; 6655 struct task_struct *next_prev; 6656 6657 struct { 6658 struct perf_event_header header; 6659 u32 next_prev_pid; 6660 u32 next_prev_tid; 6661 } event_id; 6662 }; 6663 6664 static int perf_event_switch_match(struct perf_event *event) 6665 { 6666 return event->attr.context_switch; 6667 } 6668 6669 static void perf_event_switch_output(struct perf_event *event, void *data) 6670 { 6671 struct perf_switch_event *se = data; 6672 struct perf_output_handle handle; 6673 struct perf_sample_data sample; 6674 int ret; 6675 6676 if (!perf_event_switch_match(event)) 6677 return; 6678 6679 /* Only CPU-wide events are allowed to see next/prev pid/tid */ 6680 if (event->ctx->task) { 6681 se->event_id.header.type = PERF_RECORD_SWITCH; 6682 se->event_id.header.size = sizeof(se->event_id.header); 6683 } else { 6684 se->event_id.header.type = PERF_RECORD_SWITCH_CPU_WIDE; 6685 se->event_id.header.size = sizeof(se->event_id); 6686 se->event_id.next_prev_pid = 6687 perf_event_pid(event, se->next_prev); 6688 se->event_id.next_prev_tid = 6689 perf_event_tid(event, se->next_prev); 6690 } 6691 6692 perf_event_header__init_id(&se->event_id.header, &sample, event); 6693 6694 ret = perf_output_begin(&handle, event, se->event_id.header.size); 6695 if (ret) 6696 return; 6697 6698 if (event->ctx->task) 6699 perf_output_put(&handle, se->event_id.header); 6700 else 6701 perf_output_put(&handle, se->event_id); 6702 6703 perf_event__output_id_sample(event, &handle, &sample); 6704 6705 perf_output_end(&handle); 6706 } 6707 6708 static void perf_event_switch(struct task_struct *task, 6709 struct task_struct *next_prev, bool sched_in) 6710 { 6711 struct perf_switch_event switch_event; 6712 6713 /* N.B. caller checks nr_switch_events != 0 */ 6714 6715 switch_event = (struct perf_switch_event){ 6716 .task = task, 6717 .next_prev = next_prev, 6718 .event_id = { 6719 .header = { 6720 /* .type */ 6721 .misc = sched_in ? 0 : PERF_RECORD_MISC_SWITCH_OUT, 6722 /* .size */ 6723 }, 6724 /* .next_prev_pid */ 6725 /* .next_prev_tid */ 6726 }, 6727 }; 6728 6729 perf_event_aux(perf_event_switch_output, 6730 &switch_event, 6731 NULL); 6732 } 6733 6734 /* 6735 * IRQ throttle logging 6736 */ 6737 6738 static void perf_log_throttle(struct perf_event *event, int enable) 6739 { 6740 struct perf_output_handle handle; 6741 struct perf_sample_data sample; 6742 int ret; 6743 6744 struct { 6745 struct perf_event_header header; 6746 u64 time; 6747 u64 id; 6748 u64 stream_id; 6749 } throttle_event = { 6750 .header = { 6751 .type = PERF_RECORD_THROTTLE, 6752 .misc = 0, 6753 .size = sizeof(throttle_event), 6754 }, 6755 .time = perf_event_clock(event), 6756 .id = primary_event_id(event), 6757 .stream_id = event->id, 6758 }; 6759 6760 if (enable) 6761 throttle_event.header.type = PERF_RECORD_UNTHROTTLE; 6762 6763 perf_event_header__init_id(&throttle_event.header, &sample, event); 6764 6765 ret = perf_output_begin(&handle, event, 6766 throttle_event.header.size); 6767 if (ret) 6768 return; 6769 6770 perf_output_put(&handle, throttle_event); 6771 perf_event__output_id_sample(event, &handle, &sample); 6772 perf_output_end(&handle); 6773 } 6774 6775 static void perf_log_itrace_start(struct perf_event *event) 6776 { 6777 struct perf_output_handle handle; 6778 struct perf_sample_data sample; 6779 struct perf_aux_event { 6780 struct perf_event_header header; 6781 u32 pid; 6782 u32 tid; 6783 } rec; 6784 int ret; 6785 6786 if (event->parent) 6787 event = event->parent; 6788 6789 if (!(event->pmu->capabilities & PERF_PMU_CAP_ITRACE) || 6790 event->hw.itrace_started) 6791 return; 6792 6793 rec.header.type = PERF_RECORD_ITRACE_START; 6794 rec.header.misc = 0; 6795 rec.header.size = sizeof(rec); 6796 rec.pid = perf_event_pid(event, current); 6797 rec.tid = perf_event_tid(event, current); 6798 6799 perf_event_header__init_id(&rec.header, &sample, event); 6800 ret = perf_output_begin(&handle, event, rec.header.size); 6801 6802 if (ret) 6803 return; 6804 6805 perf_output_put(&handle, rec); 6806 perf_event__output_id_sample(event, &handle, &sample); 6807 6808 perf_output_end(&handle); 6809 } 6810 6811 /* 6812 * Generic event overflow handling, sampling. 6813 */ 6814 6815 static int __perf_event_overflow(struct perf_event *event, 6816 int throttle, struct perf_sample_data *data, 6817 struct pt_regs *regs) 6818 { 6819 int events = atomic_read(&event->event_limit); 6820 struct hw_perf_event *hwc = &event->hw; 6821 u64 seq; 6822 int ret = 0; 6823 6824 /* 6825 * Non-sampling counters might still use the PMI to fold short 6826 * hardware counters, ignore those. 6827 */ 6828 if (unlikely(!is_sampling_event(event))) 6829 return 0; 6830 6831 seq = __this_cpu_read(perf_throttled_seq); 6832 if (seq != hwc->interrupts_seq) { 6833 hwc->interrupts_seq = seq; 6834 hwc->interrupts = 1; 6835 } else { 6836 hwc->interrupts++; 6837 if (unlikely(throttle 6838 && hwc->interrupts >= max_samples_per_tick)) { 6839 __this_cpu_inc(perf_throttled_count); 6840 tick_dep_set_cpu(smp_processor_id(), TICK_DEP_BIT_PERF_EVENTS); 6841 hwc->interrupts = MAX_INTERRUPTS; 6842 perf_log_throttle(event, 0); 6843 ret = 1; 6844 } 6845 } 6846 6847 if (event->attr.freq) { 6848 u64 now = perf_clock(); 6849 s64 delta = now - hwc->freq_time_stamp; 6850 6851 hwc->freq_time_stamp = now; 6852 6853 if (delta > 0 && delta < 2*TICK_NSEC) 6854 perf_adjust_period(event, delta, hwc->last_period, true); 6855 } 6856 6857 /* 6858 * XXX event_limit might not quite work as expected on inherited 6859 * events 6860 */ 6861 6862 event->pending_kill = POLL_IN; 6863 if (events && atomic_dec_and_test(&event->event_limit)) { 6864 ret = 1; 6865 event->pending_kill = POLL_HUP; 6866 event->pending_disable = 1; 6867 irq_work_queue(&event->pending); 6868 } 6869 6870 event->overflow_handler(event, data, regs); 6871 6872 if (*perf_event_fasync(event) && event->pending_kill) { 6873 event->pending_wakeup = 1; 6874 irq_work_queue(&event->pending); 6875 } 6876 6877 return ret; 6878 } 6879 6880 int perf_event_overflow(struct perf_event *event, 6881 struct perf_sample_data *data, 6882 struct pt_regs *regs) 6883 { 6884 return __perf_event_overflow(event, 1, data, regs); 6885 } 6886 6887 /* 6888 * Generic software event infrastructure 6889 */ 6890 6891 struct swevent_htable { 6892 struct swevent_hlist *swevent_hlist; 6893 struct mutex hlist_mutex; 6894 int hlist_refcount; 6895 6896 /* Recursion avoidance in each contexts */ 6897 int recursion[PERF_NR_CONTEXTS]; 6898 }; 6899 6900 static DEFINE_PER_CPU(struct swevent_htable, swevent_htable); 6901 6902 /* 6903 * We directly increment event->count and keep a second value in 6904 * event->hw.period_left to count intervals. This period event 6905 * is kept in the range [-sample_period, 0] so that we can use the 6906 * sign as trigger. 6907 */ 6908 6909 u64 perf_swevent_set_period(struct perf_event *event) 6910 { 6911 struct hw_perf_event *hwc = &event->hw; 6912 u64 period = hwc->last_period; 6913 u64 nr, offset; 6914 s64 old, val; 6915 6916 hwc->last_period = hwc->sample_period; 6917 6918 again: 6919 old = val = local64_read(&hwc->period_left); 6920 if (val < 0) 6921 return 0; 6922 6923 nr = div64_u64(period + val, period); 6924 offset = nr * period; 6925 val -= offset; 6926 if (local64_cmpxchg(&hwc->period_left, old, val) != old) 6927 goto again; 6928 6929 return nr; 6930 } 6931 6932 static void perf_swevent_overflow(struct perf_event *event, u64 overflow, 6933 struct perf_sample_data *data, 6934 struct pt_regs *regs) 6935 { 6936 struct hw_perf_event *hwc = &event->hw; 6937 int throttle = 0; 6938 6939 if (!overflow) 6940 overflow = perf_swevent_set_period(event); 6941 6942 if (hwc->interrupts == MAX_INTERRUPTS) 6943 return; 6944 6945 for (; overflow; overflow--) { 6946 if (__perf_event_overflow(event, throttle, 6947 data, regs)) { 6948 /* 6949 * We inhibit the overflow from happening when 6950 * hwc->interrupts == MAX_INTERRUPTS. 6951 */ 6952 break; 6953 } 6954 throttle = 1; 6955 } 6956 } 6957 6958 static void perf_swevent_event(struct perf_event *event, u64 nr, 6959 struct perf_sample_data *data, 6960 struct pt_regs *regs) 6961 { 6962 struct hw_perf_event *hwc = &event->hw; 6963 6964 local64_add(nr, &event->count); 6965 6966 if (!regs) 6967 return; 6968 6969 if (!is_sampling_event(event)) 6970 return; 6971 6972 if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) { 6973 data->period = nr; 6974 return perf_swevent_overflow(event, 1, data, regs); 6975 } else 6976 data->period = event->hw.last_period; 6977 6978 if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq) 6979 return perf_swevent_overflow(event, 1, data, regs); 6980 6981 if (local64_add_negative(nr, &hwc->period_left)) 6982 return; 6983 6984 perf_swevent_overflow(event, 0, data, regs); 6985 } 6986 6987 static int perf_exclude_event(struct perf_event *event, 6988 struct pt_regs *regs) 6989 { 6990 if (event->hw.state & PERF_HES_STOPPED) 6991 return 1; 6992 6993 if (regs) { 6994 if (event->attr.exclude_user && user_mode(regs)) 6995 return 1; 6996 6997 if (event->attr.exclude_kernel && !user_mode(regs)) 6998 return 1; 6999 } 7000 7001 return 0; 7002 } 7003 7004 static int perf_swevent_match(struct perf_event *event, 7005 enum perf_type_id type, 7006 u32 event_id, 7007 struct perf_sample_data *data, 7008 struct pt_regs *regs) 7009 { 7010 if (event->attr.type != type) 7011 return 0; 7012 7013 if (event->attr.config != event_id) 7014 return 0; 7015 7016 if (perf_exclude_event(event, regs)) 7017 return 0; 7018 7019 return 1; 7020 } 7021 7022 static inline u64 swevent_hash(u64 type, u32 event_id) 7023 { 7024 u64 val = event_id | (type << 32); 7025 7026 return hash_64(val, SWEVENT_HLIST_BITS); 7027 } 7028 7029 static inline struct hlist_head * 7030 __find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id) 7031 { 7032 u64 hash = swevent_hash(type, event_id); 7033 7034 return &hlist->heads[hash]; 7035 } 7036 7037 /* For the read side: events when they trigger */ 7038 static inline struct hlist_head * 7039 find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id) 7040 { 7041 struct swevent_hlist *hlist; 7042 7043 hlist = rcu_dereference(swhash->swevent_hlist); 7044 if (!hlist) 7045 return NULL; 7046 7047 return __find_swevent_head(hlist, type, event_id); 7048 } 7049 7050 /* For the event head insertion and removal in the hlist */ 7051 static inline struct hlist_head * 7052 find_swevent_head(struct swevent_htable *swhash, struct perf_event *event) 7053 { 7054 struct swevent_hlist *hlist; 7055 u32 event_id = event->attr.config; 7056 u64 type = event->attr.type; 7057 7058 /* 7059 * Event scheduling is always serialized against hlist allocation 7060 * and release. Which makes the protected version suitable here. 7061 * The context lock guarantees that. 7062 */ 7063 hlist = rcu_dereference_protected(swhash->swevent_hlist, 7064 lockdep_is_held(&event->ctx->lock)); 7065 if (!hlist) 7066 return NULL; 7067 7068 return __find_swevent_head(hlist, type, event_id); 7069 } 7070 7071 static void do_perf_sw_event(enum perf_type_id type, u32 event_id, 7072 u64 nr, 7073 struct perf_sample_data *data, 7074 struct pt_regs *regs) 7075 { 7076 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 7077 struct perf_event *event; 7078 struct hlist_head *head; 7079 7080 rcu_read_lock(); 7081 head = find_swevent_head_rcu(swhash, type, event_id); 7082 if (!head) 7083 goto end; 7084 7085 hlist_for_each_entry_rcu(event, head, hlist_entry) { 7086 if (perf_swevent_match(event, type, event_id, data, regs)) 7087 perf_swevent_event(event, nr, data, regs); 7088 } 7089 end: 7090 rcu_read_unlock(); 7091 } 7092 7093 DEFINE_PER_CPU(struct pt_regs, __perf_regs[4]); 7094 7095 int perf_swevent_get_recursion_context(void) 7096 { 7097 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 7098 7099 return get_recursion_context(swhash->recursion); 7100 } 7101 EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context); 7102 7103 void perf_swevent_put_recursion_context(int rctx) 7104 { 7105 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 7106 7107 put_recursion_context(swhash->recursion, rctx); 7108 } 7109 7110 void ___perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 7111 { 7112 struct perf_sample_data data; 7113 7114 if (WARN_ON_ONCE(!regs)) 7115 return; 7116 7117 perf_sample_data_init(&data, addr, 0); 7118 do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs); 7119 } 7120 7121 void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr) 7122 { 7123 int rctx; 7124 7125 preempt_disable_notrace(); 7126 rctx = perf_swevent_get_recursion_context(); 7127 if (unlikely(rctx < 0)) 7128 goto fail; 7129 7130 ___perf_sw_event(event_id, nr, regs, addr); 7131 7132 perf_swevent_put_recursion_context(rctx); 7133 fail: 7134 preempt_enable_notrace(); 7135 } 7136 7137 static void perf_swevent_read(struct perf_event *event) 7138 { 7139 } 7140 7141 static int perf_swevent_add(struct perf_event *event, int flags) 7142 { 7143 struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable); 7144 struct hw_perf_event *hwc = &event->hw; 7145 struct hlist_head *head; 7146 7147 if (is_sampling_event(event)) { 7148 hwc->last_period = hwc->sample_period; 7149 perf_swevent_set_period(event); 7150 } 7151 7152 hwc->state = !(flags & PERF_EF_START); 7153 7154 head = find_swevent_head(swhash, event); 7155 if (WARN_ON_ONCE(!head)) 7156 return -EINVAL; 7157 7158 hlist_add_head_rcu(&event->hlist_entry, head); 7159 perf_event_update_userpage(event); 7160 7161 return 0; 7162 } 7163 7164 static void perf_swevent_del(struct perf_event *event, int flags) 7165 { 7166 hlist_del_rcu(&event->hlist_entry); 7167 } 7168 7169 static void perf_swevent_start(struct perf_event *event, int flags) 7170 { 7171 event->hw.state = 0; 7172 } 7173 7174 static void perf_swevent_stop(struct perf_event *event, int flags) 7175 { 7176 event->hw.state = PERF_HES_STOPPED; 7177 } 7178 7179 /* Deref the hlist from the update side */ 7180 static inline struct swevent_hlist * 7181 swevent_hlist_deref(struct swevent_htable *swhash) 7182 { 7183 return rcu_dereference_protected(swhash->swevent_hlist, 7184 lockdep_is_held(&swhash->hlist_mutex)); 7185 } 7186 7187 static void swevent_hlist_release(struct swevent_htable *swhash) 7188 { 7189 struct swevent_hlist *hlist = swevent_hlist_deref(swhash); 7190 7191 if (!hlist) 7192 return; 7193 7194 RCU_INIT_POINTER(swhash->swevent_hlist, NULL); 7195 kfree_rcu(hlist, rcu_head); 7196 } 7197 7198 static void swevent_hlist_put_cpu(int cpu) 7199 { 7200 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 7201 7202 mutex_lock(&swhash->hlist_mutex); 7203 7204 if (!--swhash->hlist_refcount) 7205 swevent_hlist_release(swhash); 7206 7207 mutex_unlock(&swhash->hlist_mutex); 7208 } 7209 7210 static void swevent_hlist_put(void) 7211 { 7212 int cpu; 7213 7214 for_each_possible_cpu(cpu) 7215 swevent_hlist_put_cpu(cpu); 7216 } 7217 7218 static int swevent_hlist_get_cpu(int cpu) 7219 { 7220 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 7221 int err = 0; 7222 7223 mutex_lock(&swhash->hlist_mutex); 7224 if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) { 7225 struct swevent_hlist *hlist; 7226 7227 hlist = kzalloc(sizeof(*hlist), GFP_KERNEL); 7228 if (!hlist) { 7229 err = -ENOMEM; 7230 goto exit; 7231 } 7232 rcu_assign_pointer(swhash->swevent_hlist, hlist); 7233 } 7234 swhash->hlist_refcount++; 7235 exit: 7236 mutex_unlock(&swhash->hlist_mutex); 7237 7238 return err; 7239 } 7240 7241 static int swevent_hlist_get(void) 7242 { 7243 int err, cpu, failed_cpu; 7244 7245 get_online_cpus(); 7246 for_each_possible_cpu(cpu) { 7247 err = swevent_hlist_get_cpu(cpu); 7248 if (err) { 7249 failed_cpu = cpu; 7250 goto fail; 7251 } 7252 } 7253 put_online_cpus(); 7254 7255 return 0; 7256 fail: 7257 for_each_possible_cpu(cpu) { 7258 if (cpu == failed_cpu) 7259 break; 7260 swevent_hlist_put_cpu(cpu); 7261 } 7262 7263 put_online_cpus(); 7264 return err; 7265 } 7266 7267 struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX]; 7268 7269 static void sw_perf_event_destroy(struct perf_event *event) 7270 { 7271 u64 event_id = event->attr.config; 7272 7273 WARN_ON(event->parent); 7274 7275 static_key_slow_dec(&perf_swevent_enabled[event_id]); 7276 swevent_hlist_put(); 7277 } 7278 7279 static int perf_swevent_init(struct perf_event *event) 7280 { 7281 u64 event_id = event->attr.config; 7282 7283 if (event->attr.type != PERF_TYPE_SOFTWARE) 7284 return -ENOENT; 7285 7286 /* 7287 * no branch sampling for software events 7288 */ 7289 if (has_branch_stack(event)) 7290 return -EOPNOTSUPP; 7291 7292 switch (event_id) { 7293 case PERF_COUNT_SW_CPU_CLOCK: 7294 case PERF_COUNT_SW_TASK_CLOCK: 7295 return -ENOENT; 7296 7297 default: 7298 break; 7299 } 7300 7301 if (event_id >= PERF_COUNT_SW_MAX) 7302 return -ENOENT; 7303 7304 if (!event->parent) { 7305 int err; 7306 7307 err = swevent_hlist_get(); 7308 if (err) 7309 return err; 7310 7311 static_key_slow_inc(&perf_swevent_enabled[event_id]); 7312 event->destroy = sw_perf_event_destroy; 7313 } 7314 7315 return 0; 7316 } 7317 7318 static struct pmu perf_swevent = { 7319 .task_ctx_nr = perf_sw_context, 7320 7321 .capabilities = PERF_PMU_CAP_NO_NMI, 7322 7323 .event_init = perf_swevent_init, 7324 .add = perf_swevent_add, 7325 .del = perf_swevent_del, 7326 .start = perf_swevent_start, 7327 .stop = perf_swevent_stop, 7328 .read = perf_swevent_read, 7329 }; 7330 7331 #ifdef CONFIG_EVENT_TRACING 7332 7333 static int perf_tp_filter_match(struct perf_event *event, 7334 struct perf_sample_data *data) 7335 { 7336 void *record = data->raw->data; 7337 7338 /* only top level events have filters set */ 7339 if (event->parent) 7340 event = event->parent; 7341 7342 if (likely(!event->filter) || filter_match_preds(event->filter, record)) 7343 return 1; 7344 return 0; 7345 } 7346 7347 static int perf_tp_event_match(struct perf_event *event, 7348 struct perf_sample_data *data, 7349 struct pt_regs *regs) 7350 { 7351 if (event->hw.state & PERF_HES_STOPPED) 7352 return 0; 7353 /* 7354 * All tracepoints are from kernel-space. 7355 */ 7356 if (event->attr.exclude_kernel) 7357 return 0; 7358 7359 if (!perf_tp_filter_match(event, data)) 7360 return 0; 7361 7362 return 1; 7363 } 7364 7365 void perf_trace_run_bpf_submit(void *raw_data, int size, int rctx, 7366 struct trace_event_call *call, u64 count, 7367 struct pt_regs *regs, struct hlist_head *head, 7368 struct task_struct *task) 7369 { 7370 struct bpf_prog *prog = call->prog; 7371 7372 if (prog) { 7373 *(struct pt_regs **)raw_data = regs; 7374 if (!trace_call_bpf(prog, raw_data) || hlist_empty(head)) { 7375 perf_swevent_put_recursion_context(rctx); 7376 return; 7377 } 7378 } 7379 perf_tp_event(call->event.type, count, raw_data, size, regs, head, 7380 rctx, task); 7381 } 7382 EXPORT_SYMBOL_GPL(perf_trace_run_bpf_submit); 7383 7384 void perf_tp_event(u16 event_type, u64 count, void *record, int entry_size, 7385 struct pt_regs *regs, struct hlist_head *head, int rctx, 7386 struct task_struct *task) 7387 { 7388 struct perf_sample_data data; 7389 struct perf_event *event; 7390 7391 struct perf_raw_record raw = { 7392 .size = entry_size, 7393 .data = record, 7394 }; 7395 7396 perf_sample_data_init(&data, 0, 0); 7397 data.raw = &raw; 7398 7399 perf_trace_buf_update(record, event_type); 7400 7401 hlist_for_each_entry_rcu(event, head, hlist_entry) { 7402 if (perf_tp_event_match(event, &data, regs)) 7403 perf_swevent_event(event, count, &data, regs); 7404 } 7405 7406 /* 7407 * If we got specified a target task, also iterate its context and 7408 * deliver this event there too. 7409 */ 7410 if (task && task != current) { 7411 struct perf_event_context *ctx; 7412 struct trace_entry *entry = record; 7413 7414 rcu_read_lock(); 7415 ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]); 7416 if (!ctx) 7417 goto unlock; 7418 7419 list_for_each_entry_rcu(event, &ctx->event_list, event_entry) { 7420 if (event->attr.type != PERF_TYPE_TRACEPOINT) 7421 continue; 7422 if (event->attr.config != entry->type) 7423 continue; 7424 if (perf_tp_event_match(event, &data, regs)) 7425 perf_swevent_event(event, count, &data, regs); 7426 } 7427 unlock: 7428 rcu_read_unlock(); 7429 } 7430 7431 perf_swevent_put_recursion_context(rctx); 7432 } 7433 EXPORT_SYMBOL_GPL(perf_tp_event); 7434 7435 static void tp_perf_event_destroy(struct perf_event *event) 7436 { 7437 perf_trace_destroy(event); 7438 } 7439 7440 static int perf_tp_event_init(struct perf_event *event) 7441 { 7442 int err; 7443 7444 if (event->attr.type != PERF_TYPE_TRACEPOINT) 7445 return -ENOENT; 7446 7447 /* 7448 * no branch sampling for tracepoint events 7449 */ 7450 if (has_branch_stack(event)) 7451 return -EOPNOTSUPP; 7452 7453 err = perf_trace_init(event); 7454 if (err) 7455 return err; 7456 7457 event->destroy = tp_perf_event_destroy; 7458 7459 return 0; 7460 } 7461 7462 static struct pmu perf_tracepoint = { 7463 .task_ctx_nr = perf_sw_context, 7464 7465 .event_init = perf_tp_event_init, 7466 .add = perf_trace_add, 7467 .del = perf_trace_del, 7468 .start = perf_swevent_start, 7469 .stop = perf_swevent_stop, 7470 .read = perf_swevent_read, 7471 }; 7472 7473 static inline void perf_tp_register(void) 7474 { 7475 perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT); 7476 } 7477 7478 static void perf_event_free_filter(struct perf_event *event) 7479 { 7480 ftrace_profile_free_filter(event); 7481 } 7482 7483 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd) 7484 { 7485 bool is_kprobe, is_tracepoint; 7486 struct bpf_prog *prog; 7487 7488 if (event->attr.type != PERF_TYPE_TRACEPOINT) 7489 return -EINVAL; 7490 7491 if (event->tp_event->prog) 7492 return -EEXIST; 7493 7494 is_kprobe = event->tp_event->flags & TRACE_EVENT_FL_UKPROBE; 7495 is_tracepoint = event->tp_event->flags & TRACE_EVENT_FL_TRACEPOINT; 7496 if (!is_kprobe && !is_tracepoint) 7497 /* bpf programs can only be attached to u/kprobe or tracepoint */ 7498 return -EINVAL; 7499 7500 prog = bpf_prog_get(prog_fd); 7501 if (IS_ERR(prog)) 7502 return PTR_ERR(prog); 7503 7504 if ((is_kprobe && prog->type != BPF_PROG_TYPE_KPROBE) || 7505 (is_tracepoint && prog->type != BPF_PROG_TYPE_TRACEPOINT)) { 7506 /* valid fd, but invalid bpf program type */ 7507 bpf_prog_put(prog); 7508 return -EINVAL; 7509 } 7510 7511 if (is_tracepoint) { 7512 int off = trace_event_get_offsets(event->tp_event); 7513 7514 if (prog->aux->max_ctx_offset > off) { 7515 bpf_prog_put(prog); 7516 return -EACCES; 7517 } 7518 } 7519 event->tp_event->prog = prog; 7520 7521 return 0; 7522 } 7523 7524 static void perf_event_free_bpf_prog(struct perf_event *event) 7525 { 7526 struct bpf_prog *prog; 7527 7528 if (!event->tp_event) 7529 return; 7530 7531 prog = event->tp_event->prog; 7532 if (prog) { 7533 event->tp_event->prog = NULL; 7534 bpf_prog_put(prog); 7535 } 7536 } 7537 7538 #else 7539 7540 static inline void perf_tp_register(void) 7541 { 7542 } 7543 7544 static void perf_event_free_filter(struct perf_event *event) 7545 { 7546 } 7547 7548 static int perf_event_set_bpf_prog(struct perf_event *event, u32 prog_fd) 7549 { 7550 return -ENOENT; 7551 } 7552 7553 static void perf_event_free_bpf_prog(struct perf_event *event) 7554 { 7555 } 7556 #endif /* CONFIG_EVENT_TRACING */ 7557 7558 #ifdef CONFIG_HAVE_HW_BREAKPOINT 7559 void perf_bp_event(struct perf_event *bp, void *data) 7560 { 7561 struct perf_sample_data sample; 7562 struct pt_regs *regs = data; 7563 7564 perf_sample_data_init(&sample, bp->attr.bp_addr, 0); 7565 7566 if (!bp->hw.state && !perf_exclude_event(bp, regs)) 7567 perf_swevent_event(bp, 1, &sample, regs); 7568 } 7569 #endif 7570 7571 /* 7572 * Allocate a new address filter 7573 */ 7574 static struct perf_addr_filter * 7575 perf_addr_filter_new(struct perf_event *event, struct list_head *filters) 7576 { 7577 int node = cpu_to_node(event->cpu == -1 ? 0 : event->cpu); 7578 struct perf_addr_filter *filter; 7579 7580 filter = kzalloc_node(sizeof(*filter), GFP_KERNEL, node); 7581 if (!filter) 7582 return NULL; 7583 7584 INIT_LIST_HEAD(&filter->entry); 7585 list_add_tail(&filter->entry, filters); 7586 7587 return filter; 7588 } 7589 7590 static void free_filters_list(struct list_head *filters) 7591 { 7592 struct perf_addr_filter *filter, *iter; 7593 7594 list_for_each_entry_safe(filter, iter, filters, entry) { 7595 if (filter->inode) 7596 iput(filter->inode); 7597 list_del(&filter->entry); 7598 kfree(filter); 7599 } 7600 } 7601 7602 /* 7603 * Free existing address filters and optionally install new ones 7604 */ 7605 static void perf_addr_filters_splice(struct perf_event *event, 7606 struct list_head *head) 7607 { 7608 unsigned long flags; 7609 LIST_HEAD(list); 7610 7611 if (!has_addr_filter(event)) 7612 return; 7613 7614 /* don't bother with children, they don't have their own filters */ 7615 if (event->parent) 7616 return; 7617 7618 raw_spin_lock_irqsave(&event->addr_filters.lock, flags); 7619 7620 list_splice_init(&event->addr_filters.list, &list); 7621 if (head) 7622 list_splice(head, &event->addr_filters.list); 7623 7624 raw_spin_unlock_irqrestore(&event->addr_filters.lock, flags); 7625 7626 free_filters_list(&list); 7627 } 7628 7629 /* 7630 * Scan through mm's vmas and see if one of them matches the 7631 * @filter; if so, adjust filter's address range. 7632 * Called with mm::mmap_sem down for reading. 7633 */ 7634 static unsigned long perf_addr_filter_apply(struct perf_addr_filter *filter, 7635 struct mm_struct *mm) 7636 { 7637 struct vm_area_struct *vma; 7638 7639 for (vma = mm->mmap; vma; vma = vma->vm_next) { 7640 struct file *file = vma->vm_file; 7641 unsigned long off = vma->vm_pgoff << PAGE_SHIFT; 7642 unsigned long vma_size = vma->vm_end - vma->vm_start; 7643 7644 if (!file) 7645 continue; 7646 7647 if (!perf_addr_filter_match(filter, file, off, vma_size)) 7648 continue; 7649 7650 return vma->vm_start; 7651 } 7652 7653 return 0; 7654 } 7655 7656 /* 7657 * Update event's address range filters based on the 7658 * task's existing mappings, if any. 7659 */ 7660 static void perf_event_addr_filters_apply(struct perf_event *event) 7661 { 7662 struct perf_addr_filters_head *ifh = perf_event_addr_filters(event); 7663 struct task_struct *task = READ_ONCE(event->ctx->task); 7664 struct perf_addr_filter *filter; 7665 struct mm_struct *mm = NULL; 7666 unsigned int count = 0; 7667 unsigned long flags; 7668 7669 /* 7670 * We may observe TASK_TOMBSTONE, which means that the event tear-down 7671 * will stop on the parent's child_mutex that our caller is also holding 7672 */ 7673 if (task == TASK_TOMBSTONE) 7674 return; 7675 7676 mm = get_task_mm(event->ctx->task); 7677 if (!mm) 7678 goto restart; 7679 7680 down_read(&mm->mmap_sem); 7681 7682 raw_spin_lock_irqsave(&ifh->lock, flags); 7683 list_for_each_entry(filter, &ifh->list, entry) { 7684 event->addr_filters_offs[count] = 0; 7685 7686 if (perf_addr_filter_needs_mmap(filter)) 7687 event->addr_filters_offs[count] = 7688 perf_addr_filter_apply(filter, mm); 7689 7690 count++; 7691 } 7692 7693 event->addr_filters_gen++; 7694 raw_spin_unlock_irqrestore(&ifh->lock, flags); 7695 7696 up_read(&mm->mmap_sem); 7697 7698 mmput(mm); 7699 7700 restart: 7701 perf_event_restart(event); 7702 } 7703 7704 /* 7705 * Address range filtering: limiting the data to certain 7706 * instruction address ranges. Filters are ioctl()ed to us from 7707 * userspace as ascii strings. 7708 * 7709 * Filter string format: 7710 * 7711 * ACTION RANGE_SPEC 7712 * where ACTION is one of the 7713 * * "filter": limit the trace to this region 7714 * * "start": start tracing from this address 7715 * * "stop": stop tracing at this address/region; 7716 * RANGE_SPEC is 7717 * * for kernel addresses: <start address>[/<size>] 7718 * * for object files: <start address>[/<size>]@</path/to/object/file> 7719 * 7720 * if <size> is not specified, the range is treated as a single address. 7721 */ 7722 enum { 7723 IF_ACT_FILTER, 7724 IF_ACT_START, 7725 IF_ACT_STOP, 7726 IF_SRC_FILE, 7727 IF_SRC_KERNEL, 7728 IF_SRC_FILEADDR, 7729 IF_SRC_KERNELADDR, 7730 }; 7731 7732 enum { 7733 IF_STATE_ACTION = 0, 7734 IF_STATE_SOURCE, 7735 IF_STATE_END, 7736 }; 7737 7738 static const match_table_t if_tokens = { 7739 { IF_ACT_FILTER, "filter" }, 7740 { IF_ACT_START, "start" }, 7741 { IF_ACT_STOP, "stop" }, 7742 { IF_SRC_FILE, "%u/%u@%s" }, 7743 { IF_SRC_KERNEL, "%u/%u" }, 7744 { IF_SRC_FILEADDR, "%u@%s" }, 7745 { IF_SRC_KERNELADDR, "%u" }, 7746 }; 7747 7748 /* 7749 * Address filter string parser 7750 */ 7751 static int 7752 perf_event_parse_addr_filter(struct perf_event *event, char *fstr, 7753 struct list_head *filters) 7754 { 7755 struct perf_addr_filter *filter = NULL; 7756 char *start, *orig, *filename = NULL; 7757 struct path path; 7758 substring_t args[MAX_OPT_ARGS]; 7759 int state = IF_STATE_ACTION, token; 7760 unsigned int kernel = 0; 7761 int ret = -EINVAL; 7762 7763 orig = fstr = kstrdup(fstr, GFP_KERNEL); 7764 if (!fstr) 7765 return -ENOMEM; 7766 7767 while ((start = strsep(&fstr, " ,\n")) != NULL) { 7768 ret = -EINVAL; 7769 7770 if (!*start) 7771 continue; 7772 7773 /* filter definition begins */ 7774 if (state == IF_STATE_ACTION) { 7775 filter = perf_addr_filter_new(event, filters); 7776 if (!filter) 7777 goto fail; 7778 } 7779 7780 token = match_token(start, if_tokens, args); 7781 switch (token) { 7782 case IF_ACT_FILTER: 7783 case IF_ACT_START: 7784 filter->filter = 1; 7785 7786 case IF_ACT_STOP: 7787 if (state != IF_STATE_ACTION) 7788 goto fail; 7789 7790 state = IF_STATE_SOURCE; 7791 break; 7792 7793 case IF_SRC_KERNELADDR: 7794 case IF_SRC_KERNEL: 7795 kernel = 1; 7796 7797 case IF_SRC_FILEADDR: 7798 case IF_SRC_FILE: 7799 if (state != IF_STATE_SOURCE) 7800 goto fail; 7801 7802 if (token == IF_SRC_FILE || token == IF_SRC_KERNEL) 7803 filter->range = 1; 7804 7805 *args[0].to = 0; 7806 ret = kstrtoul(args[0].from, 0, &filter->offset); 7807 if (ret) 7808 goto fail; 7809 7810 if (filter->range) { 7811 *args[1].to = 0; 7812 ret = kstrtoul(args[1].from, 0, &filter->size); 7813 if (ret) 7814 goto fail; 7815 } 7816 7817 if (token == IF_SRC_FILE) { 7818 filename = match_strdup(&args[2]); 7819 if (!filename) { 7820 ret = -ENOMEM; 7821 goto fail; 7822 } 7823 } 7824 7825 state = IF_STATE_END; 7826 break; 7827 7828 default: 7829 goto fail; 7830 } 7831 7832 /* 7833 * Filter definition is fully parsed, validate and install it. 7834 * Make sure that it doesn't contradict itself or the event's 7835 * attribute. 7836 */ 7837 if (state == IF_STATE_END) { 7838 if (kernel && event->attr.exclude_kernel) 7839 goto fail; 7840 7841 if (!kernel) { 7842 if (!filename) 7843 goto fail; 7844 7845 /* look up the path and grab its inode */ 7846 ret = kern_path(filename, LOOKUP_FOLLOW, &path); 7847 if (ret) 7848 goto fail_free_name; 7849 7850 filter->inode = igrab(d_inode(path.dentry)); 7851 path_put(&path); 7852 kfree(filename); 7853 filename = NULL; 7854 7855 ret = -EINVAL; 7856 if (!filter->inode || 7857 !S_ISREG(filter->inode->i_mode)) 7858 /* free_filters_list() will iput() */ 7859 goto fail; 7860 } 7861 7862 /* ready to consume more filters */ 7863 state = IF_STATE_ACTION; 7864 filter = NULL; 7865 } 7866 } 7867 7868 if (state != IF_STATE_ACTION) 7869 goto fail; 7870 7871 kfree(orig); 7872 7873 return 0; 7874 7875 fail_free_name: 7876 kfree(filename); 7877 fail: 7878 free_filters_list(filters); 7879 kfree(orig); 7880 7881 return ret; 7882 } 7883 7884 static int 7885 perf_event_set_addr_filter(struct perf_event *event, char *filter_str) 7886 { 7887 LIST_HEAD(filters); 7888 int ret; 7889 7890 /* 7891 * Since this is called in perf_ioctl() path, we're already holding 7892 * ctx::mutex. 7893 */ 7894 lockdep_assert_held(&event->ctx->mutex); 7895 7896 if (WARN_ON_ONCE(event->parent)) 7897 return -EINVAL; 7898 7899 /* 7900 * For now, we only support filtering in per-task events; doing so 7901 * for CPU-wide events requires additional context switching trickery, 7902 * since same object code will be mapped at different virtual 7903 * addresses in different processes. 7904 */ 7905 if (!event->ctx->task) 7906 return -EOPNOTSUPP; 7907 7908 ret = perf_event_parse_addr_filter(event, filter_str, &filters); 7909 if (ret) 7910 return ret; 7911 7912 ret = event->pmu->addr_filters_validate(&filters); 7913 if (ret) { 7914 free_filters_list(&filters); 7915 return ret; 7916 } 7917 7918 /* remove existing filters, if any */ 7919 perf_addr_filters_splice(event, &filters); 7920 7921 /* install new filters */ 7922 perf_event_for_each_child(event, perf_event_addr_filters_apply); 7923 7924 return ret; 7925 } 7926 7927 static int perf_event_set_filter(struct perf_event *event, void __user *arg) 7928 { 7929 char *filter_str; 7930 int ret = -EINVAL; 7931 7932 if ((event->attr.type != PERF_TYPE_TRACEPOINT || 7933 !IS_ENABLED(CONFIG_EVENT_TRACING)) && 7934 !has_addr_filter(event)) 7935 return -EINVAL; 7936 7937 filter_str = strndup_user(arg, PAGE_SIZE); 7938 if (IS_ERR(filter_str)) 7939 return PTR_ERR(filter_str); 7940 7941 if (IS_ENABLED(CONFIG_EVENT_TRACING) && 7942 event->attr.type == PERF_TYPE_TRACEPOINT) 7943 ret = ftrace_profile_set_filter(event, event->attr.config, 7944 filter_str); 7945 else if (has_addr_filter(event)) 7946 ret = perf_event_set_addr_filter(event, filter_str); 7947 7948 kfree(filter_str); 7949 return ret; 7950 } 7951 7952 /* 7953 * hrtimer based swevent callback 7954 */ 7955 7956 static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer) 7957 { 7958 enum hrtimer_restart ret = HRTIMER_RESTART; 7959 struct perf_sample_data data; 7960 struct pt_regs *regs; 7961 struct perf_event *event; 7962 u64 period; 7963 7964 event = container_of(hrtimer, struct perf_event, hw.hrtimer); 7965 7966 if (event->state != PERF_EVENT_STATE_ACTIVE) 7967 return HRTIMER_NORESTART; 7968 7969 event->pmu->read(event); 7970 7971 perf_sample_data_init(&data, 0, event->hw.last_period); 7972 regs = get_irq_regs(); 7973 7974 if (regs && !perf_exclude_event(event, regs)) { 7975 if (!(event->attr.exclude_idle && is_idle_task(current))) 7976 if (__perf_event_overflow(event, 1, &data, regs)) 7977 ret = HRTIMER_NORESTART; 7978 } 7979 7980 period = max_t(u64, 10000, event->hw.sample_period); 7981 hrtimer_forward_now(hrtimer, ns_to_ktime(period)); 7982 7983 return ret; 7984 } 7985 7986 static void perf_swevent_start_hrtimer(struct perf_event *event) 7987 { 7988 struct hw_perf_event *hwc = &event->hw; 7989 s64 period; 7990 7991 if (!is_sampling_event(event)) 7992 return; 7993 7994 period = local64_read(&hwc->period_left); 7995 if (period) { 7996 if (period < 0) 7997 period = 10000; 7998 7999 local64_set(&hwc->period_left, 0); 8000 } else { 8001 period = max_t(u64, 10000, hwc->sample_period); 8002 } 8003 hrtimer_start(&hwc->hrtimer, ns_to_ktime(period), 8004 HRTIMER_MODE_REL_PINNED); 8005 } 8006 8007 static void perf_swevent_cancel_hrtimer(struct perf_event *event) 8008 { 8009 struct hw_perf_event *hwc = &event->hw; 8010 8011 if (is_sampling_event(event)) { 8012 ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer); 8013 local64_set(&hwc->period_left, ktime_to_ns(remaining)); 8014 8015 hrtimer_cancel(&hwc->hrtimer); 8016 } 8017 } 8018 8019 static void perf_swevent_init_hrtimer(struct perf_event *event) 8020 { 8021 struct hw_perf_event *hwc = &event->hw; 8022 8023 if (!is_sampling_event(event)) 8024 return; 8025 8026 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 8027 hwc->hrtimer.function = perf_swevent_hrtimer; 8028 8029 /* 8030 * Since hrtimers have a fixed rate, we can do a static freq->period 8031 * mapping and avoid the whole period adjust feedback stuff. 8032 */ 8033 if (event->attr.freq) { 8034 long freq = event->attr.sample_freq; 8035 8036 event->attr.sample_period = NSEC_PER_SEC / freq; 8037 hwc->sample_period = event->attr.sample_period; 8038 local64_set(&hwc->period_left, hwc->sample_period); 8039 hwc->last_period = hwc->sample_period; 8040 event->attr.freq = 0; 8041 } 8042 } 8043 8044 /* 8045 * Software event: cpu wall time clock 8046 */ 8047 8048 static void cpu_clock_event_update(struct perf_event *event) 8049 { 8050 s64 prev; 8051 u64 now; 8052 8053 now = local_clock(); 8054 prev = local64_xchg(&event->hw.prev_count, now); 8055 local64_add(now - prev, &event->count); 8056 } 8057 8058 static void cpu_clock_event_start(struct perf_event *event, int flags) 8059 { 8060 local64_set(&event->hw.prev_count, local_clock()); 8061 perf_swevent_start_hrtimer(event); 8062 } 8063 8064 static void cpu_clock_event_stop(struct perf_event *event, int flags) 8065 { 8066 perf_swevent_cancel_hrtimer(event); 8067 cpu_clock_event_update(event); 8068 } 8069 8070 static int cpu_clock_event_add(struct perf_event *event, int flags) 8071 { 8072 if (flags & PERF_EF_START) 8073 cpu_clock_event_start(event, flags); 8074 perf_event_update_userpage(event); 8075 8076 return 0; 8077 } 8078 8079 static void cpu_clock_event_del(struct perf_event *event, int flags) 8080 { 8081 cpu_clock_event_stop(event, flags); 8082 } 8083 8084 static void cpu_clock_event_read(struct perf_event *event) 8085 { 8086 cpu_clock_event_update(event); 8087 } 8088 8089 static int cpu_clock_event_init(struct perf_event *event) 8090 { 8091 if (event->attr.type != PERF_TYPE_SOFTWARE) 8092 return -ENOENT; 8093 8094 if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK) 8095 return -ENOENT; 8096 8097 /* 8098 * no branch sampling for software events 8099 */ 8100 if (has_branch_stack(event)) 8101 return -EOPNOTSUPP; 8102 8103 perf_swevent_init_hrtimer(event); 8104 8105 return 0; 8106 } 8107 8108 static struct pmu perf_cpu_clock = { 8109 .task_ctx_nr = perf_sw_context, 8110 8111 .capabilities = PERF_PMU_CAP_NO_NMI, 8112 8113 .event_init = cpu_clock_event_init, 8114 .add = cpu_clock_event_add, 8115 .del = cpu_clock_event_del, 8116 .start = cpu_clock_event_start, 8117 .stop = cpu_clock_event_stop, 8118 .read = cpu_clock_event_read, 8119 }; 8120 8121 /* 8122 * Software event: task time clock 8123 */ 8124 8125 static void task_clock_event_update(struct perf_event *event, u64 now) 8126 { 8127 u64 prev; 8128 s64 delta; 8129 8130 prev = local64_xchg(&event->hw.prev_count, now); 8131 delta = now - prev; 8132 local64_add(delta, &event->count); 8133 } 8134 8135 static void task_clock_event_start(struct perf_event *event, int flags) 8136 { 8137 local64_set(&event->hw.prev_count, event->ctx->time); 8138 perf_swevent_start_hrtimer(event); 8139 } 8140 8141 static void task_clock_event_stop(struct perf_event *event, int flags) 8142 { 8143 perf_swevent_cancel_hrtimer(event); 8144 task_clock_event_update(event, event->ctx->time); 8145 } 8146 8147 static int task_clock_event_add(struct perf_event *event, int flags) 8148 { 8149 if (flags & PERF_EF_START) 8150 task_clock_event_start(event, flags); 8151 perf_event_update_userpage(event); 8152 8153 return 0; 8154 } 8155 8156 static void task_clock_event_del(struct perf_event *event, int flags) 8157 { 8158 task_clock_event_stop(event, PERF_EF_UPDATE); 8159 } 8160 8161 static void task_clock_event_read(struct perf_event *event) 8162 { 8163 u64 now = perf_clock(); 8164 u64 delta = now - event->ctx->timestamp; 8165 u64 time = event->ctx->time + delta; 8166 8167 task_clock_event_update(event, time); 8168 } 8169 8170 static int task_clock_event_init(struct perf_event *event) 8171 { 8172 if (event->attr.type != PERF_TYPE_SOFTWARE) 8173 return -ENOENT; 8174 8175 if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK) 8176 return -ENOENT; 8177 8178 /* 8179 * no branch sampling for software events 8180 */ 8181 if (has_branch_stack(event)) 8182 return -EOPNOTSUPP; 8183 8184 perf_swevent_init_hrtimer(event); 8185 8186 return 0; 8187 } 8188 8189 static struct pmu perf_task_clock = { 8190 .task_ctx_nr = perf_sw_context, 8191 8192 .capabilities = PERF_PMU_CAP_NO_NMI, 8193 8194 .event_init = task_clock_event_init, 8195 .add = task_clock_event_add, 8196 .del = task_clock_event_del, 8197 .start = task_clock_event_start, 8198 .stop = task_clock_event_stop, 8199 .read = task_clock_event_read, 8200 }; 8201 8202 static void perf_pmu_nop_void(struct pmu *pmu) 8203 { 8204 } 8205 8206 static void perf_pmu_nop_txn(struct pmu *pmu, unsigned int flags) 8207 { 8208 } 8209 8210 static int perf_pmu_nop_int(struct pmu *pmu) 8211 { 8212 return 0; 8213 } 8214 8215 static DEFINE_PER_CPU(unsigned int, nop_txn_flags); 8216 8217 static void perf_pmu_start_txn(struct pmu *pmu, unsigned int flags) 8218 { 8219 __this_cpu_write(nop_txn_flags, flags); 8220 8221 if (flags & ~PERF_PMU_TXN_ADD) 8222 return; 8223 8224 perf_pmu_disable(pmu); 8225 } 8226 8227 static int perf_pmu_commit_txn(struct pmu *pmu) 8228 { 8229 unsigned int flags = __this_cpu_read(nop_txn_flags); 8230 8231 __this_cpu_write(nop_txn_flags, 0); 8232 8233 if (flags & ~PERF_PMU_TXN_ADD) 8234 return 0; 8235 8236 perf_pmu_enable(pmu); 8237 return 0; 8238 } 8239 8240 static void perf_pmu_cancel_txn(struct pmu *pmu) 8241 { 8242 unsigned int flags = __this_cpu_read(nop_txn_flags); 8243 8244 __this_cpu_write(nop_txn_flags, 0); 8245 8246 if (flags & ~PERF_PMU_TXN_ADD) 8247 return; 8248 8249 perf_pmu_enable(pmu); 8250 } 8251 8252 static int perf_event_idx_default(struct perf_event *event) 8253 { 8254 return 0; 8255 } 8256 8257 /* 8258 * Ensures all contexts with the same task_ctx_nr have the same 8259 * pmu_cpu_context too. 8260 */ 8261 static struct perf_cpu_context __percpu *find_pmu_context(int ctxn) 8262 { 8263 struct pmu *pmu; 8264 8265 if (ctxn < 0) 8266 return NULL; 8267 8268 list_for_each_entry(pmu, &pmus, entry) { 8269 if (pmu->task_ctx_nr == ctxn) 8270 return pmu->pmu_cpu_context; 8271 } 8272 8273 return NULL; 8274 } 8275 8276 static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu) 8277 { 8278 int cpu; 8279 8280 for_each_possible_cpu(cpu) { 8281 struct perf_cpu_context *cpuctx; 8282 8283 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 8284 8285 if (cpuctx->unique_pmu == old_pmu) 8286 cpuctx->unique_pmu = pmu; 8287 } 8288 } 8289 8290 static void free_pmu_context(struct pmu *pmu) 8291 { 8292 struct pmu *i; 8293 8294 mutex_lock(&pmus_lock); 8295 /* 8296 * Like a real lame refcount. 8297 */ 8298 list_for_each_entry(i, &pmus, entry) { 8299 if (i->pmu_cpu_context == pmu->pmu_cpu_context) { 8300 update_pmu_context(i, pmu); 8301 goto out; 8302 } 8303 } 8304 8305 free_percpu(pmu->pmu_cpu_context); 8306 out: 8307 mutex_unlock(&pmus_lock); 8308 } 8309 8310 /* 8311 * Let userspace know that this PMU supports address range filtering: 8312 */ 8313 static ssize_t nr_addr_filters_show(struct device *dev, 8314 struct device_attribute *attr, 8315 char *page) 8316 { 8317 struct pmu *pmu = dev_get_drvdata(dev); 8318 8319 return snprintf(page, PAGE_SIZE - 1, "%d\n", pmu->nr_addr_filters); 8320 } 8321 DEVICE_ATTR_RO(nr_addr_filters); 8322 8323 static struct idr pmu_idr; 8324 8325 static ssize_t 8326 type_show(struct device *dev, struct device_attribute *attr, char *page) 8327 { 8328 struct pmu *pmu = dev_get_drvdata(dev); 8329 8330 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type); 8331 } 8332 static DEVICE_ATTR_RO(type); 8333 8334 static ssize_t 8335 perf_event_mux_interval_ms_show(struct device *dev, 8336 struct device_attribute *attr, 8337 char *page) 8338 { 8339 struct pmu *pmu = dev_get_drvdata(dev); 8340 8341 return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms); 8342 } 8343 8344 static DEFINE_MUTEX(mux_interval_mutex); 8345 8346 static ssize_t 8347 perf_event_mux_interval_ms_store(struct device *dev, 8348 struct device_attribute *attr, 8349 const char *buf, size_t count) 8350 { 8351 struct pmu *pmu = dev_get_drvdata(dev); 8352 int timer, cpu, ret; 8353 8354 ret = kstrtoint(buf, 0, &timer); 8355 if (ret) 8356 return ret; 8357 8358 if (timer < 1) 8359 return -EINVAL; 8360 8361 /* same value, noting to do */ 8362 if (timer == pmu->hrtimer_interval_ms) 8363 return count; 8364 8365 mutex_lock(&mux_interval_mutex); 8366 pmu->hrtimer_interval_ms = timer; 8367 8368 /* update all cpuctx for this PMU */ 8369 get_online_cpus(); 8370 for_each_online_cpu(cpu) { 8371 struct perf_cpu_context *cpuctx; 8372 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 8373 cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer); 8374 8375 cpu_function_call(cpu, 8376 (remote_function_f)perf_mux_hrtimer_restart, cpuctx); 8377 } 8378 put_online_cpus(); 8379 mutex_unlock(&mux_interval_mutex); 8380 8381 return count; 8382 } 8383 static DEVICE_ATTR_RW(perf_event_mux_interval_ms); 8384 8385 static struct attribute *pmu_dev_attrs[] = { 8386 &dev_attr_type.attr, 8387 &dev_attr_perf_event_mux_interval_ms.attr, 8388 NULL, 8389 }; 8390 ATTRIBUTE_GROUPS(pmu_dev); 8391 8392 static int pmu_bus_running; 8393 static struct bus_type pmu_bus = { 8394 .name = "event_source", 8395 .dev_groups = pmu_dev_groups, 8396 }; 8397 8398 static void pmu_dev_release(struct device *dev) 8399 { 8400 kfree(dev); 8401 } 8402 8403 static int pmu_dev_alloc(struct pmu *pmu) 8404 { 8405 int ret = -ENOMEM; 8406 8407 pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL); 8408 if (!pmu->dev) 8409 goto out; 8410 8411 pmu->dev->groups = pmu->attr_groups; 8412 device_initialize(pmu->dev); 8413 ret = dev_set_name(pmu->dev, "%s", pmu->name); 8414 if (ret) 8415 goto free_dev; 8416 8417 dev_set_drvdata(pmu->dev, pmu); 8418 pmu->dev->bus = &pmu_bus; 8419 pmu->dev->release = pmu_dev_release; 8420 ret = device_add(pmu->dev); 8421 if (ret) 8422 goto free_dev; 8423 8424 /* For PMUs with address filters, throw in an extra attribute: */ 8425 if (pmu->nr_addr_filters) 8426 ret = device_create_file(pmu->dev, &dev_attr_nr_addr_filters); 8427 8428 if (ret) 8429 goto del_dev; 8430 8431 out: 8432 return ret; 8433 8434 del_dev: 8435 device_del(pmu->dev); 8436 8437 free_dev: 8438 put_device(pmu->dev); 8439 goto out; 8440 } 8441 8442 static struct lock_class_key cpuctx_mutex; 8443 static struct lock_class_key cpuctx_lock; 8444 8445 int perf_pmu_register(struct pmu *pmu, const char *name, int type) 8446 { 8447 int cpu, ret; 8448 8449 mutex_lock(&pmus_lock); 8450 ret = -ENOMEM; 8451 pmu->pmu_disable_count = alloc_percpu(int); 8452 if (!pmu->pmu_disable_count) 8453 goto unlock; 8454 8455 pmu->type = -1; 8456 if (!name) 8457 goto skip_type; 8458 pmu->name = name; 8459 8460 if (type < 0) { 8461 type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL); 8462 if (type < 0) { 8463 ret = type; 8464 goto free_pdc; 8465 } 8466 } 8467 pmu->type = type; 8468 8469 if (pmu_bus_running) { 8470 ret = pmu_dev_alloc(pmu); 8471 if (ret) 8472 goto free_idr; 8473 } 8474 8475 skip_type: 8476 if (pmu->task_ctx_nr == perf_hw_context) { 8477 static int hw_context_taken = 0; 8478 8479 /* 8480 * Other than systems with heterogeneous CPUs, it never makes 8481 * sense for two PMUs to share perf_hw_context. PMUs which are 8482 * uncore must use perf_invalid_context. 8483 */ 8484 if (WARN_ON_ONCE(hw_context_taken && 8485 !(pmu->capabilities & PERF_PMU_CAP_HETEROGENEOUS_CPUS))) 8486 pmu->task_ctx_nr = perf_invalid_context; 8487 8488 hw_context_taken = 1; 8489 } 8490 8491 pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr); 8492 if (pmu->pmu_cpu_context) 8493 goto got_cpu_context; 8494 8495 ret = -ENOMEM; 8496 pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context); 8497 if (!pmu->pmu_cpu_context) 8498 goto free_dev; 8499 8500 for_each_possible_cpu(cpu) { 8501 struct perf_cpu_context *cpuctx; 8502 8503 cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu); 8504 __perf_event_init_context(&cpuctx->ctx); 8505 lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex); 8506 lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock); 8507 cpuctx->ctx.pmu = pmu; 8508 8509 __perf_mux_hrtimer_init(cpuctx, cpu); 8510 8511 cpuctx->unique_pmu = pmu; 8512 } 8513 8514 got_cpu_context: 8515 if (!pmu->start_txn) { 8516 if (pmu->pmu_enable) { 8517 /* 8518 * If we have pmu_enable/pmu_disable calls, install 8519 * transaction stubs that use that to try and batch 8520 * hardware accesses. 8521 */ 8522 pmu->start_txn = perf_pmu_start_txn; 8523 pmu->commit_txn = perf_pmu_commit_txn; 8524 pmu->cancel_txn = perf_pmu_cancel_txn; 8525 } else { 8526 pmu->start_txn = perf_pmu_nop_txn; 8527 pmu->commit_txn = perf_pmu_nop_int; 8528 pmu->cancel_txn = perf_pmu_nop_void; 8529 } 8530 } 8531 8532 if (!pmu->pmu_enable) { 8533 pmu->pmu_enable = perf_pmu_nop_void; 8534 pmu->pmu_disable = perf_pmu_nop_void; 8535 } 8536 8537 if (!pmu->event_idx) 8538 pmu->event_idx = perf_event_idx_default; 8539 8540 list_add_rcu(&pmu->entry, &pmus); 8541 atomic_set(&pmu->exclusive_cnt, 0); 8542 ret = 0; 8543 unlock: 8544 mutex_unlock(&pmus_lock); 8545 8546 return ret; 8547 8548 free_dev: 8549 device_del(pmu->dev); 8550 put_device(pmu->dev); 8551 8552 free_idr: 8553 if (pmu->type >= PERF_TYPE_MAX) 8554 idr_remove(&pmu_idr, pmu->type); 8555 8556 free_pdc: 8557 free_percpu(pmu->pmu_disable_count); 8558 goto unlock; 8559 } 8560 EXPORT_SYMBOL_GPL(perf_pmu_register); 8561 8562 void perf_pmu_unregister(struct pmu *pmu) 8563 { 8564 mutex_lock(&pmus_lock); 8565 list_del_rcu(&pmu->entry); 8566 mutex_unlock(&pmus_lock); 8567 8568 /* 8569 * We dereference the pmu list under both SRCU and regular RCU, so 8570 * synchronize against both of those. 8571 */ 8572 synchronize_srcu(&pmus_srcu); 8573 synchronize_rcu(); 8574 8575 free_percpu(pmu->pmu_disable_count); 8576 if (pmu->type >= PERF_TYPE_MAX) 8577 idr_remove(&pmu_idr, pmu->type); 8578 if (pmu->nr_addr_filters) 8579 device_remove_file(pmu->dev, &dev_attr_nr_addr_filters); 8580 device_del(pmu->dev); 8581 put_device(pmu->dev); 8582 free_pmu_context(pmu); 8583 } 8584 EXPORT_SYMBOL_GPL(perf_pmu_unregister); 8585 8586 static int perf_try_init_event(struct pmu *pmu, struct perf_event *event) 8587 { 8588 struct perf_event_context *ctx = NULL; 8589 int ret; 8590 8591 if (!try_module_get(pmu->module)) 8592 return -ENODEV; 8593 8594 if (event->group_leader != event) { 8595 /* 8596 * This ctx->mutex can nest when we're called through 8597 * inheritance. See the perf_event_ctx_lock_nested() comment. 8598 */ 8599 ctx = perf_event_ctx_lock_nested(event->group_leader, 8600 SINGLE_DEPTH_NESTING); 8601 BUG_ON(!ctx); 8602 } 8603 8604 event->pmu = pmu; 8605 ret = pmu->event_init(event); 8606 8607 if (ctx) 8608 perf_event_ctx_unlock(event->group_leader, ctx); 8609 8610 if (ret) 8611 module_put(pmu->module); 8612 8613 return ret; 8614 } 8615 8616 static struct pmu *perf_init_event(struct perf_event *event) 8617 { 8618 struct pmu *pmu = NULL; 8619 int idx; 8620 int ret; 8621 8622 idx = srcu_read_lock(&pmus_srcu); 8623 8624 rcu_read_lock(); 8625 pmu = idr_find(&pmu_idr, event->attr.type); 8626 rcu_read_unlock(); 8627 if (pmu) { 8628 ret = perf_try_init_event(pmu, event); 8629 if (ret) 8630 pmu = ERR_PTR(ret); 8631 goto unlock; 8632 } 8633 8634 list_for_each_entry_rcu(pmu, &pmus, entry) { 8635 ret = perf_try_init_event(pmu, event); 8636 if (!ret) 8637 goto unlock; 8638 8639 if (ret != -ENOENT) { 8640 pmu = ERR_PTR(ret); 8641 goto unlock; 8642 } 8643 } 8644 pmu = ERR_PTR(-ENOENT); 8645 unlock: 8646 srcu_read_unlock(&pmus_srcu, idx); 8647 8648 return pmu; 8649 } 8650 8651 static void account_event_cpu(struct perf_event *event, int cpu) 8652 { 8653 if (event->parent) 8654 return; 8655 8656 if (is_cgroup_event(event)) 8657 atomic_inc(&per_cpu(perf_cgroup_events, cpu)); 8658 } 8659 8660 /* Freq events need the tick to stay alive (see perf_event_task_tick). */ 8661 static void account_freq_event_nohz(void) 8662 { 8663 #ifdef CONFIG_NO_HZ_FULL 8664 /* Lock so we don't race with concurrent unaccount */ 8665 spin_lock(&nr_freq_lock); 8666 if (atomic_inc_return(&nr_freq_events) == 1) 8667 tick_nohz_dep_set(TICK_DEP_BIT_PERF_EVENTS); 8668 spin_unlock(&nr_freq_lock); 8669 #endif 8670 } 8671 8672 static void account_freq_event(void) 8673 { 8674 if (tick_nohz_full_enabled()) 8675 account_freq_event_nohz(); 8676 else 8677 atomic_inc(&nr_freq_events); 8678 } 8679 8680 8681 static void account_event(struct perf_event *event) 8682 { 8683 bool inc = false; 8684 8685 if (event->parent) 8686 return; 8687 8688 if (event->attach_state & PERF_ATTACH_TASK) 8689 inc = true; 8690 if (event->attr.mmap || event->attr.mmap_data) 8691 atomic_inc(&nr_mmap_events); 8692 if (event->attr.comm) 8693 atomic_inc(&nr_comm_events); 8694 if (event->attr.task) 8695 atomic_inc(&nr_task_events); 8696 if (event->attr.freq) 8697 account_freq_event(); 8698 if (event->attr.context_switch) { 8699 atomic_inc(&nr_switch_events); 8700 inc = true; 8701 } 8702 if (has_branch_stack(event)) 8703 inc = true; 8704 if (is_cgroup_event(event)) 8705 inc = true; 8706 8707 if (inc) { 8708 if (atomic_inc_not_zero(&perf_sched_count)) 8709 goto enabled; 8710 8711 mutex_lock(&perf_sched_mutex); 8712 if (!atomic_read(&perf_sched_count)) { 8713 static_branch_enable(&perf_sched_events); 8714 /* 8715 * Guarantee that all CPUs observe they key change and 8716 * call the perf scheduling hooks before proceeding to 8717 * install events that need them. 8718 */ 8719 synchronize_sched(); 8720 } 8721 /* 8722 * Now that we have waited for the sync_sched(), allow further 8723 * increments to by-pass the mutex. 8724 */ 8725 atomic_inc(&perf_sched_count); 8726 mutex_unlock(&perf_sched_mutex); 8727 } 8728 enabled: 8729 8730 account_event_cpu(event, event->cpu); 8731 } 8732 8733 /* 8734 * Allocate and initialize a event structure 8735 */ 8736 static struct perf_event * 8737 perf_event_alloc(struct perf_event_attr *attr, int cpu, 8738 struct task_struct *task, 8739 struct perf_event *group_leader, 8740 struct perf_event *parent_event, 8741 perf_overflow_handler_t overflow_handler, 8742 void *context, int cgroup_fd) 8743 { 8744 struct pmu *pmu; 8745 struct perf_event *event; 8746 struct hw_perf_event *hwc; 8747 long err = -EINVAL; 8748 8749 if ((unsigned)cpu >= nr_cpu_ids) { 8750 if (!task || cpu != -1) 8751 return ERR_PTR(-EINVAL); 8752 } 8753 8754 event = kzalloc(sizeof(*event), GFP_KERNEL); 8755 if (!event) 8756 return ERR_PTR(-ENOMEM); 8757 8758 /* 8759 * Single events are their own group leaders, with an 8760 * empty sibling list: 8761 */ 8762 if (!group_leader) 8763 group_leader = event; 8764 8765 mutex_init(&event->child_mutex); 8766 INIT_LIST_HEAD(&event->child_list); 8767 8768 INIT_LIST_HEAD(&event->group_entry); 8769 INIT_LIST_HEAD(&event->event_entry); 8770 INIT_LIST_HEAD(&event->sibling_list); 8771 INIT_LIST_HEAD(&event->rb_entry); 8772 INIT_LIST_HEAD(&event->active_entry); 8773 INIT_LIST_HEAD(&event->addr_filters.list); 8774 INIT_HLIST_NODE(&event->hlist_entry); 8775 8776 8777 init_waitqueue_head(&event->waitq); 8778 init_irq_work(&event->pending, perf_pending_event); 8779 8780 mutex_init(&event->mmap_mutex); 8781 raw_spin_lock_init(&event->addr_filters.lock); 8782 8783 atomic_long_set(&event->refcount, 1); 8784 event->cpu = cpu; 8785 event->attr = *attr; 8786 event->group_leader = group_leader; 8787 event->pmu = NULL; 8788 event->oncpu = -1; 8789 8790 event->parent = parent_event; 8791 8792 event->ns = get_pid_ns(task_active_pid_ns(current)); 8793 event->id = atomic64_inc_return(&perf_event_id); 8794 8795 event->state = PERF_EVENT_STATE_INACTIVE; 8796 8797 if (task) { 8798 event->attach_state = PERF_ATTACH_TASK; 8799 /* 8800 * XXX pmu::event_init needs to know what task to account to 8801 * and we cannot use the ctx information because we need the 8802 * pmu before we get a ctx. 8803 */ 8804 event->hw.target = task; 8805 } 8806 8807 event->clock = &local_clock; 8808 if (parent_event) 8809 event->clock = parent_event->clock; 8810 8811 if (!overflow_handler && parent_event) { 8812 overflow_handler = parent_event->overflow_handler; 8813 context = parent_event->overflow_handler_context; 8814 } 8815 8816 if (overflow_handler) { 8817 event->overflow_handler = overflow_handler; 8818 event->overflow_handler_context = context; 8819 } else if (is_write_backward(event)){ 8820 event->overflow_handler = perf_event_output_backward; 8821 event->overflow_handler_context = NULL; 8822 } else { 8823 event->overflow_handler = perf_event_output_forward; 8824 event->overflow_handler_context = NULL; 8825 } 8826 8827 perf_event__state_init(event); 8828 8829 pmu = NULL; 8830 8831 hwc = &event->hw; 8832 hwc->sample_period = attr->sample_period; 8833 if (attr->freq && attr->sample_freq) 8834 hwc->sample_period = 1; 8835 hwc->last_period = hwc->sample_period; 8836 8837 local64_set(&hwc->period_left, hwc->sample_period); 8838 8839 /* 8840 * we currently do not support PERF_FORMAT_GROUP on inherited events 8841 */ 8842 if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP)) 8843 goto err_ns; 8844 8845 if (!has_branch_stack(event)) 8846 event->attr.branch_sample_type = 0; 8847 8848 if (cgroup_fd != -1) { 8849 err = perf_cgroup_connect(cgroup_fd, event, attr, group_leader); 8850 if (err) 8851 goto err_ns; 8852 } 8853 8854 pmu = perf_init_event(event); 8855 if (!pmu) 8856 goto err_ns; 8857 else if (IS_ERR(pmu)) { 8858 err = PTR_ERR(pmu); 8859 goto err_ns; 8860 } 8861 8862 err = exclusive_event_init(event); 8863 if (err) 8864 goto err_pmu; 8865 8866 if (has_addr_filter(event)) { 8867 event->addr_filters_offs = kcalloc(pmu->nr_addr_filters, 8868 sizeof(unsigned long), 8869 GFP_KERNEL); 8870 if (!event->addr_filters_offs) 8871 goto err_per_task; 8872 8873 /* force hw sync on the address filters */ 8874 event->addr_filters_gen = 1; 8875 } 8876 8877 if (!event->parent) { 8878 if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) { 8879 err = get_callchain_buffers(); 8880 if (err) 8881 goto err_addr_filters; 8882 } 8883 } 8884 8885 /* symmetric to unaccount_event() in _free_event() */ 8886 account_event(event); 8887 8888 return event; 8889 8890 err_addr_filters: 8891 kfree(event->addr_filters_offs); 8892 8893 err_per_task: 8894 exclusive_event_destroy(event); 8895 8896 err_pmu: 8897 if (event->destroy) 8898 event->destroy(event); 8899 module_put(pmu->module); 8900 err_ns: 8901 if (is_cgroup_event(event)) 8902 perf_detach_cgroup(event); 8903 if (event->ns) 8904 put_pid_ns(event->ns); 8905 kfree(event); 8906 8907 return ERR_PTR(err); 8908 } 8909 8910 static int perf_copy_attr(struct perf_event_attr __user *uattr, 8911 struct perf_event_attr *attr) 8912 { 8913 u32 size; 8914 int ret; 8915 8916 if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0)) 8917 return -EFAULT; 8918 8919 /* 8920 * zero the full structure, so that a short copy will be nice. 8921 */ 8922 memset(attr, 0, sizeof(*attr)); 8923 8924 ret = get_user(size, &uattr->size); 8925 if (ret) 8926 return ret; 8927 8928 if (size > PAGE_SIZE) /* silly large */ 8929 goto err_size; 8930 8931 if (!size) /* abi compat */ 8932 size = PERF_ATTR_SIZE_VER0; 8933 8934 if (size < PERF_ATTR_SIZE_VER0) 8935 goto err_size; 8936 8937 /* 8938 * If we're handed a bigger struct than we know of, 8939 * ensure all the unknown bits are 0 - i.e. new 8940 * user-space does not rely on any kernel feature 8941 * extensions we dont know about yet. 8942 */ 8943 if (size > sizeof(*attr)) { 8944 unsigned char __user *addr; 8945 unsigned char __user *end; 8946 unsigned char val; 8947 8948 addr = (void __user *)uattr + sizeof(*attr); 8949 end = (void __user *)uattr + size; 8950 8951 for (; addr < end; addr++) { 8952 ret = get_user(val, addr); 8953 if (ret) 8954 return ret; 8955 if (val) 8956 goto err_size; 8957 } 8958 size = sizeof(*attr); 8959 } 8960 8961 ret = copy_from_user(attr, uattr, size); 8962 if (ret) 8963 return -EFAULT; 8964 8965 if (attr->__reserved_1) 8966 return -EINVAL; 8967 8968 if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) 8969 return -EINVAL; 8970 8971 if (attr->read_format & ~(PERF_FORMAT_MAX-1)) 8972 return -EINVAL; 8973 8974 if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) { 8975 u64 mask = attr->branch_sample_type; 8976 8977 /* only using defined bits */ 8978 if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1)) 8979 return -EINVAL; 8980 8981 /* at least one branch bit must be set */ 8982 if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL)) 8983 return -EINVAL; 8984 8985 /* propagate priv level, when not set for branch */ 8986 if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) { 8987 8988 /* exclude_kernel checked on syscall entry */ 8989 if (!attr->exclude_kernel) 8990 mask |= PERF_SAMPLE_BRANCH_KERNEL; 8991 8992 if (!attr->exclude_user) 8993 mask |= PERF_SAMPLE_BRANCH_USER; 8994 8995 if (!attr->exclude_hv) 8996 mask |= PERF_SAMPLE_BRANCH_HV; 8997 /* 8998 * adjust user setting (for HW filter setup) 8999 */ 9000 attr->branch_sample_type = mask; 9001 } 9002 /* privileged levels capture (kernel, hv): check permissions */ 9003 if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM) 9004 && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 9005 return -EACCES; 9006 } 9007 9008 if (attr->sample_type & PERF_SAMPLE_REGS_USER) { 9009 ret = perf_reg_validate(attr->sample_regs_user); 9010 if (ret) 9011 return ret; 9012 } 9013 9014 if (attr->sample_type & PERF_SAMPLE_STACK_USER) { 9015 if (!arch_perf_have_user_stack_dump()) 9016 return -ENOSYS; 9017 9018 /* 9019 * We have __u32 type for the size, but so far 9020 * we can only use __u16 as maximum due to the 9021 * __u16 sample size limit. 9022 */ 9023 if (attr->sample_stack_user >= USHRT_MAX) 9024 ret = -EINVAL; 9025 else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64))) 9026 ret = -EINVAL; 9027 } 9028 9029 if (attr->sample_type & PERF_SAMPLE_REGS_INTR) 9030 ret = perf_reg_validate(attr->sample_regs_intr); 9031 out: 9032 return ret; 9033 9034 err_size: 9035 put_user(sizeof(*attr), &uattr->size); 9036 ret = -E2BIG; 9037 goto out; 9038 } 9039 9040 static int 9041 perf_event_set_output(struct perf_event *event, struct perf_event *output_event) 9042 { 9043 struct ring_buffer *rb = NULL; 9044 int ret = -EINVAL; 9045 9046 if (!output_event) 9047 goto set; 9048 9049 /* don't allow circular references */ 9050 if (event == output_event) 9051 goto out; 9052 9053 /* 9054 * Don't allow cross-cpu buffers 9055 */ 9056 if (output_event->cpu != event->cpu) 9057 goto out; 9058 9059 /* 9060 * If its not a per-cpu rb, it must be the same task. 9061 */ 9062 if (output_event->cpu == -1 && output_event->ctx != event->ctx) 9063 goto out; 9064 9065 /* 9066 * Mixing clocks in the same buffer is trouble you don't need. 9067 */ 9068 if (output_event->clock != event->clock) 9069 goto out; 9070 9071 /* 9072 * Either writing ring buffer from beginning or from end. 9073 * Mixing is not allowed. 9074 */ 9075 if (is_write_backward(output_event) != is_write_backward(event)) 9076 goto out; 9077 9078 /* 9079 * If both events generate aux data, they must be on the same PMU 9080 */ 9081 if (has_aux(event) && has_aux(output_event) && 9082 event->pmu != output_event->pmu) 9083 goto out; 9084 9085 set: 9086 mutex_lock(&event->mmap_mutex); 9087 /* Can't redirect output if we've got an active mmap() */ 9088 if (atomic_read(&event->mmap_count)) 9089 goto unlock; 9090 9091 if (output_event) { 9092 /* get the rb we want to redirect to */ 9093 rb = ring_buffer_get(output_event); 9094 if (!rb) 9095 goto unlock; 9096 } 9097 9098 ring_buffer_attach(event, rb); 9099 9100 ret = 0; 9101 unlock: 9102 mutex_unlock(&event->mmap_mutex); 9103 9104 out: 9105 return ret; 9106 } 9107 9108 static void mutex_lock_double(struct mutex *a, struct mutex *b) 9109 { 9110 if (b < a) 9111 swap(a, b); 9112 9113 mutex_lock(a); 9114 mutex_lock_nested(b, SINGLE_DEPTH_NESTING); 9115 } 9116 9117 static int perf_event_set_clock(struct perf_event *event, clockid_t clk_id) 9118 { 9119 bool nmi_safe = false; 9120 9121 switch (clk_id) { 9122 case CLOCK_MONOTONIC: 9123 event->clock = &ktime_get_mono_fast_ns; 9124 nmi_safe = true; 9125 break; 9126 9127 case CLOCK_MONOTONIC_RAW: 9128 event->clock = &ktime_get_raw_fast_ns; 9129 nmi_safe = true; 9130 break; 9131 9132 case CLOCK_REALTIME: 9133 event->clock = &ktime_get_real_ns; 9134 break; 9135 9136 case CLOCK_BOOTTIME: 9137 event->clock = &ktime_get_boot_ns; 9138 break; 9139 9140 case CLOCK_TAI: 9141 event->clock = &ktime_get_tai_ns; 9142 break; 9143 9144 default: 9145 return -EINVAL; 9146 } 9147 9148 if (!nmi_safe && !(event->pmu->capabilities & PERF_PMU_CAP_NO_NMI)) 9149 return -EINVAL; 9150 9151 return 0; 9152 } 9153 9154 /** 9155 * sys_perf_event_open - open a performance event, associate it to a task/cpu 9156 * 9157 * @attr_uptr: event_id type attributes for monitoring/sampling 9158 * @pid: target pid 9159 * @cpu: target cpu 9160 * @group_fd: group leader event fd 9161 */ 9162 SYSCALL_DEFINE5(perf_event_open, 9163 struct perf_event_attr __user *, attr_uptr, 9164 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags) 9165 { 9166 struct perf_event *group_leader = NULL, *output_event = NULL; 9167 struct perf_event *event, *sibling; 9168 struct perf_event_attr attr; 9169 struct perf_event_context *ctx, *uninitialized_var(gctx); 9170 struct file *event_file = NULL; 9171 struct fd group = {NULL, 0}; 9172 struct task_struct *task = NULL; 9173 struct pmu *pmu; 9174 int event_fd; 9175 int move_group = 0; 9176 int err; 9177 int f_flags = O_RDWR; 9178 int cgroup_fd = -1; 9179 9180 /* for future expandability... */ 9181 if (flags & ~PERF_FLAG_ALL) 9182 return -EINVAL; 9183 9184 err = perf_copy_attr(attr_uptr, &attr); 9185 if (err) 9186 return err; 9187 9188 if (!attr.exclude_kernel) { 9189 if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN)) 9190 return -EACCES; 9191 } 9192 9193 if (attr.freq) { 9194 if (attr.sample_freq > sysctl_perf_event_sample_rate) 9195 return -EINVAL; 9196 } else { 9197 if (attr.sample_period & (1ULL << 63)) 9198 return -EINVAL; 9199 } 9200 9201 /* 9202 * In cgroup mode, the pid argument is used to pass the fd 9203 * opened to the cgroup directory in cgroupfs. The cpu argument 9204 * designates the cpu on which to monitor threads from that 9205 * cgroup. 9206 */ 9207 if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1)) 9208 return -EINVAL; 9209 9210 if (flags & PERF_FLAG_FD_CLOEXEC) 9211 f_flags |= O_CLOEXEC; 9212 9213 event_fd = get_unused_fd_flags(f_flags); 9214 if (event_fd < 0) 9215 return event_fd; 9216 9217 if (group_fd != -1) { 9218 err = perf_fget_light(group_fd, &group); 9219 if (err) 9220 goto err_fd; 9221 group_leader = group.file->private_data; 9222 if (flags & PERF_FLAG_FD_OUTPUT) 9223 output_event = group_leader; 9224 if (flags & PERF_FLAG_FD_NO_GROUP) 9225 group_leader = NULL; 9226 } 9227 9228 if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) { 9229 task = find_lively_task_by_vpid(pid); 9230 if (IS_ERR(task)) { 9231 err = PTR_ERR(task); 9232 goto err_group_fd; 9233 } 9234 } 9235 9236 if (task && group_leader && 9237 group_leader->attr.inherit != attr.inherit) { 9238 err = -EINVAL; 9239 goto err_task; 9240 } 9241 9242 get_online_cpus(); 9243 9244 if (task) { 9245 err = mutex_lock_interruptible(&task->signal->cred_guard_mutex); 9246 if (err) 9247 goto err_cpus; 9248 9249 /* 9250 * Reuse ptrace permission checks for now. 9251 * 9252 * We must hold cred_guard_mutex across this and any potential 9253 * perf_install_in_context() call for this new event to 9254 * serialize against exec() altering our credentials (and the 9255 * perf_event_exit_task() that could imply). 9256 */ 9257 err = -EACCES; 9258 if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) 9259 goto err_cred; 9260 } 9261 9262 if (flags & PERF_FLAG_PID_CGROUP) 9263 cgroup_fd = pid; 9264 9265 event = perf_event_alloc(&attr, cpu, task, group_leader, NULL, 9266 NULL, NULL, cgroup_fd); 9267 if (IS_ERR(event)) { 9268 err = PTR_ERR(event); 9269 goto err_cred; 9270 } 9271 9272 if (is_sampling_event(event)) { 9273 if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) { 9274 err = -ENOTSUPP; 9275 goto err_alloc; 9276 } 9277 } 9278 9279 /* 9280 * Special case software events and allow them to be part of 9281 * any hardware group. 9282 */ 9283 pmu = event->pmu; 9284 9285 if (attr.use_clockid) { 9286 err = perf_event_set_clock(event, attr.clockid); 9287 if (err) 9288 goto err_alloc; 9289 } 9290 9291 if (group_leader && 9292 (is_software_event(event) != is_software_event(group_leader))) { 9293 if (is_software_event(event)) { 9294 /* 9295 * If event and group_leader are not both a software 9296 * event, and event is, then group leader is not. 9297 * 9298 * Allow the addition of software events to !software 9299 * groups, this is safe because software events never 9300 * fail to schedule. 9301 */ 9302 pmu = group_leader->pmu; 9303 } else if (is_software_event(group_leader) && 9304 (group_leader->group_flags & PERF_GROUP_SOFTWARE)) { 9305 /* 9306 * In case the group is a pure software group, and we 9307 * try to add a hardware event, move the whole group to 9308 * the hardware context. 9309 */ 9310 move_group = 1; 9311 } 9312 } 9313 9314 /* 9315 * Get the target context (task or percpu): 9316 */ 9317 ctx = find_get_context(pmu, task, event); 9318 if (IS_ERR(ctx)) { 9319 err = PTR_ERR(ctx); 9320 goto err_alloc; 9321 } 9322 9323 if ((pmu->capabilities & PERF_PMU_CAP_EXCLUSIVE) && group_leader) { 9324 err = -EBUSY; 9325 goto err_context; 9326 } 9327 9328 /* 9329 * Look up the group leader (we will attach this event to it): 9330 */ 9331 if (group_leader) { 9332 err = -EINVAL; 9333 9334 /* 9335 * Do not allow a recursive hierarchy (this new sibling 9336 * becoming part of another group-sibling): 9337 */ 9338 if (group_leader->group_leader != group_leader) 9339 goto err_context; 9340 9341 /* All events in a group should have the same clock */ 9342 if (group_leader->clock != event->clock) 9343 goto err_context; 9344 9345 /* 9346 * Do not allow to attach to a group in a different 9347 * task or CPU context: 9348 */ 9349 if (move_group) { 9350 /* 9351 * Make sure we're both on the same task, or both 9352 * per-cpu events. 9353 */ 9354 if (group_leader->ctx->task != ctx->task) 9355 goto err_context; 9356 9357 /* 9358 * Make sure we're both events for the same CPU; 9359 * grouping events for different CPUs is broken; since 9360 * you can never concurrently schedule them anyhow. 9361 */ 9362 if (group_leader->cpu != event->cpu) 9363 goto err_context; 9364 } else { 9365 if (group_leader->ctx != ctx) 9366 goto err_context; 9367 } 9368 9369 /* 9370 * Only a group leader can be exclusive or pinned 9371 */ 9372 if (attr.exclusive || attr.pinned) 9373 goto err_context; 9374 } 9375 9376 if (output_event) { 9377 err = perf_event_set_output(event, output_event); 9378 if (err) 9379 goto err_context; 9380 } 9381 9382 event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, 9383 f_flags); 9384 if (IS_ERR(event_file)) { 9385 err = PTR_ERR(event_file); 9386 event_file = NULL; 9387 goto err_context; 9388 } 9389 9390 if (move_group) { 9391 gctx = group_leader->ctx; 9392 mutex_lock_double(&gctx->mutex, &ctx->mutex); 9393 if (gctx->task == TASK_TOMBSTONE) { 9394 err = -ESRCH; 9395 goto err_locked; 9396 } 9397 } else { 9398 mutex_lock(&ctx->mutex); 9399 } 9400 9401 if (ctx->task == TASK_TOMBSTONE) { 9402 err = -ESRCH; 9403 goto err_locked; 9404 } 9405 9406 if (!perf_event_validate_size(event)) { 9407 err = -E2BIG; 9408 goto err_locked; 9409 } 9410 9411 /* 9412 * Must be under the same ctx::mutex as perf_install_in_context(), 9413 * because we need to serialize with concurrent event creation. 9414 */ 9415 if (!exclusive_event_installable(event, ctx)) { 9416 /* exclusive and group stuff are assumed mutually exclusive */ 9417 WARN_ON_ONCE(move_group); 9418 9419 err = -EBUSY; 9420 goto err_locked; 9421 } 9422 9423 WARN_ON_ONCE(ctx->parent_ctx); 9424 9425 /* 9426 * This is the point on no return; we cannot fail hereafter. This is 9427 * where we start modifying current state. 9428 */ 9429 9430 if (move_group) { 9431 /* 9432 * See perf_event_ctx_lock() for comments on the details 9433 * of swizzling perf_event::ctx. 9434 */ 9435 perf_remove_from_context(group_leader, 0); 9436 9437 list_for_each_entry(sibling, &group_leader->sibling_list, 9438 group_entry) { 9439 perf_remove_from_context(sibling, 0); 9440 put_ctx(gctx); 9441 } 9442 9443 /* 9444 * Wait for everybody to stop referencing the events through 9445 * the old lists, before installing it on new lists. 9446 */ 9447 synchronize_rcu(); 9448 9449 /* 9450 * Install the group siblings before the group leader. 9451 * 9452 * Because a group leader will try and install the entire group 9453 * (through the sibling list, which is still in-tact), we can 9454 * end up with siblings installed in the wrong context. 9455 * 9456 * By installing siblings first we NO-OP because they're not 9457 * reachable through the group lists. 9458 */ 9459 list_for_each_entry(sibling, &group_leader->sibling_list, 9460 group_entry) { 9461 perf_event__state_init(sibling); 9462 perf_install_in_context(ctx, sibling, sibling->cpu); 9463 get_ctx(ctx); 9464 } 9465 9466 /* 9467 * Removing from the context ends up with disabled 9468 * event. What we want here is event in the initial 9469 * startup state, ready to be add into new context. 9470 */ 9471 perf_event__state_init(group_leader); 9472 perf_install_in_context(ctx, group_leader, group_leader->cpu); 9473 get_ctx(ctx); 9474 9475 /* 9476 * Now that all events are installed in @ctx, nothing 9477 * references @gctx anymore, so drop the last reference we have 9478 * on it. 9479 */ 9480 put_ctx(gctx); 9481 } 9482 9483 /* 9484 * Precalculate sample_data sizes; do while holding ctx::mutex such 9485 * that we're serialized against further additions and before 9486 * perf_install_in_context() which is the point the event is active and 9487 * can use these values. 9488 */ 9489 perf_event__header_size(event); 9490 perf_event__id_header_size(event); 9491 9492 event->owner = current; 9493 9494 perf_install_in_context(ctx, event, event->cpu); 9495 perf_unpin_context(ctx); 9496 9497 if (move_group) 9498 mutex_unlock(&gctx->mutex); 9499 mutex_unlock(&ctx->mutex); 9500 9501 if (task) { 9502 mutex_unlock(&task->signal->cred_guard_mutex); 9503 put_task_struct(task); 9504 } 9505 9506 put_online_cpus(); 9507 9508 mutex_lock(¤t->perf_event_mutex); 9509 list_add_tail(&event->owner_entry, ¤t->perf_event_list); 9510 mutex_unlock(¤t->perf_event_mutex); 9511 9512 /* 9513 * Drop the reference on the group_event after placing the 9514 * new event on the sibling_list. This ensures destruction 9515 * of the group leader will find the pointer to itself in 9516 * perf_group_detach(). 9517 */ 9518 fdput(group); 9519 fd_install(event_fd, event_file); 9520 return event_fd; 9521 9522 err_locked: 9523 if (move_group) 9524 mutex_unlock(&gctx->mutex); 9525 mutex_unlock(&ctx->mutex); 9526 /* err_file: */ 9527 fput(event_file); 9528 err_context: 9529 perf_unpin_context(ctx); 9530 put_ctx(ctx); 9531 err_alloc: 9532 /* 9533 * If event_file is set, the fput() above will have called ->release() 9534 * and that will take care of freeing the event. 9535 */ 9536 if (!event_file) 9537 free_event(event); 9538 err_cred: 9539 if (task) 9540 mutex_unlock(&task->signal->cred_guard_mutex); 9541 err_cpus: 9542 put_online_cpus(); 9543 err_task: 9544 if (task) 9545 put_task_struct(task); 9546 err_group_fd: 9547 fdput(group); 9548 err_fd: 9549 put_unused_fd(event_fd); 9550 return err; 9551 } 9552 9553 /** 9554 * perf_event_create_kernel_counter 9555 * 9556 * @attr: attributes of the counter to create 9557 * @cpu: cpu in which the counter is bound 9558 * @task: task to profile (NULL for percpu) 9559 */ 9560 struct perf_event * 9561 perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu, 9562 struct task_struct *task, 9563 perf_overflow_handler_t overflow_handler, 9564 void *context) 9565 { 9566 struct perf_event_context *ctx; 9567 struct perf_event *event; 9568 int err; 9569 9570 /* 9571 * Get the target context (task or percpu): 9572 */ 9573 9574 event = perf_event_alloc(attr, cpu, task, NULL, NULL, 9575 overflow_handler, context, -1); 9576 if (IS_ERR(event)) { 9577 err = PTR_ERR(event); 9578 goto err; 9579 } 9580 9581 /* Mark owner so we could distinguish it from user events. */ 9582 event->owner = TASK_TOMBSTONE; 9583 9584 ctx = find_get_context(event->pmu, task, event); 9585 if (IS_ERR(ctx)) { 9586 err = PTR_ERR(ctx); 9587 goto err_free; 9588 } 9589 9590 WARN_ON_ONCE(ctx->parent_ctx); 9591 mutex_lock(&ctx->mutex); 9592 if (ctx->task == TASK_TOMBSTONE) { 9593 err = -ESRCH; 9594 goto err_unlock; 9595 } 9596 9597 if (!exclusive_event_installable(event, ctx)) { 9598 err = -EBUSY; 9599 goto err_unlock; 9600 } 9601 9602 perf_install_in_context(ctx, event, cpu); 9603 perf_unpin_context(ctx); 9604 mutex_unlock(&ctx->mutex); 9605 9606 return event; 9607 9608 err_unlock: 9609 mutex_unlock(&ctx->mutex); 9610 perf_unpin_context(ctx); 9611 put_ctx(ctx); 9612 err_free: 9613 free_event(event); 9614 err: 9615 return ERR_PTR(err); 9616 } 9617 EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter); 9618 9619 void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu) 9620 { 9621 struct perf_event_context *src_ctx; 9622 struct perf_event_context *dst_ctx; 9623 struct perf_event *event, *tmp; 9624 LIST_HEAD(events); 9625 9626 src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx; 9627 dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx; 9628 9629 /* 9630 * See perf_event_ctx_lock() for comments on the details 9631 * of swizzling perf_event::ctx. 9632 */ 9633 mutex_lock_double(&src_ctx->mutex, &dst_ctx->mutex); 9634 list_for_each_entry_safe(event, tmp, &src_ctx->event_list, 9635 event_entry) { 9636 perf_remove_from_context(event, 0); 9637 unaccount_event_cpu(event, src_cpu); 9638 put_ctx(src_ctx); 9639 list_add(&event->migrate_entry, &events); 9640 } 9641 9642 /* 9643 * Wait for the events to quiesce before re-instating them. 9644 */ 9645 synchronize_rcu(); 9646 9647 /* 9648 * Re-instate events in 2 passes. 9649 * 9650 * Skip over group leaders and only install siblings on this first 9651 * pass, siblings will not get enabled without a leader, however a 9652 * leader will enable its siblings, even if those are still on the old 9653 * context. 9654 */ 9655 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 9656 if (event->group_leader == event) 9657 continue; 9658 9659 list_del(&event->migrate_entry); 9660 if (event->state >= PERF_EVENT_STATE_OFF) 9661 event->state = PERF_EVENT_STATE_INACTIVE; 9662 account_event_cpu(event, dst_cpu); 9663 perf_install_in_context(dst_ctx, event, dst_cpu); 9664 get_ctx(dst_ctx); 9665 } 9666 9667 /* 9668 * Once all the siblings are setup properly, install the group leaders 9669 * to make it go. 9670 */ 9671 list_for_each_entry_safe(event, tmp, &events, migrate_entry) { 9672 list_del(&event->migrate_entry); 9673 if (event->state >= PERF_EVENT_STATE_OFF) 9674 event->state = PERF_EVENT_STATE_INACTIVE; 9675 account_event_cpu(event, dst_cpu); 9676 perf_install_in_context(dst_ctx, event, dst_cpu); 9677 get_ctx(dst_ctx); 9678 } 9679 mutex_unlock(&dst_ctx->mutex); 9680 mutex_unlock(&src_ctx->mutex); 9681 } 9682 EXPORT_SYMBOL_GPL(perf_pmu_migrate_context); 9683 9684 static void sync_child_event(struct perf_event *child_event, 9685 struct task_struct *child) 9686 { 9687 struct perf_event *parent_event = child_event->parent; 9688 u64 child_val; 9689 9690 if (child_event->attr.inherit_stat) 9691 perf_event_read_event(child_event, child); 9692 9693 child_val = perf_event_count(child_event); 9694 9695 /* 9696 * Add back the child's count to the parent's count: 9697 */ 9698 atomic64_add(child_val, &parent_event->child_count); 9699 atomic64_add(child_event->total_time_enabled, 9700 &parent_event->child_total_time_enabled); 9701 atomic64_add(child_event->total_time_running, 9702 &parent_event->child_total_time_running); 9703 } 9704 9705 static void 9706 perf_event_exit_event(struct perf_event *child_event, 9707 struct perf_event_context *child_ctx, 9708 struct task_struct *child) 9709 { 9710 struct perf_event *parent_event = child_event->parent; 9711 9712 /* 9713 * Do not destroy the 'original' grouping; because of the context 9714 * switch optimization the original events could've ended up in a 9715 * random child task. 9716 * 9717 * If we were to destroy the original group, all group related 9718 * operations would cease to function properly after this random 9719 * child dies. 9720 * 9721 * Do destroy all inherited groups, we don't care about those 9722 * and being thorough is better. 9723 */ 9724 raw_spin_lock_irq(&child_ctx->lock); 9725 WARN_ON_ONCE(child_ctx->is_active); 9726 9727 if (parent_event) 9728 perf_group_detach(child_event); 9729 list_del_event(child_event, child_ctx); 9730 child_event->state = PERF_EVENT_STATE_EXIT; /* is_event_hup() */ 9731 raw_spin_unlock_irq(&child_ctx->lock); 9732 9733 /* 9734 * Parent events are governed by their filedesc, retain them. 9735 */ 9736 if (!parent_event) { 9737 perf_event_wakeup(child_event); 9738 return; 9739 } 9740 /* 9741 * Child events can be cleaned up. 9742 */ 9743 9744 sync_child_event(child_event, child); 9745 9746 /* 9747 * Remove this event from the parent's list 9748 */ 9749 WARN_ON_ONCE(parent_event->ctx->parent_ctx); 9750 mutex_lock(&parent_event->child_mutex); 9751 list_del_init(&child_event->child_list); 9752 mutex_unlock(&parent_event->child_mutex); 9753 9754 /* 9755 * Kick perf_poll() for is_event_hup(). 9756 */ 9757 perf_event_wakeup(parent_event); 9758 free_event(child_event); 9759 put_event(parent_event); 9760 } 9761 9762 static void perf_event_exit_task_context(struct task_struct *child, int ctxn) 9763 { 9764 struct perf_event_context *child_ctx, *clone_ctx = NULL; 9765 struct perf_event *child_event, *next; 9766 9767 WARN_ON_ONCE(child != current); 9768 9769 child_ctx = perf_pin_task_context(child, ctxn); 9770 if (!child_ctx) 9771 return; 9772 9773 /* 9774 * In order to reduce the amount of tricky in ctx tear-down, we hold 9775 * ctx::mutex over the entire thing. This serializes against almost 9776 * everything that wants to access the ctx. 9777 * 9778 * The exception is sys_perf_event_open() / 9779 * perf_event_create_kernel_count() which does find_get_context() 9780 * without ctx::mutex (it cannot because of the move_group double mutex 9781 * lock thing). See the comments in perf_install_in_context(). 9782 */ 9783 mutex_lock(&child_ctx->mutex); 9784 9785 /* 9786 * In a single ctx::lock section, de-schedule the events and detach the 9787 * context from the task such that we cannot ever get it scheduled back 9788 * in. 9789 */ 9790 raw_spin_lock_irq(&child_ctx->lock); 9791 task_ctx_sched_out(__get_cpu_context(child_ctx), child_ctx); 9792 9793 /* 9794 * Now that the context is inactive, destroy the task <-> ctx relation 9795 * and mark the context dead. 9796 */ 9797 RCU_INIT_POINTER(child->perf_event_ctxp[ctxn], NULL); 9798 put_ctx(child_ctx); /* cannot be last */ 9799 WRITE_ONCE(child_ctx->task, TASK_TOMBSTONE); 9800 put_task_struct(current); /* cannot be last */ 9801 9802 clone_ctx = unclone_ctx(child_ctx); 9803 raw_spin_unlock_irq(&child_ctx->lock); 9804 9805 if (clone_ctx) 9806 put_ctx(clone_ctx); 9807 9808 /* 9809 * Report the task dead after unscheduling the events so that we 9810 * won't get any samples after PERF_RECORD_EXIT. We can however still 9811 * get a few PERF_RECORD_READ events. 9812 */ 9813 perf_event_task(child, child_ctx, 0); 9814 9815 list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry) 9816 perf_event_exit_event(child_event, child_ctx, child); 9817 9818 mutex_unlock(&child_ctx->mutex); 9819 9820 put_ctx(child_ctx); 9821 } 9822 9823 /* 9824 * When a child task exits, feed back event values to parent events. 9825 * 9826 * Can be called with cred_guard_mutex held when called from 9827 * install_exec_creds(). 9828 */ 9829 void perf_event_exit_task(struct task_struct *child) 9830 { 9831 struct perf_event *event, *tmp; 9832 int ctxn; 9833 9834 mutex_lock(&child->perf_event_mutex); 9835 list_for_each_entry_safe(event, tmp, &child->perf_event_list, 9836 owner_entry) { 9837 list_del_init(&event->owner_entry); 9838 9839 /* 9840 * Ensure the list deletion is visible before we clear 9841 * the owner, closes a race against perf_release() where 9842 * we need to serialize on the owner->perf_event_mutex. 9843 */ 9844 smp_store_release(&event->owner, NULL); 9845 } 9846 mutex_unlock(&child->perf_event_mutex); 9847 9848 for_each_task_context_nr(ctxn) 9849 perf_event_exit_task_context(child, ctxn); 9850 9851 /* 9852 * The perf_event_exit_task_context calls perf_event_task 9853 * with child's task_ctx, which generates EXIT events for 9854 * child contexts and sets child->perf_event_ctxp[] to NULL. 9855 * At this point we need to send EXIT events to cpu contexts. 9856 */ 9857 perf_event_task(child, NULL, 0); 9858 } 9859 9860 static void perf_free_event(struct perf_event *event, 9861 struct perf_event_context *ctx) 9862 { 9863 struct perf_event *parent = event->parent; 9864 9865 if (WARN_ON_ONCE(!parent)) 9866 return; 9867 9868 mutex_lock(&parent->child_mutex); 9869 list_del_init(&event->child_list); 9870 mutex_unlock(&parent->child_mutex); 9871 9872 put_event(parent); 9873 9874 raw_spin_lock_irq(&ctx->lock); 9875 perf_group_detach(event); 9876 list_del_event(event, ctx); 9877 raw_spin_unlock_irq(&ctx->lock); 9878 free_event(event); 9879 } 9880 9881 /* 9882 * Free an unexposed, unused context as created by inheritance by 9883 * perf_event_init_task below, used by fork() in case of fail. 9884 * 9885 * Not all locks are strictly required, but take them anyway to be nice and 9886 * help out with the lockdep assertions. 9887 */ 9888 void perf_event_free_task(struct task_struct *task) 9889 { 9890 struct perf_event_context *ctx; 9891 struct perf_event *event, *tmp; 9892 int ctxn; 9893 9894 for_each_task_context_nr(ctxn) { 9895 ctx = task->perf_event_ctxp[ctxn]; 9896 if (!ctx) 9897 continue; 9898 9899 mutex_lock(&ctx->mutex); 9900 again: 9901 list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, 9902 group_entry) 9903 perf_free_event(event, ctx); 9904 9905 list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, 9906 group_entry) 9907 perf_free_event(event, ctx); 9908 9909 if (!list_empty(&ctx->pinned_groups) || 9910 !list_empty(&ctx->flexible_groups)) 9911 goto again; 9912 9913 mutex_unlock(&ctx->mutex); 9914 9915 put_ctx(ctx); 9916 } 9917 } 9918 9919 void perf_event_delayed_put(struct task_struct *task) 9920 { 9921 int ctxn; 9922 9923 for_each_task_context_nr(ctxn) 9924 WARN_ON_ONCE(task->perf_event_ctxp[ctxn]); 9925 } 9926 9927 struct file *perf_event_get(unsigned int fd) 9928 { 9929 struct file *file; 9930 9931 file = fget_raw(fd); 9932 if (!file) 9933 return ERR_PTR(-EBADF); 9934 9935 if (file->f_op != &perf_fops) { 9936 fput(file); 9937 return ERR_PTR(-EBADF); 9938 } 9939 9940 return file; 9941 } 9942 9943 const struct perf_event_attr *perf_event_attrs(struct perf_event *event) 9944 { 9945 if (!event) 9946 return ERR_PTR(-EINVAL); 9947 9948 return &event->attr; 9949 } 9950 9951 /* 9952 * inherit a event from parent task to child task: 9953 */ 9954 static struct perf_event * 9955 inherit_event(struct perf_event *parent_event, 9956 struct task_struct *parent, 9957 struct perf_event_context *parent_ctx, 9958 struct task_struct *child, 9959 struct perf_event *group_leader, 9960 struct perf_event_context *child_ctx) 9961 { 9962 enum perf_event_active_state parent_state = parent_event->state; 9963 struct perf_event *child_event; 9964 unsigned long flags; 9965 9966 /* 9967 * Instead of creating recursive hierarchies of events, 9968 * we link inherited events back to the original parent, 9969 * which has a filp for sure, which we use as the reference 9970 * count: 9971 */ 9972 if (parent_event->parent) 9973 parent_event = parent_event->parent; 9974 9975 child_event = perf_event_alloc(&parent_event->attr, 9976 parent_event->cpu, 9977 child, 9978 group_leader, parent_event, 9979 NULL, NULL, -1); 9980 if (IS_ERR(child_event)) 9981 return child_event; 9982 9983 /* 9984 * is_orphaned_event() and list_add_tail(&parent_event->child_list) 9985 * must be under the same lock in order to serialize against 9986 * perf_event_release_kernel(), such that either we must observe 9987 * is_orphaned_event() or they will observe us on the child_list. 9988 */ 9989 mutex_lock(&parent_event->child_mutex); 9990 if (is_orphaned_event(parent_event) || 9991 !atomic_long_inc_not_zero(&parent_event->refcount)) { 9992 mutex_unlock(&parent_event->child_mutex); 9993 free_event(child_event); 9994 return NULL; 9995 } 9996 9997 get_ctx(child_ctx); 9998 9999 /* 10000 * Make the child state follow the state of the parent event, 10001 * not its attr.disabled bit. We hold the parent's mutex, 10002 * so we won't race with perf_event_{en, dis}able_family. 10003 */ 10004 if (parent_state >= PERF_EVENT_STATE_INACTIVE) 10005 child_event->state = PERF_EVENT_STATE_INACTIVE; 10006 else 10007 child_event->state = PERF_EVENT_STATE_OFF; 10008 10009 if (parent_event->attr.freq) { 10010 u64 sample_period = parent_event->hw.sample_period; 10011 struct hw_perf_event *hwc = &child_event->hw; 10012 10013 hwc->sample_period = sample_period; 10014 hwc->last_period = sample_period; 10015 10016 local64_set(&hwc->period_left, sample_period); 10017 } 10018 10019 child_event->ctx = child_ctx; 10020 child_event->overflow_handler = parent_event->overflow_handler; 10021 child_event->overflow_handler_context 10022 = parent_event->overflow_handler_context; 10023 10024 /* 10025 * Precalculate sample_data sizes 10026 */ 10027 perf_event__header_size(child_event); 10028 perf_event__id_header_size(child_event); 10029 10030 /* 10031 * Link it up in the child's context: 10032 */ 10033 raw_spin_lock_irqsave(&child_ctx->lock, flags); 10034 add_event_to_ctx(child_event, child_ctx); 10035 raw_spin_unlock_irqrestore(&child_ctx->lock, flags); 10036 10037 /* 10038 * Link this into the parent event's child list 10039 */ 10040 list_add_tail(&child_event->child_list, &parent_event->child_list); 10041 mutex_unlock(&parent_event->child_mutex); 10042 10043 return child_event; 10044 } 10045 10046 static int inherit_group(struct perf_event *parent_event, 10047 struct task_struct *parent, 10048 struct perf_event_context *parent_ctx, 10049 struct task_struct *child, 10050 struct perf_event_context *child_ctx) 10051 { 10052 struct perf_event *leader; 10053 struct perf_event *sub; 10054 struct perf_event *child_ctr; 10055 10056 leader = inherit_event(parent_event, parent, parent_ctx, 10057 child, NULL, child_ctx); 10058 if (IS_ERR(leader)) 10059 return PTR_ERR(leader); 10060 list_for_each_entry(sub, &parent_event->sibling_list, group_entry) { 10061 child_ctr = inherit_event(sub, parent, parent_ctx, 10062 child, leader, child_ctx); 10063 if (IS_ERR(child_ctr)) 10064 return PTR_ERR(child_ctr); 10065 } 10066 return 0; 10067 } 10068 10069 static int 10070 inherit_task_group(struct perf_event *event, struct task_struct *parent, 10071 struct perf_event_context *parent_ctx, 10072 struct task_struct *child, int ctxn, 10073 int *inherited_all) 10074 { 10075 int ret; 10076 struct perf_event_context *child_ctx; 10077 10078 if (!event->attr.inherit) { 10079 *inherited_all = 0; 10080 return 0; 10081 } 10082 10083 child_ctx = child->perf_event_ctxp[ctxn]; 10084 if (!child_ctx) { 10085 /* 10086 * This is executed from the parent task context, so 10087 * inherit events that have been marked for cloning. 10088 * First allocate and initialize a context for the 10089 * child. 10090 */ 10091 10092 child_ctx = alloc_perf_context(parent_ctx->pmu, child); 10093 if (!child_ctx) 10094 return -ENOMEM; 10095 10096 child->perf_event_ctxp[ctxn] = child_ctx; 10097 } 10098 10099 ret = inherit_group(event, parent, parent_ctx, 10100 child, child_ctx); 10101 10102 if (ret) 10103 *inherited_all = 0; 10104 10105 return ret; 10106 } 10107 10108 /* 10109 * Initialize the perf_event context in task_struct 10110 */ 10111 static int perf_event_init_context(struct task_struct *child, int ctxn) 10112 { 10113 struct perf_event_context *child_ctx, *parent_ctx; 10114 struct perf_event_context *cloned_ctx; 10115 struct perf_event *event; 10116 struct task_struct *parent = current; 10117 int inherited_all = 1; 10118 unsigned long flags; 10119 int ret = 0; 10120 10121 if (likely(!parent->perf_event_ctxp[ctxn])) 10122 return 0; 10123 10124 /* 10125 * If the parent's context is a clone, pin it so it won't get 10126 * swapped under us. 10127 */ 10128 parent_ctx = perf_pin_task_context(parent, ctxn); 10129 if (!parent_ctx) 10130 return 0; 10131 10132 /* 10133 * No need to check if parent_ctx != NULL here; since we saw 10134 * it non-NULL earlier, the only reason for it to become NULL 10135 * is if we exit, and since we're currently in the middle of 10136 * a fork we can't be exiting at the same time. 10137 */ 10138 10139 /* 10140 * Lock the parent list. No need to lock the child - not PID 10141 * hashed yet and not running, so nobody can access it. 10142 */ 10143 mutex_lock(&parent_ctx->mutex); 10144 10145 /* 10146 * We dont have to disable NMIs - we are only looking at 10147 * the list, not manipulating it: 10148 */ 10149 list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) { 10150 ret = inherit_task_group(event, parent, parent_ctx, 10151 child, ctxn, &inherited_all); 10152 if (ret) 10153 break; 10154 } 10155 10156 /* 10157 * We can't hold ctx->lock when iterating the ->flexible_group list due 10158 * to allocations, but we need to prevent rotation because 10159 * rotate_ctx() will change the list from interrupt context. 10160 */ 10161 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 10162 parent_ctx->rotate_disable = 1; 10163 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 10164 10165 list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) { 10166 ret = inherit_task_group(event, parent, parent_ctx, 10167 child, ctxn, &inherited_all); 10168 if (ret) 10169 break; 10170 } 10171 10172 raw_spin_lock_irqsave(&parent_ctx->lock, flags); 10173 parent_ctx->rotate_disable = 0; 10174 10175 child_ctx = child->perf_event_ctxp[ctxn]; 10176 10177 if (child_ctx && inherited_all) { 10178 /* 10179 * Mark the child context as a clone of the parent 10180 * context, or of whatever the parent is a clone of. 10181 * 10182 * Note that if the parent is a clone, the holding of 10183 * parent_ctx->lock avoids it from being uncloned. 10184 */ 10185 cloned_ctx = parent_ctx->parent_ctx; 10186 if (cloned_ctx) { 10187 child_ctx->parent_ctx = cloned_ctx; 10188 child_ctx->parent_gen = parent_ctx->parent_gen; 10189 } else { 10190 child_ctx->parent_ctx = parent_ctx; 10191 child_ctx->parent_gen = parent_ctx->generation; 10192 } 10193 get_ctx(child_ctx->parent_ctx); 10194 } 10195 10196 raw_spin_unlock_irqrestore(&parent_ctx->lock, flags); 10197 mutex_unlock(&parent_ctx->mutex); 10198 10199 perf_unpin_context(parent_ctx); 10200 put_ctx(parent_ctx); 10201 10202 return ret; 10203 } 10204 10205 /* 10206 * Initialize the perf_event context in task_struct 10207 */ 10208 int perf_event_init_task(struct task_struct *child) 10209 { 10210 int ctxn, ret; 10211 10212 memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp)); 10213 mutex_init(&child->perf_event_mutex); 10214 INIT_LIST_HEAD(&child->perf_event_list); 10215 10216 for_each_task_context_nr(ctxn) { 10217 ret = perf_event_init_context(child, ctxn); 10218 if (ret) { 10219 perf_event_free_task(child); 10220 return ret; 10221 } 10222 } 10223 10224 return 0; 10225 } 10226 10227 static void __init perf_event_init_all_cpus(void) 10228 { 10229 struct swevent_htable *swhash; 10230 int cpu; 10231 10232 for_each_possible_cpu(cpu) { 10233 swhash = &per_cpu(swevent_htable, cpu); 10234 mutex_init(&swhash->hlist_mutex); 10235 INIT_LIST_HEAD(&per_cpu(active_ctx_list, cpu)); 10236 } 10237 } 10238 10239 static void perf_event_init_cpu(int cpu) 10240 { 10241 struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu); 10242 10243 mutex_lock(&swhash->hlist_mutex); 10244 if (swhash->hlist_refcount > 0 && !swevent_hlist_deref(swhash)) { 10245 struct swevent_hlist *hlist; 10246 10247 hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu)); 10248 WARN_ON(!hlist); 10249 rcu_assign_pointer(swhash->swevent_hlist, hlist); 10250 } 10251 mutex_unlock(&swhash->hlist_mutex); 10252 } 10253 10254 #if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC_CORE 10255 static void __perf_event_exit_context(void *__info) 10256 { 10257 struct perf_event_context *ctx = __info; 10258 struct perf_cpu_context *cpuctx = __get_cpu_context(ctx); 10259 struct perf_event *event; 10260 10261 raw_spin_lock(&ctx->lock); 10262 list_for_each_entry(event, &ctx->event_list, event_entry) 10263 __perf_remove_from_context(event, cpuctx, ctx, (void *)DETACH_GROUP); 10264 raw_spin_unlock(&ctx->lock); 10265 } 10266 10267 static void perf_event_exit_cpu_context(int cpu) 10268 { 10269 struct perf_event_context *ctx; 10270 struct pmu *pmu; 10271 int idx; 10272 10273 idx = srcu_read_lock(&pmus_srcu); 10274 list_for_each_entry_rcu(pmu, &pmus, entry) { 10275 ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx; 10276 10277 mutex_lock(&ctx->mutex); 10278 smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1); 10279 mutex_unlock(&ctx->mutex); 10280 } 10281 srcu_read_unlock(&pmus_srcu, idx); 10282 } 10283 10284 static void perf_event_exit_cpu(int cpu) 10285 { 10286 perf_event_exit_cpu_context(cpu); 10287 } 10288 #else 10289 static inline void perf_event_exit_cpu(int cpu) { } 10290 #endif 10291 10292 static int 10293 perf_reboot(struct notifier_block *notifier, unsigned long val, void *v) 10294 { 10295 int cpu; 10296 10297 for_each_online_cpu(cpu) 10298 perf_event_exit_cpu(cpu); 10299 10300 return NOTIFY_OK; 10301 } 10302 10303 /* 10304 * Run the perf reboot notifier at the very last possible moment so that 10305 * the generic watchdog code runs as long as possible. 10306 */ 10307 static struct notifier_block perf_reboot_notifier = { 10308 .notifier_call = perf_reboot, 10309 .priority = INT_MIN, 10310 }; 10311 10312 static int 10313 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu) 10314 { 10315 unsigned int cpu = (long)hcpu; 10316 10317 switch (action & ~CPU_TASKS_FROZEN) { 10318 10319 case CPU_UP_PREPARE: 10320 /* 10321 * This must be done before the CPU comes alive, because the 10322 * moment we can run tasks we can encounter (software) events. 10323 * 10324 * Specifically, someone can have inherited events on kthreadd 10325 * or a pre-existing worker thread that gets re-bound. 10326 */ 10327 perf_event_init_cpu(cpu); 10328 break; 10329 10330 case CPU_DOWN_PREPARE: 10331 /* 10332 * This must be done before the CPU dies because after that an 10333 * active event might want to IPI the CPU and that'll not work 10334 * so great for dead CPUs. 10335 * 10336 * XXX smp_call_function_single() return -ENXIO without a warn 10337 * so we could possibly deal with this. 10338 * 10339 * This is safe against new events arriving because 10340 * sys_perf_event_open() serializes against hotplug using 10341 * get_online_cpus(). 10342 */ 10343 perf_event_exit_cpu(cpu); 10344 break; 10345 default: 10346 break; 10347 } 10348 10349 return NOTIFY_OK; 10350 } 10351 10352 void __init perf_event_init(void) 10353 { 10354 int ret; 10355 10356 idr_init(&pmu_idr); 10357 10358 perf_event_init_all_cpus(); 10359 init_srcu_struct(&pmus_srcu); 10360 perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE); 10361 perf_pmu_register(&perf_cpu_clock, NULL, -1); 10362 perf_pmu_register(&perf_task_clock, NULL, -1); 10363 perf_tp_register(); 10364 perf_cpu_notifier(perf_cpu_notify); 10365 register_reboot_notifier(&perf_reboot_notifier); 10366 10367 ret = init_hw_breakpoint(); 10368 WARN(ret, "hw_breakpoint initialization failed with: %d", ret); 10369 10370 /* 10371 * Build time assertion that we keep the data_head at the intended 10372 * location. IOW, validation we got the __reserved[] size right. 10373 */ 10374 BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head)) 10375 != 1024); 10376 } 10377 10378 ssize_t perf_event_sysfs_show(struct device *dev, struct device_attribute *attr, 10379 char *page) 10380 { 10381 struct perf_pmu_events_attr *pmu_attr = 10382 container_of(attr, struct perf_pmu_events_attr, attr); 10383 10384 if (pmu_attr->event_str) 10385 return sprintf(page, "%s\n", pmu_attr->event_str); 10386 10387 return 0; 10388 } 10389 EXPORT_SYMBOL_GPL(perf_event_sysfs_show); 10390 10391 static int __init perf_event_sysfs_init(void) 10392 { 10393 struct pmu *pmu; 10394 int ret; 10395 10396 mutex_lock(&pmus_lock); 10397 10398 ret = bus_register(&pmu_bus); 10399 if (ret) 10400 goto unlock; 10401 10402 list_for_each_entry(pmu, &pmus, entry) { 10403 if (!pmu->name || pmu->type < 0) 10404 continue; 10405 10406 ret = pmu_dev_alloc(pmu); 10407 WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret); 10408 } 10409 pmu_bus_running = 1; 10410 ret = 0; 10411 10412 unlock: 10413 mutex_unlock(&pmus_lock); 10414 10415 return ret; 10416 } 10417 device_initcall(perf_event_sysfs_init); 10418 10419 #ifdef CONFIG_CGROUP_PERF 10420 static struct cgroup_subsys_state * 10421 perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) 10422 { 10423 struct perf_cgroup *jc; 10424 10425 jc = kzalloc(sizeof(*jc), GFP_KERNEL); 10426 if (!jc) 10427 return ERR_PTR(-ENOMEM); 10428 10429 jc->info = alloc_percpu(struct perf_cgroup_info); 10430 if (!jc->info) { 10431 kfree(jc); 10432 return ERR_PTR(-ENOMEM); 10433 } 10434 10435 return &jc->css; 10436 } 10437 10438 static void perf_cgroup_css_free(struct cgroup_subsys_state *css) 10439 { 10440 struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css); 10441 10442 free_percpu(jc->info); 10443 kfree(jc); 10444 } 10445 10446 static int __perf_cgroup_move(void *info) 10447 { 10448 struct task_struct *task = info; 10449 rcu_read_lock(); 10450 perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN); 10451 rcu_read_unlock(); 10452 return 0; 10453 } 10454 10455 static void perf_cgroup_attach(struct cgroup_taskset *tset) 10456 { 10457 struct task_struct *task; 10458 struct cgroup_subsys_state *css; 10459 10460 cgroup_taskset_for_each(task, css, tset) 10461 task_function_call(task, __perf_cgroup_move, task); 10462 } 10463 10464 struct cgroup_subsys perf_event_cgrp_subsys = { 10465 .css_alloc = perf_cgroup_css_alloc, 10466 .css_free = perf_cgroup_css_free, 10467 .attach = perf_cgroup_attach, 10468 }; 10469 #endif /* CONFIG_CGROUP_PERF */ 10470