Lines Matching +full:ia32 +full:- +full:3 +full:a
5 * Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
8 * Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra
12 * For licencing details see kernel-base/COPYING
63 * This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined
64 * from just a typename, as opposed to an actual function.
126 struct hw_perf_event *hwc = &event->hw; in x86_perf_event_update()
127 int shift = 64 - x86_pmu.cntval_bits; in x86_perf_event_update()
131 if (unlikely(!hwc->event_base)) in x86_perf_event_update()
138 * exchange a new raw count - then add that new-prev delta in x86_perf_event_update()
141 prev_raw_count = local64_read(&hwc->prev_count); in x86_perf_event_update()
143 new_raw_count = rdpmc(hwc->event_base_rdpmc); in x86_perf_event_update()
144 } while (!local64_try_cmpxchg(&hwc->prev_count, in x86_perf_event_update()
150 * (event-)time and add that to the generic event. in x86_perf_event_update()
152 * Careful, not all hw sign-extends above the physical width in x86_perf_event_update()
155 delta = (new_raw_count << shift) - (prev_raw_count << shift); in x86_perf_event_update()
158 local64_add(delta, &event->count); in x86_perf_event_update()
159 local64_sub(delta, &hwc->period_left); in x86_perf_event_update()
169 struct extra_reg *extra_regs = hybrid(event->pmu, extra_regs); in x86_pmu_extra_regs()
173 reg = &event->hw.extra_reg; in x86_pmu_extra_regs()
178 for (er = extra_regs; er->msr; er++) { in x86_pmu_extra_regs()
179 if (er->event != (config & er->config_mask)) in x86_pmu_extra_regs()
181 if (event->attr.config1 & ~er->valid_mask) in x86_pmu_extra_regs()
182 return -EINVAL; in x86_pmu_extra_regs()
184 if (!er->extra_msr_access) in x86_pmu_extra_regs()
185 return -ENXIO; in x86_pmu_extra_regs()
187 reg->idx = er->idx; in x86_pmu_extra_regs()
188 reg->config = event->attr.config1; in x86_pmu_extra_regs()
189 reg->reg = er->msr; in x86_pmu_extra_regs()
267 u64 val, val_fail = -1, val_new= ~0; in check_hw_exists()
268 int i, reg, reg_fail = -1, ret = 0; in check_hw_exists()
270 int reg_safe = -1; in check_hw_exists()
312 if (reg_safe == -1) { in check_hw_exists()
336 pr_err(FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", in check_hw_exists()
376 struct perf_event_attr *attr = &event->attr; in set_ext_hw_attr()
380 config = attr->config; in set_ext_hw_attr()
384 return -EINVAL; in set_ext_hw_attr()
389 return -EINVAL; in set_ext_hw_attr()
394 return -EINVAL; in set_ext_hw_attr()
397 val = hybrid_var(event->pmu, hw_cache_event_ids)[cache_type][cache_op][cache_result]; in set_ext_hw_attr()
399 return -ENOENT; in set_ext_hw_attr()
401 if (val == -1) in set_ext_hw_attr()
402 return -EINVAL; in set_ext_hw_attr()
404 hwc->config |= val; in set_ext_hw_attr()
405 attr->config1 = hybrid_var(event->pmu, hw_cache_extra_regs)[cache_type][cache_op][cache_result]; in set_ext_hw_attr()
417 err = -EBUSY; in x86_reserve_hardware()
442 * Check if we can create event of a certain type (that no conflicting events
472 return -EBUSY; in x86_add_exclusive()
490 struct perf_event_attr *attr = &event->attr; in x86_setup_perfctr()
491 struct hw_perf_event *hwc = &event->hw; in x86_setup_perfctr()
495 hwc->sample_period = x86_pmu.max_period; in x86_setup_perfctr()
496 hwc->last_period = hwc->sample_period; in x86_setup_perfctr()
497 local64_set(&hwc->period_left, hwc->sample_period); in x86_setup_perfctr()
500 if (attr->type == event->pmu->type) in x86_setup_perfctr()
501 return x86_pmu_extra_regs(event->attr.config, event); in x86_setup_perfctr()
503 if (attr->type == PERF_TYPE_HW_CACHE) in x86_setup_perfctr()
506 if (attr->config >= x86_pmu.max_events) in x86_setup_perfctr()
507 return -EINVAL; in x86_setup_perfctr()
509 attr->config = array_index_nospec((unsigned long)attr->config, x86_pmu.max_events); in x86_setup_perfctr()
514 config = x86_pmu.event_map(attr->config); in x86_setup_perfctr()
517 return -ENOENT; in x86_setup_perfctr()
519 if (config == -1LL) in x86_setup_perfctr()
520 return -EINVAL; in x86_setup_perfctr()
522 hwc->config |= config; in x86_setup_perfctr()
535 u64 m = event->attr.branch_sample_type; in precise_br_compat()
544 if (!event->attr.exclude_user) in precise_br_compat()
547 if (!event->attr.exclude_kernel) in precise_br_compat()
577 if (event->attr.precise_ip) { in x86_pmu_hw_config()
580 if (event->attr.precise_ip > precise) in x86_pmu_hw_config()
581 return -EOPNOTSUPP; in x86_pmu_hw_config()
585 return -EINVAL; in x86_pmu_hw_config()
589 * whatever the user is asking with attr->branch_sample_type in x86_pmu_hw_config()
591 if (event->attr.precise_ip > 1 && x86_pmu.intel_cap.pebs_format < 2) { in x86_pmu_hw_config()
592 u64 *br_type = &event->attr.branch_sample_type; in x86_pmu_hw_config()
596 return -EOPNOTSUPP; in x86_pmu_hw_config()
610 if (!event->attr.exclude_user) in x86_pmu_hw_config()
613 if (!event->attr.exclude_kernel) in x86_pmu_hw_config()
619 event->attach_state |= PERF_ATTACH_TASK_DATA; in x86_pmu_hw_config()
625 event->hw.config = ARCH_PERFMON_EVENTSEL_INT; in x86_pmu_hw_config()
630 if (!event->attr.exclude_user) in x86_pmu_hw_config()
631 event->hw.config |= ARCH_PERFMON_EVENTSEL_USR; in x86_pmu_hw_config()
632 if (!event->attr.exclude_kernel) in x86_pmu_hw_config()
633 event->hw.config |= ARCH_PERFMON_EVENTSEL_OS; in x86_pmu_hw_config()
635 if (event->attr.type == event->pmu->type) in x86_pmu_hw_config()
636 event->hw.config |= x86_pmu_get_event_config(event); in x86_pmu_hw_config()
638 if (is_sampling_event(event) && !event->attr.freq && x86_pmu.limit_period) { in x86_pmu_hw_config()
639 s64 left = event->attr.sample_period; in x86_pmu_hw_config()
641 if (left > event->attr.sample_period) in x86_pmu_hw_config()
642 return -EINVAL; in x86_pmu_hw_config()
646 if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK)) in x86_pmu_hw_config()
647 return -EINVAL; in x86_pmu_hw_config()
652 if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) { in x86_pmu_hw_config()
653 if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS)) in x86_pmu_hw_config()
654 return -EINVAL; in x86_pmu_hw_config()
656 if (!event->attr.precise_ip) in x86_pmu_hw_config()
657 return -EINVAL; in x86_pmu_hw_config()
664 * Setup the hardware configuration for a given attr_type
671 return -ENODEV; in __x86_pmu_event_init()
678 event->destroy = hw_perf_event_destroy; in __x86_pmu_event_init()
680 event->hw.idx = -1; in __x86_pmu_event_init()
681 event->hw.last_cpu = -1; in __x86_pmu_event_init()
682 event->hw.last_tag = ~0ULL; in __x86_pmu_event_init()
683 event->hw.dyn_constraint = ~0ULL; in __x86_pmu_event_init()
686 event->hw.extra_reg.idx = EXTRA_REG_NONE; in __x86_pmu_event_init()
687 event->hw.branch_reg.idx = EXTRA_REG_NONE; in __x86_pmu_event_init()
698 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; in x86_pmu_disable_all()
701 if (!test_bit(idx, cpuc->active_mask)) in x86_pmu_disable_all()
724 * It will not be re-enabled in the NMI handler again, because enabled=0. After
739 if (!cpuc->enabled) in x86_pmu_disable()
742 cpuc->n_added = 0; in x86_pmu_disable()
743 cpuc->enabled = 0; in x86_pmu_disable()
755 struct hw_perf_event *hwc = &cpuc->events[idx]->hw; in x86_pmu_enable_all()
757 if (!test_bit(idx, cpuc->active_mask)) in x86_pmu_enable_all()
767 * For a non-hybrid platforms, the type of X86 pmu is in is_x86_event()
769 * For a hybrid platform, the PERF_PMU_CAP_EXTENDED_HW_TYPE in is_x86_event()
770 * is a unique capability for the X86 PMU. in is_x86_event()
771 * Use them to detect a X86 event. in is_x86_event()
773 if (event->pmu->type == PERF_TYPE_RAW || in is_x86_event()
774 event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_HW_TYPE) in is_x86_event()
788 if (WARN_ON_ONCE(!cpuc->pmu)) in x86_get_pmu()
791 return cpuc->pmu; in x86_get_pmu()
831 sched->max_events = num; in perf_sched_init()
832 sched->max_weight = wmax; in perf_sched_init()
833 sched->max_gp = gpmax; in perf_sched_init()
834 sched->constraints = constraints; in perf_sched_init()
837 if (constraints[idx]->weight == wmin) in perf_sched_init()
841 sched->state.event = idx; /* start with min weight */ in perf_sched_init()
842 sched->state.weight = wmin; in perf_sched_init()
843 sched->state.unassigned = num; in perf_sched_init()
848 if (WARN_ON_ONCE(sched->saved_states >= SCHED_STATES_MAX)) in perf_sched_save_state()
851 sched->saved[sched->saved_states] = sched->state; in perf_sched_save_state()
852 sched->saved_states++; in perf_sched_save_state()
857 if (!sched->saved_states) in perf_sched_restore_state()
860 sched->saved_states--; in perf_sched_restore_state()
861 sched->state = sched->saved[sched->saved_states]; in perf_sched_restore_state()
865 sched->state.used &= ~BIT_ULL(sched->state.counter); in perf_sched_restore_state()
868 sched->state.counter++; in perf_sched_restore_state()
874 * Select a counter for the current event to schedule. Return true on
882 if (!sched->state.unassigned) in __perf_sched_find_counter()
885 if (sched->state.event >= sched->max_events) in __perf_sched_find_counter()
888 c = sched->constraints[sched->state.event]; in __perf_sched_find_counter()
890 if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) { in __perf_sched_find_counter()
892 for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) { in __perf_sched_find_counter()
895 if (sched->state.used & mask) in __perf_sched_find_counter()
898 sched->state.used |= mask; in __perf_sched_find_counter()
904 idx = sched->state.counter; in __perf_sched_find_counter()
905 for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) { in __perf_sched_find_counter()
908 if (c->flags & PERF_X86_EVENT_PAIR) in __perf_sched_find_counter()
911 if (sched->state.used & mask) in __perf_sched_find_counter()
914 if (sched->state.nr_gp++ >= sched->max_gp) in __perf_sched_find_counter()
917 sched->state.used |= mask; in __perf_sched_find_counter()
924 sched->state.counter = idx; in __perf_sched_find_counter()
926 if (c->overlap) in __perf_sched_find_counter()
950 if (!sched->state.unassigned || !--sched->state.unassigned) in perf_sched_next_event()
955 sched->state.event++; in perf_sched_next_event()
956 if (sched->state.event >= sched->max_events) { in perf_sched_next_event()
958 sched->state.event = 0; in perf_sched_next_event()
959 sched->state.weight++; in perf_sched_next_event()
960 if (sched->state.weight > sched->max_weight) in perf_sched_next_event()
963 c = sched->constraints[sched->state.event]; in perf_sched_next_event()
964 } while (c->weight != sched->state.weight); in perf_sched_next_event()
966 sched->state.counter = 0; /* start with first counter */ in perf_sched_next_event()
972 * Assign a counter for each event.
1003 * cpuc->n_events hasn't been updated yet, while for the latter in x86_schedule_events()
1004 * cpuc->n_txn contains the number of events added in the current in x86_schedule_events()
1007 n0 = cpuc->n_events; in x86_schedule_events()
1008 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) in x86_schedule_events()
1009 n0 -= cpuc->n_txn; in x86_schedule_events()
1014 c = cpuc->event_constraint[i]; in x86_schedule_events()
1017 * Previously scheduled events should have a cached constraint, in x86_schedule_events()
1024 * have a dynamic constraint -- for those the constraint can in x86_schedule_events()
1027 if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) { in x86_schedule_events()
1028 c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]); in x86_schedule_events()
1029 cpuc->event_constraint[i] = c; in x86_schedule_events()
1032 wmin = min(wmin, c->weight); in x86_schedule_events()
1033 wmax = max(wmax, c->weight); in x86_schedule_events()
1042 hwc = &cpuc->event_list[i]->hw; in x86_schedule_events()
1043 c = cpuc->event_constraint[i]; in x86_schedule_events()
1046 if (hwc->idx == -1) in x86_schedule_events()
1050 if (!test_bit(hwc->idx, c->idxmsk)) in x86_schedule_events()
1053 mask = BIT_ULL(hwc->idx); in x86_schedule_events()
1064 assign[i] = hwc->idx; in x86_schedule_events()
1069 int gpmax = x86_pmu_max_num_counters(cpuc->pmu); in x86_schedule_events()
1081 if (is_ht_workaround_enabled() && !cpuc->is_fake && in x86_schedule_events()
1082 READ_ONCE(cpuc->excl_cntrs->exclusive_present)) in x86_schedule_events()
1090 gpmax -= cpuc->n_pair; in x86_schedule_events()
1094 unsched = perf_assign_events(cpuc->event_constraint, n, wmin, in x86_schedule_events()
1113 e = cpuc->event_list[i]; in x86_schedule_events()
1120 cpuc->event_constraint[i] = NULL; in x86_schedule_events()
1126 return unsched ? -EINVAL : 0; in x86_schedule_events()
1133 if (cpuc->n_metric == INTEL_TD_METRIC_NUM) in add_nr_metric_event()
1134 return -EINVAL; in add_nr_metric_event()
1135 cpuc->n_metric++; in add_nr_metric_event()
1136 cpuc->n_txn_metric++; in add_nr_metric_event()
1146 cpuc->n_metric--; in del_nr_metric_event()
1152 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap); in collect_event()
1155 return -EINVAL; in collect_event()
1157 if (n >= max_count + cpuc->n_metric) in collect_event()
1158 return -EINVAL; in collect_event()
1160 cpuc->event_list[n] = event; in collect_event()
1161 if (is_counter_pair(&event->hw)) { in collect_event()
1162 cpuc->n_pair++; in collect_event()
1163 cpuc->n_txn_pair++; in collect_event()
1178 max_count = x86_pmu_num_counters(cpuc->pmu) + x86_pmu_num_counters_fixed(cpuc->pmu); in collect_events()
1181 n = cpuc->n_events; in collect_events()
1182 if (!cpuc->n_events) in collect_events()
1183 cpuc->pebs_output = 0; in collect_events()
1185 if (!cpuc->is_fake && leader->attr.precise_ip) { in collect_events()
1187 * For PEBS->PT, if !aux_event, the group leader (PT) went in collect_events()
1191 if (is_pebs_pt(leader) && !leader->aux_event) in collect_events()
1192 return -EINVAL; in collect_events()
1197 if (cpuc->pebs_output && in collect_events()
1198 cpuc->pebs_output != is_pebs_pt(leader) + 1) in collect_events()
1199 return -EINVAL; in collect_events()
1201 cpuc->pebs_output = is_pebs_pt(leader) + 1; in collect_events()
1206 return -EINVAL; in collect_events()
1214 if (!is_x86_event(event) || event->state <= PERF_EVENT_STATE_OFF) in collect_events()
1218 return -EINVAL; in collect_events()
1228 struct hw_perf_event *hwc = &event->hw; in x86_assign_hw_event()
1231 idx = hwc->idx = cpuc->assign[i]; in x86_assign_hw_event()
1232 hwc->last_cpu = smp_processor_id(); in x86_assign_hw_event()
1233 hwc->last_tag = ++cpuc->tags[i]; in x86_assign_hw_event()
1237 switch (hwc->idx) { in x86_assign_hw_event()
1240 hwc->config_base = 0; in x86_assign_hw_event()
1241 hwc->event_base = 0; in x86_assign_hw_event()
1245 /* All the metric events are mapped onto the fixed counter 3. */ in x86_assign_hw_event()
1248 case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1: in x86_assign_hw_event()
1249 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL; in x86_assign_hw_event()
1250 hwc->event_base = x86_pmu_fixed_ctr_addr(idx - INTEL_PMC_IDX_FIXED); in x86_assign_hw_event()
1251 hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) | in x86_assign_hw_event()
1256 hwc->config_base = x86_pmu_config_addr(hwc->idx); in x86_assign_hw_event()
1257 hwc->event_base = x86_pmu_event_addr(hwc->idx); in x86_assign_hw_event()
1258 hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx); in x86_assign_hw_event()
1264 * x86_perf_rdpmc_index - Return PMC counter used for event
1281 return event->hw.event_base_rdpmc; in x86_perf_rdpmc_index()
1288 return hwc->idx == cpuc->assign[i] && in match_prev_assignment()
1289 hwc->last_cpu == smp_processor_id() && in match_prev_assignment()
1290 hwc->last_tag == cpuc->tags[i]; in match_prev_assignment()
1300 int i, added = cpuc->n_added; in x86_pmu_enable()
1305 if (cpuc->enabled) in x86_pmu_enable()
1308 if (cpuc->n_added) { in x86_pmu_enable()
1309 int n_running = cpuc->n_events - cpuc->n_added; in x86_pmu_enable()
1326 event = cpuc->event_list[i]; in x86_pmu_enable()
1327 hwc = &event->hw; in x86_pmu_enable()
1331 * - assigned same counter as last time in x86_pmu_enable()
1332 * - running on same CPU as last time in x86_pmu_enable()
1333 * - no other event has used the counter since in x86_pmu_enable()
1335 if (hwc->idx == -1 || in x86_pmu_enable()
1340 * Ensure we don't accidentally enable a stopped in x86_pmu_enable()
1343 if (hwc->state & PERF_HES_STOPPED) in x86_pmu_enable()
1344 hwc->state |= PERF_HES_ARCH; in x86_pmu_enable()
1352 for (i = 0; i < cpuc->n_events; i++) { in x86_pmu_enable()
1353 event = cpuc->event_list[i]; in x86_pmu_enable()
1354 hwc = &event->hw; in x86_pmu_enable()
1361 if (hwc->state & PERF_HES_ARCH) in x86_pmu_enable()
1365 * if cpuc->enabled = 0, then no wrmsr as in x86_pmu_enable()
1370 cpuc->n_added = 0; in x86_pmu_enable()
1374 cpuc->enabled = 1; in x86_pmu_enable()
1383 * Set the next IRQ period, based on the hwc->period_left value.
1388 struct hw_perf_event *hwc = &event->hw; in x86_perf_event_set_period()
1389 s64 left = local64_read(&hwc->period_left); in x86_perf_event_set_period()
1390 s64 period = hwc->sample_period; in x86_perf_event_set_period()
1391 int ret = 0, idx = hwc->idx; in x86_perf_event_set_period()
1393 if (unlikely(!hwc->event_base)) in x86_perf_event_set_period()
1397 * If we are way outside a reasonable range then just skip forward: in x86_perf_event_set_period()
1399 if (unlikely(left <= -period)) { in x86_perf_event_set_period()
1401 local64_set(&hwc->period_left, left); in x86_perf_event_set_period()
1402 hwc->last_period = period; in x86_perf_event_set_period()
1408 local64_set(&hwc->period_left, left); in x86_perf_event_set_period()
1409 hwc->last_period = period; in x86_perf_event_set_period()
1429 local64_set(&hwc->prev_count, (u64)-left); in x86_perf_event_set_period()
1431 wrmsrq(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask); in x86_perf_event_set_period()
1435 * we currently declare a 48-bit counter width in x86_perf_event_set_period()
1448 __x86_pmu_enable_event(&event->hw, in x86_pmu_enable_event()
1453 * Add a single event to the PMU.
1465 hwc = &event->hw; in x86_pmu_add()
1467 n0 = cpuc->n_events; in x86_pmu_add()
1472 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in x86_pmu_add()
1474 hwc->state |= PERF_HES_ARCH; in x86_pmu_add()
1479 * at commit time (->commit_txn) as a whole. in x86_pmu_add()
1481 * If commit fails, we'll call ->del() on all events in x86_pmu_add()
1482 * for which ->add() was called. in x86_pmu_add()
1484 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) in x86_pmu_add()
1494 memcpy(cpuc->assign, assign, n*sizeof(int)); in x86_pmu_add()
1501 cpuc->n_events = n; in x86_pmu_add()
1502 cpuc->n_added += n - n0; in x86_pmu_add()
1503 cpuc->n_txn += n - n0; in x86_pmu_add()
1519 int idx = event->hw.idx; in x86_pmu_start()
1521 if (WARN_ON_ONCE(!(event->hw.state & PERF_HES_STOPPED))) in x86_pmu_start()
1524 if (WARN_ON_ONCE(idx == -1)) in x86_pmu_start()
1528 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE)); in x86_pmu_start()
1532 event->hw.state = 0; in x86_pmu_start()
1534 cpuc->events[idx] = event; in x86_pmu_start()
1535 __set_bit(idx, cpuc->active_mask); in x86_pmu_start()
1553 cntr_mask = hybrid(cpuc->pmu, cntr_mask); in perf_event_print_debug()
1554 fixed_cntr_mask = hybrid(cpuc->pmu, fixed_cntr_mask); in perf_event_print_debug()
1555 pebs_constraints = hybrid(cpuc->pmu, pebs_constraints); in perf_event_print_debug()
1580 pr_info("CPU#%d: active: %016llx\n", cpu, *(u64 *)cpuc->active_mask); in perf_event_print_debug()
1588 pr_info("CPU#%d: gen-PMC%d ctrl: %016llx\n", in perf_event_print_debug()
1590 pr_info("CPU#%d: gen-PMC%d count: %016llx\n", in perf_event_print_debug()
1592 pr_info("CPU#%d: gen-PMC%d left: %016llx\n", in perf_event_print_debug()
1596 if (fixed_counter_disabled(idx, cpuc->pmu)) in perf_event_print_debug()
1600 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", in perf_event_print_debug()
1608 struct hw_perf_event *hwc = &event->hw; in x86_pmu_stop()
1610 if (test_bit(hwc->idx, cpuc->active_mask)) { in x86_pmu_stop()
1612 __clear_bit(hwc->idx, cpuc->active_mask); in x86_pmu_stop()
1613 cpuc->events[hwc->idx] = NULL; in x86_pmu_stop()
1614 WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED); in x86_pmu_stop()
1615 hwc->state |= PERF_HES_STOPPED; in x86_pmu_stop()
1618 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { in x86_pmu_stop()
1620 * Drain the remaining delta count out of a event in x86_pmu_stop()
1624 hwc->state |= PERF_HES_UPTODATE; in x86_pmu_stop()
1631 union perf_capabilities intel_cap = hybrid(cpuc->pmu, intel_cap); in x86_pmu_del()
1635 * If we're called during a txn, we only need to undo x86_pmu.add. in x86_pmu_del()
1636 * The events never got scheduled and ->cancel_txn will truncate in x86_pmu_del()
1639 * XXX assumes any ->del() called during a TXN will only be on in x86_pmu_del()
1642 if (cpuc->txn_flags & PERF_PMU_TXN_ADD) in x86_pmu_del()
1645 __set_bit(event->hw.idx, cpuc->dirty); in x86_pmu_del()
1648 * Not a TXN, therefore cleanup properly. in x86_pmu_del()
1652 for (i = 0; i < cpuc->n_events; i++) { in x86_pmu_del()
1653 if (event == cpuc->event_list[i]) in x86_pmu_del()
1657 if (WARN_ON_ONCE(i == cpuc->n_events)) /* called ->del() without ->add() ? */ in x86_pmu_del()
1660 /* If we have a newly added event; make sure to decrease n_added. */ in x86_pmu_del()
1661 if (i >= cpuc->n_events - cpuc->n_added) in x86_pmu_del()
1662 --cpuc->n_added; in x86_pmu_del()
1667 while (++i < cpuc->n_events) { in x86_pmu_del()
1668 cpuc->event_list[i-1] = cpuc->event_list[i]; in x86_pmu_del()
1669 cpuc->event_constraint[i-1] = cpuc->event_constraint[i]; in x86_pmu_del()
1670 cpuc->assign[i-1] = cpuc->assign[i]; in x86_pmu_del()
1672 cpuc->event_constraint[i-1] = NULL; in x86_pmu_del()
1673 --cpuc->n_events; in x86_pmu_del()
1700 * Some chipsets need to unmask the LVTPC in a particular spot in x86_pmu_handle_irq()
1701 * inside the nmi handler. As a result, the unmasking was pushed in x86_pmu_handle_irq()
1710 if (!test_bit(idx, cpuc->active_mask)) in x86_pmu_handle_irq()
1713 event = cpuc->events[idx]; in x86_pmu_handle_irq()
1714 last_period = event->hw.last_period; in x86_pmu_handle_irq()
1717 if (val & (1ULL << (x86_pmu.cntval_bits - 1))) in x86_pmu_handle_irq()
1730 perf_sample_save_brstack(&data, event, &cpuc->lbr_stack, NULL); in x86_pmu_handle_irq()
1770 perf_sample_event_took(finish_clock - start_clock); in perf_event_nmi_handler()
1785 cpuc->kfree_on_online[i] = NULL; in x86_pmu_prepare_cpu()
1804 kfree(cpuc->kfree_on_online[i]); in x86_pmu_online_cpu()
1805 cpuc->kfree_on_online[i] = NULL; in x86_pmu_online_cpu()
1830 pr_info("no APIC, boot with the \"lapic\" boot parameter to force-enable it.\n"); in pmu_check_apic()
1834 * If we have a PMU initialized but no APIC in pmu_check_apic()
1836 * events (user-space has to fall back and in pmu_check_apic()
1837 * sample via a hrtimer based software event): in pmu_check_apic()
1854 if (pmu_attr->id < x86_pmu.max_events) in events_sysfs_show()
1855 config = x86_pmu.event_map(pmu_attr->id); in events_sysfs_show()
1858 if (pmu_attr->event_str) in events_sysfs_show()
1859 return sprintf(page, "%s\n", pmu_attr->event_str); in events_sysfs_show()
1872 * Report conditional events depending on Hyper-Threading. in events_ht_sysfs_show()
1880 * has to re-read when a thread sibling gets onlined later. in events_ht_sysfs_show()
1884 pmu_attr->event_str_ht : in events_ht_sysfs_show()
1885 pmu_attr->event_str_noht); in events_ht_sysfs_show()
1898 if (hweight64(pmu_attr->pmu_type) == 1) in events_hybrid_sysfs_show()
1899 return sprintf(page, "%s", pmu_attr->event_str); in events_hybrid_sysfs_show()
1903 * event encoding, e.g., the mem-loads event on an Atom PMU has in events_hybrid_sysfs_show()
1904 * different event encoding from a Core PMU. in events_hybrid_sysfs_show()
1912 str = pmu_attr->event_str; in events_hybrid_sysfs_show()
1914 if (!(x86_pmu.hybrid_pmu[i].pmu_type & pmu_attr->pmu_type)) in events_hybrid_sysfs_show()
1916 if (x86_pmu.hybrid_pmu[i].pmu_type & pmu->pmu_type) { in events_hybrid_sysfs_show()
1919 return snprintf(page, next_str - str + 1, "%s", str); in events_hybrid_sysfs_show()
1931 EVENT_ATTR(cpu-cycles, CPU_CYCLES );
1933 EVENT_ATTR(cache-references, CACHE_REFERENCES );
1934 EVENT_ATTR(cache-misses, CACHE_MISSES );
1935 EVENT_ATTR(branch-instructions, BRANCH_INSTRUCTIONS );
1936 EVENT_ATTR(branch-misses, BRANCH_MISSES );
1937 EVENT_ATTR(bus-cycles, BUS_CYCLES );
1938 EVENT_ATTR(stalled-cycles-frontend, STALLED_CYCLES_FRONTEND );
1939 EVENT_ATTR(stalled-cycles-backend, STALLED_CYCLES_BACKEND );
1940 EVENT_ATTR(ref-cycles, REF_CPU_CYCLES );
1972 return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0; in is_visible()
2076 pr_info("... fixed-purpose counters: %d\n", x86_pmu_num_counters_fixed(pmu)); in x86_pmu_show_pmu_cap()
2077 pr_info("... fixed-purpose bitmap: %016llx\n", hybrid(pmu, fixed_cntr_mask64)); in x86_pmu_show_pmu_cap()
2106 err = -ENOTSUPP; in init_hw_perf_events()
2124 for (quirk = x86_pmu.quirks; quirk; quirk = quirk->next) in init_hw_perf_events()
2125 quirk->func(); in init_hw_perf_events()
2195 hybrid_pmu->pmu = pmu; in init_hw_perf_events()
2196 hybrid_pmu->pmu.type = -1; in init_hw_perf_events()
2197 hybrid_pmu->pmu.attr_update = x86_pmu.attr_update; in init_hw_perf_events()
2198 hybrid_pmu->pmu.capabilities |= PERF_PMU_CAP_EXTENDED_HW_TYPE; in init_hw_perf_events()
2200 err = perf_pmu_register(&hybrid_pmu->pmu, hybrid_pmu->name, in init_hw_perf_events()
2201 (hybrid_pmu->pmu_type == hybrid_big) ? PERF_TYPE_RAW : -1); in init_hw_perf_events()
2242 * transaction flags but otherwise ignore non-PERF_PMU_TXN_ADD
2249 WARN_ON_ONCE(cpuc->txn_flags); /* txn already in flight */ in x86_pmu_start_txn()
2251 cpuc->txn_flags = txn_flags; in x86_pmu_start_txn()
2271 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ in x86_pmu_cancel_txn()
2273 txn_flags = cpuc->txn_flags; in x86_pmu_cancel_txn()
2274 cpuc->txn_flags = 0; in x86_pmu_cancel_txn()
2291 * Perform the group schedulability test as a whole
2302 WARN_ON_ONCE(!cpuc->txn_flags); /* no txn in flight */ in x86_pmu_commit_txn()
2304 if (cpuc->txn_flags & ~PERF_PMU_TXN_ADD) { in x86_pmu_commit_txn()
2305 cpuc->txn_flags = 0; in x86_pmu_commit_txn()
2309 n = cpuc->n_events; in x86_pmu_commit_txn()
2312 return -EAGAIN; in x86_pmu_commit_txn()
2322 memcpy(cpuc->assign, assign, n*sizeof(int)); in x86_pmu_commit_txn()
2324 cpuc->txn_flags = 0; in x86_pmu_commit_txn()
2329 * a fake_cpuc is used to validate event groups. Due to
2330 * the extra reg logic, we need to also allocate a fake
2349 return ERR_PTR(-ENOMEM); in allocate_fake_cpuc()
2350 cpuc->is_fake = 1; in allocate_fake_cpuc()
2356 if (cpumask_empty(&h_pmu->supported_cpus)) in allocate_fake_cpuc()
2358 cpu = cpumask_first(&h_pmu->supported_cpus); in allocate_fake_cpuc()
2361 cpuc->pmu = event_pmu; in allocate_fake_cpuc()
2369 return ERR_PTR(-ENOMEM); in allocate_fake_cpuc()
2381 fake_cpuc = allocate_fake_cpuc(event->pmu); in validate_event()
2387 if (!c || !c->weight) in validate_event()
2388 ret = -EINVAL; in validate_event()
2399 * validate a single event group
2402 * - check events are compatible which each other
2403 * - events do not compete for the same counter
2404 * - number of events <= number of counters
2411 struct perf_event *leader = event->group_leader; in validate_group()
2413 int ret = -EINVAL, n; in validate_group()
2423 pmu = leader->pmu; in validate_group()
2429 pmu = sibling->pmu; in validate_group()
2430 else if (pmu != sibling->pmu) in validate_group()
2435 fake_cpuc = allocate_fake_cpuc(event->pmu); in validate_group()
2448 fake_cpuc->n_events = n; in validate_group()
2453 fake_cpuc->n_events = 0; in validate_group()
2466 if ((event->attr.type != event->pmu->type) && in x86_pmu_event_init()
2467 (event->attr.type != PERF_TYPE_HARDWARE) && in x86_pmu_event_init()
2468 (event->attr.type != PERF_TYPE_HW_CACHE)) in x86_pmu_event_init()
2469 return -ENOENT; in x86_pmu_event_init()
2471 if (is_hybrid() && (event->cpu != -1)) { in x86_pmu_event_init()
2472 pmu = hybrid_pmu(event->pmu); in x86_pmu_event_init()
2473 if (!cpumask_test_cpu(event->cpu, &pmu->supported_cpus)) in x86_pmu_event_init()
2474 return -ENOENT; in x86_pmu_event_init()
2479 if (event->group_leader != event) in x86_pmu_event_init()
2485 if (event->destroy) in x86_pmu_event_init()
2486 event->destroy(event); in x86_pmu_event_init()
2487 event->destroy = NULL; in x86_pmu_event_init()
2491 !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) in x86_pmu_event_init()
2492 event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; in x86_pmu_event_init()
2503 for (i = 0; i < cpuc->n_events; i++) in perf_clear_dirty_counters()
2504 __clear_bit(cpuc->assign[i], cpuc->dirty); in perf_clear_dirty_counters()
2506 if (bitmap_empty(cpuc->dirty, X86_PMC_IDX_MAX)) in perf_clear_dirty_counters()
2509 for_each_set_bit(i, cpuc->dirty, X86_PMC_IDX_MAX) { in perf_clear_dirty_counters()
2512 if (!test_bit(i - INTEL_PMC_IDX_FIXED, hybrid(cpuc->pmu, fixed_cntr_mask))) in perf_clear_dirty_counters()
2515 wrmsrq(x86_pmu_fixed_ctr_addr(i - INTEL_PMC_IDX_FIXED), 0); in perf_clear_dirty_counters()
2521 bitmap_zero(cpuc->dirty, X86_PMC_IDX_MAX); in perf_clear_dirty_counters()
2526 if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) in x86_pmu_event_mapped()
2537 * for write. If this changes, we'll need a different solution. in x86_pmu_event_mapped()
2541 if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1) in x86_pmu_event_mapped()
2547 if (!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT)) in x86_pmu_event_unmapped()
2550 if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed)) in x86_pmu_event_unmapped()
2556 struct hw_perf_event *hwc = &event->hw; in x86_pmu_event_idx()
2558 if (!(hwc->flags & PERF_EVENT_FLAG_USER_READ_CNT)) in x86_pmu_event_idx()
2561 if (is_metric_idx(hwc->idx)) in x86_pmu_event_idx()
2564 return hwc->event_base_rdpmc + 1; in x86_pmu_event_idx()
2587 return -EINVAL; in set_attr_rdpmc()
2590 return -ENOTSUPP; in set_attr_rdpmc()
2597 * aka perf-event-bypassing mode. This path is extremely slow, in set_attr_rdpmc()
2670 return -EINVAL; in x86_pmu_check_period()
2676 return -EINVAL; in x86_pmu_check_period()
2738 userpg->cap_user_time = 0; in arch_perf_update_userpage()
2739 userpg->cap_user_time_zero = 0; in arch_perf_update_userpage()
2740 userpg->cap_user_rdpmc = in arch_perf_update_userpage()
2741 !!(event->hw.flags & PERF_EVENT_FLAG_USER_READ_CNT); in arch_perf_update_userpage()
2742 userpg->pmc_width = x86_pmu.cntval_bits; in arch_perf_update_userpage()
2755 userpg->cap_user_time = 1; in arch_perf_update_userpage()
2756 userpg->time_mult = data.cyc2ns_mul; in arch_perf_update_userpage()
2757 userpg->time_shift = data.cyc2ns_shift; in arch_perf_update_userpage()
2758 userpg->time_offset = offset - now; in arch_perf_update_userpage()
2761 * cap_user_time_zero doesn't make sense when we're using a different in arch_perf_update_userpage()
2764 if (!event->attr.use_clockid) { in arch_perf_update_userpage()
2765 userpg->cap_user_time_zero = 1; in arch_perf_update_userpage()
2766 userpg->time_zero = offset; in arch_perf_update_userpage()
2778 return regs->flags & X86_EFLAGS_FIXED; in perf_hw_regs()
2792 if (perf_callchain_store(entry, regs->ip)) in perf_callchain_kernel()
2798 unwind_start(&state, current, NULL, (void *)regs->sp); in perf_callchain_kernel()
2816 unsigned int idx = segment >> 3; in get_segment_base()
2823 * If we're not in a valid context with a real (not just lazy) in get_segment_base()
2830 ldt = smp_load_acquire(¤t->mm->context.ldt); in get_segment_base()
2831 if (!ldt || idx >= ldt->nr_entries) in get_segment_base()
2834 desc = &ldt->entries[idx]; in get_segment_base()
2850 * Heuristic-based check if uprobe is installed at the function entry.
2853 * `push %rbp/%ebp` is a good indicator that we indeed are.
2855 * Similarly, `endbr64` (assuming 64-bit mode) is also a common pattern.
2863 if (!current->utask) in is_uprobe_at_func_entry()
2866 auprobe = current->utask->auprobe; in is_uprobe_at_func_entry()
2871 if (auprobe->insn[0] == 0x55) in is_uprobe_at_func_entry()
2874 /* endbr64 (64-bit only) */ in is_uprobe_at_func_entry()
2875 if (user_64bit_mode(regs) && is_endbr((u32 *)auprobe->insn)) in is_uprobe_at_func_entry()
2895 /* 32-bit process in 64-bit kernel. */ in perf_callchain_user32()
2904 cs_base = get_segment_base(regs->cs); in perf_callchain_user32()
2905 ss_base = get_segment_base(regs->ss); in perf_callchain_user32()
2907 fp = compat_ptr(ss_base + regs->bp); in perf_callchain_user32()
2912 !get_user(ret_addr, (const u32 __user *)regs->sp)) in perf_callchain_user32()
2915 while (entry->nr < entry->max_stack) { in perf_callchain_user32()
2919 if (__get_user(frame.next_frame, &fp->next_frame)) in perf_callchain_user32()
2921 if (__get_user(frame.return_address, &fp->return_address)) in perf_callchain_user32()
2953 if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM)) in perf_callchain_user()
2956 fp = (void __user *)regs->bp; in perf_callchain_user()
2958 perf_callchain_store(entry, regs->ip); in perf_callchain_user()
2970 * entry to user function (which is normally a `push %rbp` instruction, in perf_callchain_user()
2972 * we should read return address from *regs->sp before proceeding in perf_callchain_user()
2977 !get_user(ret_addr, (const unsigned long __user *)regs->sp)) in perf_callchain_user()
2980 while (entry->nr < entry->max_stack) { in perf_callchain_user()
2984 if (__get_user(frame.next_frame, &fp->next_frame)) in perf_callchain_user()
2986 if (__get_user(frame.return_address, &fp->return_address)) in perf_callchain_user()
2998 * VM86 - the good olde 16 bit days, where the linear address is
2999 * 20 bits and we use regs->ip + 0x10 * regs->cs.
3001 * IA32 - Where we need to look at GDT/LDT segment descriptor tables
3004 * X32 - has TIF_X32 set, but is running in x86_64
3006 * X86_64 - CS,DS,SS,ES are all zero based.
3011 * For IA32 we look at the GDT/LDT segment base to convert the in code_segment_base()
3012 * effective IP to a linear address. in code_segment_base()
3017 * If we are in VM86 mode, add the segment offset to convert to a in code_segment_base()
3020 if (regs->flags & X86_VM_MASK) in code_segment_base()
3021 return 0x10 * regs->cs; in code_segment_base()
3023 if (user_mode(regs) && regs->cs != __USER_CS) in code_segment_base()
3024 return get_segment_base(regs->cs); in code_segment_base()
3027 regs->cs != __USER32_CS) in code_segment_base()
3028 return get_segment_base(regs->cs); in code_segment_base()
3035 return regs->ip + code_segment_base(regs); in perf_arch_instruction_pointer()
3040 if (regs->flags & PERF_EFLAGS_EXACT) in common_misc_flags()
3097 * all E-cores are disabled via BIOS. When E-cores are disabled, the in perf_get_x86_pmu_capability()
3098 * base PMU holds the correct number of counters for P-cores. in perf_get_x86_pmu_capability()
3100 cap->version = x86_pmu.version; in perf_get_x86_pmu_capability()
3101 cap->num_counters_gp = x86_pmu_num_counters(NULL); in perf_get_x86_pmu_capability()
3102 cap->num_counters_fixed = x86_pmu_num_counters_fixed(NULL); in perf_get_x86_pmu_capability()
3103 cap->bit_width_gp = x86_pmu.cntval_bits; in perf_get_x86_pmu_capability()
3104 cap->bit_width_fixed = x86_pmu.cntval_bits; in perf_get_x86_pmu_capability()
3105 cap->events_mask = (unsigned int)x86_pmu.events_maskl; in perf_get_x86_pmu_capability()
3106 cap->events_mask_len = x86_pmu.events_mask_len; in perf_get_x86_pmu_capability()
3107 cap->pebs_ept = x86_pmu.pebs_ept; in perf_get_x86_pmu_capability()