Lines Matching +full:re +full:- +full:sampling
1 // SPDX-License-Identifier: GPL-2.0
3 * Performance event support for the System z CPU-measurement Sampling Facility
26 /* Perf PMU definitions for the sampling facility */
28 #define PERF_EVENT_CPUM_SF 0xB0000UL /* Event: Basic-sampling */
29 #define PERF_EVENT_CPUM_SF_DIAG 0xBD000UL /* Event: Combined-sampling */
30 #define PERF_CPUM_SF_BASIC_MODE 0x0001 /* Basic-sampling flag */
31 #define PERF_CPUM_SF_DIAG_MODE 0x0002 /* Diagnostic-sampling flag */
32 #define PERF_CPUM_SF_FREQ_MODE 0x0008 /* Sampling with frequency */
34 #define OVERFLOW_REG(hwc) ((hwc)->extra_reg.config)
35 #define SFB_ALLOC_REG(hwc) ((hwc)->extra_reg.alloc)
36 #define TEAR_REG(hwc) ((hwc)->last_tag)
37 #define SAMPL_RATE(hwc) ((hwc)->event_base)
38 #define SAMPL_FLAGS(hwc) ((hwc)->config_base)
42 /* Minimum number of sample-data-block-tables:
43 * At least one table is required for the sampling buffer structure.
44 * A single table contains up to 511 pointers to sample-data-blocks.
48 /* Number of sample-data-blocks per sample-data-block-table (SDBT):
49 * A table contains SDB pointers (8 bytes) and one table-link entry
52 #define CPUM_SF_SDB_PER_TABLE ((PAGE_SIZE - 8) / 8)
54 /* Maximum page offset for an SDBT table-link entry:
55 * If this page offset is reached, a table-link entry to the next SDBT
64 /* Minimum and maximum sampling buffer sizes:
66 * This number represents the maximum size of the sampling buffer taking
67 * the number of sample-data-block-tables into account. Note that these
68 * numbers apply to the basic-sampling function only.
70 * the diagnostic-sampling function is active.
72 * Sampling buffer size Buffer characteristics
73 * ---------------------------------------------------
75 * 1 page for SDB-tables
79 * 16 pages for SDB-tables
87 unsigned long *sdbt; /* Sample-data-block-table origin */
89 unsigned long num_sdb; /* Number of sample-data-blocks */
90 unsigned long num_sdbt; /* Number of sample-data-block-tables */
91 unsigned long *tail; /* last sample-data-block-table */
104 /* CPU-measurement sampling information block */
106 /* CPU-measurement sampling control block */
108 struct sf_buffer sfb; /* Sampling buffer */
118 /* Sampling control helper functions */
122 return (USEC_PER_SEC / freq) * qsi->cpu_speed; in freq_to_sample_rate()
128 return USEC_PER_SEC * qsi->cpu_speed / rate; in sample_rate_to_freq()
138 ret -= sizeof(struct hws_trailer_entry); in trailer_entry_ptr()
159 * sf_disable() - Switch off sampling facility
170 * sf_buffer_available() - Check for an allocated sampling buffer
174 return !!cpuhw->sfb.sdbt; in sf_buffer_available()
178 * deallocate sampling facility buffer
184 sdbt = sfb->sdbt; in free_sampling_buffer()
187 sfb->sdbt = NULL; in free_sampling_buffer()
193 /* Process table-link entries */ in free_sampling_buffer()
211 /* Allocate and initialize sample-data-block */ in alloc_sample_data_block()
214 return -ENOMEM; in alloc_sample_data_block()
216 te->header.a = 1; in alloc_sample_data_block()
218 /* Link SDB into the sample-data-block-table */ in alloc_sample_data_block()
225 * realloc_sampling_buffer() - extend sampler memory
227 * Allocates new sample-data-blocks and adds them to the specified sampling
230 * Important: This modifies the sampling buffer and must be called when the
231 * sampling facility is disabled.
233 * Returns zero on success, non-zero otherwise.
241 if (!sfb->sdbt || !sfb->tail) in realloc_sampling_buffer()
242 return -EINVAL; in realloc_sampling_buffer()
244 if (!is_link_entry(sfb->tail)) in realloc_sampling_buffer()
245 return -EINVAL; in realloc_sampling_buffer()
247 /* Append to the existing sampling buffer, overwriting the table-link in realloc_sampling_buffer()
249 * The tail variables always points to the "tail" (last and table-link) in realloc_sampling_buffer()
250 * entry in an SDB-table. in realloc_sampling_buffer()
252 tail = sfb->tail; in realloc_sampling_buffer()
254 /* Do a sanity check whether the table-link entry points to in realloc_sampling_buffer()
255 * the sampling buffer origin. in realloc_sampling_buffer()
257 if (sfb->sdbt != get_next_sdbt(tail)) { in realloc_sampling_buffer()
259 __func__, (unsigned long)sfb->sdbt, in realloc_sampling_buffer()
261 return -EINVAL; in realloc_sampling_buffer()
267 /* Allocate a new SDB-table if it is full. */ in realloc_sampling_buffer()
271 rc = -ENOMEM; in realloc_sampling_buffer()
274 sfb->num_sdbt++; in realloc_sampling_buffer()
281 /* Allocate a new sample-data-block. in realloc_sampling_buffer()
294 sfb->num_sdbt--; in realloc_sampling_buffer()
300 sfb->num_sdb++; in realloc_sampling_buffer()
305 /* Link sampling buffer to its origin */ in realloc_sampling_buffer()
306 *tail = virt_to_phys(sfb->sdbt) + 1; in realloc_sampling_buffer()
307 sfb->tail = tail; in realloc_sampling_buffer()
313 * allocate_sampling_buffer() - allocate sampler memory
315 * Allocates and initializes a sampling buffer structure using the
316 * specified number of sample-data-blocks (SDB). For each allocation,
317 * a 4K page is used. The number of sample-data-block-tables (SDBT)
321 * Returns zero on success, non-zero otherwise.
327 if (sfb->sdbt) in alloc_sampling_buffer()
328 return -EINVAL; in alloc_sampling_buffer()
330 /* Allocate the sample-data-block-table origin */ in alloc_sampling_buffer()
331 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL); in alloc_sampling_buffer()
332 if (!sfb->sdbt) in alloc_sampling_buffer()
333 return -ENOMEM; in alloc_sampling_buffer()
334 sfb->num_sdb = 0; in alloc_sampling_buffer()
335 sfb->num_sdbt = 1; in alloc_sampling_buffer()
340 sfb->tail = sfb->sdbt; in alloc_sampling_buffer()
341 *sfb->tail = virt_to_phys((void *)sfb->sdbt) + 1; in alloc_sampling_buffer()
343 /* Allocate requested number of sample-data-blocks */ in alloc_sampling_buffer()
371 if (!sfb->sdbt) in sfb_pending_allocs()
373 if (SFB_ALLOC_REG(hwc) > sfb->num_sdb) in sfb_pending_allocs()
374 return SFB_ALLOC_REG(hwc) - sfb->num_sdb; in sfb_pending_allocs()
381 num = min_t(unsigned long, num, sfb_max_limit(hwc) - SFB_ALLOC_REG(hwc)); in sfb_account_allocs()
395 free_sampling_buffer(&cpuhw->sfb); in deallocate_buffers()
402 /* Calculate sampling buffers using 4K pages in allocate_buffers()
404 * 1. The sampling size is 32 bytes for basic sampling. This size in allocate_buffers()
406 * sampling uses auxlilary data buffer setup which provides the in allocate_buffers()
411 * Control indicator to trigger a measurement-alert to harvest in allocate_buffers()
412 * sample-data-blocks (SDB). This is done per SDB. This in allocate_buffers()
417 * n-th page. This is counterproductive as one IRQ triggers in allocate_buffers()
420 * 3. Use the sampling frequency as input. in allocate_buffers()
423 * SDBs to handle a higher sampling rate. in allocate_buffers()
427 * 4. Compute the number of sample-data-block-tables (SDBT) and in allocate_buffers()
431 freq = sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc)); in allocate_buffers()
434 /* If there is already a sampling buffer allocated, it is very likely in allocate_buffers()
435 * that the sampling facility is enabled too. If the event to be in allocate_buffers()
436 * initialized requires a greater sampling buffer, the allocation must in allocate_buffers()
437 * be postponed. Changing the sampling buffer requires the sampling in allocate_buffers()
446 return alloc_sampling_buffer(&cpuhw->sfb, in allocate_buffers()
447 sfb_pending_allocs(&cpuhw->sfb, hwc)); in allocate_buffers()
458 /* Use a percentage-based approach to extend the sampling facility in compute_sfb_extent()
461 * sample-data-blocks. in compute_sfb_extent()
488 * that has been lost because sample-data-blocks were full. in sfb_account_overflows()
494 ratio = DIV_ROUND_UP(100 * OVERFLOW_REG(hwc) * cpuhw->sfb.num_sdb, in sfb_account_overflows()
495 sample_rate_to_freq(&cpuhw->qsi, SAMPL_RATE(hwc))); in sfb_account_overflows()
497 /* Compute number of sample-data-blocks */ in sfb_account_overflows()
498 num = compute_sfb_extent(ratio, cpuhw->sfb.num_sdb); in sfb_account_overflows()
505 /* extend_sampling_buffer() - Extend sampling buffer
506 * @sfb: Sampling buffer structure (for local CPU)
509 * Use this function to extend the sampling buffer based on the overflow counter
512 * Important: This function disables the sampling facility in order to safely
513 * change the sampling buffer structure. Do not call this function
525 /* Disable the sampling facility to reset any states and also in extend_sampling_buffer()
530 /* Extend the sampling buffer. in extend_sampling_buffer()
533 * new SDB-request cannot be satisfied immediately. in extend_sampling_buffer()
553 qsi(&cpuhw->qsi); in setup_pmc_cpu()
554 cpuhw->flags |= PMU_F_RESERVED; in setup_pmc_cpu()
557 cpuhw->flags &= ~PMU_F_RESERVED; in setup_pmc_cpu()
590 hwc->sample_period = period; in hw_init_period()
591 hwc->last_period = hwc->sample_period; in hw_init_period()
592 local64_set(&hwc->period_left, hwc->sample_period); in hw_init_period()
599 si->min_sampl_rate, si->max_sampl_rate); in hw_limit_rate()
612 pid = -1; in cpumsf_pid_type()
618 if (event->parent) in cpumsf_pid_type()
619 event = event->parent; in cpumsf_pid_type()
620 pid = __task_pid_nr_ns(tsk, type, event->ns); in cpumsf_pid_type()
626 pid = -1; in cpumsf_pid_type()
641 * Obtain the PID from the basic-sampling data entry and in cpumsf_output_event_pid()
642 * correct the data->tid_entry.pid value. in cpumsf_output_event_pid()
644 pid = data->tid_entry.pid; in cpumsf_output_event_pid()
655 data->tid_entry.pid = cpumsf_pid_type(event, pid, PIDTYPE_TGID); in cpumsf_output_event_pid()
656 data->tid_entry.tid = cpumsf_pid_type(event, pid, PIDTYPE_PID); in cpumsf_output_event_pid()
673 /* The min/max sampling rates specifies the valid range in getrate()
681 * sampling rate does not exceed this value. This also helps in getrate()
693 /* The sampling information (si) contains information about the
694 * min/max sampling intervals and the CPU speed. So calculate the
695 * correct sampling interval and avoid the whole period adjust
698 * Since the CPU Measurement sampling facility can not handle frequency
699 * calculate the sampling interval when frequency is specified using
704 * set to the correct sampling rate.
713 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init_rate()
714 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init_rate()
717 if (attr->freq) { in __hw_perf_event_init_rate()
718 if (!attr->sample_freq) in __hw_perf_event_init_rate()
719 return -EINVAL; in __hw_perf_event_init_rate()
720 rate = getrate(attr->freq, attr->sample_freq, si); in __hw_perf_event_init_rate()
721 attr->freq = 0; /* Don't call perf_adjust_period() */ in __hw_perf_event_init_rate()
724 rate = getrate(attr->freq, attr->sample_period, si); in __hw_perf_event_init_rate()
726 return -EINVAL; in __hw_perf_event_init_rate()
728 attr->sample_period = rate; in __hw_perf_event_init_rate()
738 struct perf_event_attr *attr = &event->attr; in __hw_perf_event_init()
739 struct hw_perf_event *hwc = &event->hw; in __hw_perf_event_init()
742 /* Reserve CPU-measurement sampling facility */ in __hw_perf_event_init()
748 event->destroy = hw_perf_event_destroy; in __hw_perf_event_init()
750 /* Access per-CPU sampling information (query sampling info) */ in __hw_perf_event_init()
752 * The event->cpu value can be -1 to count on every CPU, for example, in __hw_perf_event_init()
754 * sampling info from the current CPU, otherwise use event->cpu to in __hw_perf_event_init()
755 * retrieve the per-CPU information. in __hw_perf_event_init()
756 * Later, cpuhw indicates whether to allocate sampling buffers for a in __hw_perf_event_init()
761 if (event->cpu == -1) { in __hw_perf_event_init()
764 /* Event is pinned to a particular CPU, retrieve the per-CPU in __hw_perf_event_init()
765 * sampling structure for accessing the CPU-specific QSI. in __hw_perf_event_init()
767 cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in __hw_perf_event_init()
768 si = cpuhw->qsi; in __hw_perf_event_init()
771 /* Check sampling facility authorization and, if not authorized, in __hw_perf_event_init()
776 err = -ENOENT; in __hw_perf_event_init()
781 pr_warn("CPU Measurement Facility sampling is temporarily not available\n"); in __hw_perf_event_init()
782 err = -EBUSY; in __hw_perf_event_init()
786 /* Always enable basic sampling */ in __hw_perf_event_init()
789 /* Check if diagnostic sampling is requested. Deny if the required in __hw_perf_event_init()
790 * sampling authorization is missing. in __hw_perf_event_init()
792 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) { in __hw_perf_event_init()
794 err = -EPERM; in __hw_perf_event_init()
805 if (attr->config == PERF_EVENT_CPUM_SF_DIAG) in __hw_perf_event_init()
808 /* Allocate the per-CPU sampling buffer using the CPU information in __hw_perf_event_init()
810 * CPU (event->cpu == -1; or cpuhw == NULL), allocate sampling in __hw_perf_event_init()
817 /* Event is not pinned, allocate sampling buffer on in __hw_perf_event_init()
828 /* If PID/TID sampling is active, replace the default overflow in __hw_perf_event_init()
829 * handler to extract and resolve the PIDs from the basic-sampling in __hw_perf_event_init()
832 if (event->attr.sample_type & PERF_SAMPLE_TID) in __hw_perf_event_init()
834 event->overflow_handler = cpumsf_output_event_pid; in __hw_perf_event_init()
842 u64 sample_type = event->attr.sample_type; in is_callchain_event()
852 /* No support for taken branch sampling */ in cpumsf_pmu_event_init()
855 return -EOPNOTSUPP; in cpumsf_pmu_event_init()
857 switch (event->attr.type) { in cpumsf_pmu_event_init()
859 if ((event->attr.config != PERF_EVENT_CPUM_SF) && in cpumsf_pmu_event_init()
860 (event->attr.config != PERF_EVENT_CPUM_SF_DIAG)) in cpumsf_pmu_event_init()
861 return -ENOENT; in cpumsf_pmu_event_init()
864 /* Support sampling of CPU cycles in addition to the in cpumsf_pmu_event_init()
867 * sampling events only. in cpumsf_pmu_event_init()
869 if (event->attr.config != PERF_COUNT_HW_CPU_CYCLES) in cpumsf_pmu_event_init()
870 return -ENOENT; in cpumsf_pmu_event_init()
872 return -ENOENT; in cpumsf_pmu_event_init()
875 return -ENOENT; in cpumsf_pmu_event_init()
881 if (event->attr.exclude_hv) in cpumsf_pmu_event_init()
882 event->attr.exclude_hv = 0; in cpumsf_pmu_event_init()
883 if (event->attr.exclude_idle) in cpumsf_pmu_event_init()
884 event->attr.exclude_idle = 0; in cpumsf_pmu_event_init()
898 * - added/started on this CPU (PMU_F_IN_USE set) in cpumsf_pmu_enable()
899 * - and CPU must be available (PMU_F_RESERVED set) in cpumsf_pmu_enable()
900 * - and not already enabled (PMU_F_ENABLED not set) in cpumsf_pmu_enable()
901 * - and not in error condition (PMU_F_ERR_MASK not set) in cpumsf_pmu_enable()
903 if (cpuhw->flags != (PMU_F_IN_USE | PMU_F_RESERVED)) in cpumsf_pmu_enable()
906 /* Check whether to extent the sampling buffer. in cpumsf_pmu_enable()
908 * Two conditions trigger an increase of the sampling buffer for a in cpumsf_pmu_enable()
911 * 2. Sampling overflows that contribute to pending allocations. in cpumsf_pmu_enable()
913 * Note that the extend_sampling_buffer() function disables the sampling in cpumsf_pmu_enable()
914 * facility, but it can be fully re-enabled using sampling controls that in cpumsf_pmu_enable()
917 hwc = &cpuhw->event->hw; in cpumsf_pmu_enable()
920 * Account number of overflow-designated buffer extents in cpumsf_pmu_enable()
923 extend_sampling_buffer(&cpuhw->sfb, hwc); in cpumsf_pmu_enable()
926 cpuhw->lsctl.interval = SAMPL_RATE(hwc); in cpumsf_pmu_enable()
928 /* (Re)enable the PMU and sampling facility */ in cpumsf_pmu_enable()
929 err = lsctl(&cpuhw->lsctl); in cpumsf_pmu_enable()
931 pr_err("Loading sampling controls failed: op 1 err %i\n", err); in cpumsf_pmu_enable()
936 lpp(&get_lowcore()->lpp); in cpumsf_pmu_enable()
937 cpuhw->flags |= PMU_F_ENABLED; in cpumsf_pmu_enable()
947 if (!(cpuhw->flags & PMU_F_ENABLED)) in cpumsf_pmu_disable()
950 if (cpuhw->flags & PMU_F_ERR_MASK) in cpumsf_pmu_disable()
953 /* Switch off sampling activation control */ in cpumsf_pmu_disable()
954 inactive = cpuhw->lsctl; in cpumsf_pmu_disable()
960 pr_err("Loading sampling controls failed: op 2 err %i\n", err); in cpumsf_pmu_disable()
966 * TEAR/DEAR values are valid only if the sampling facility is in cpumsf_pmu_disable()
968 * for a disabled sampling facility because cpumsf_pmu_enable() in cpumsf_pmu_disable()
973 cpuhw->lsctl.tear = si.tear; in cpumsf_pmu_disable()
974 cpuhw->lsctl.dear = si.dear; in cpumsf_pmu_disable()
977 cpuhw->flags &= ~PMU_F_ENABLED; in cpumsf_pmu_disable()
980 /* perf_event_exclude() - Filter event
983 * @sde_regs: Sample-data-entry (sde) regs structure
987 * Return non-zero if the event shall be excluded.
992 if (event->attr.exclude_user && user_mode(regs)) in perf_event_exclude()
994 if (event->attr.exclude_kernel && !user_mode(regs)) in perf_event_exclude()
996 if (event->attr.exclude_guest && sde_regs->in_guest) in perf_event_exclude()
998 if (event->attr.exclude_host && !sde_regs->in_guest) in perf_event_exclude()
1003 /* perf_push_sample() - Push samples to perf
1012 * Return non-zero if an event overflow occurred.
1023 perf_sample_data_init(&data, 0, event->hw.last_period); in perf_push_sample()
1025 /* Setup pt_regs to look like an CPU-measurement external interrupt in perf_push_sample()
1027 * field which is unused contains additional sample-data-entry related in perf_push_sample()
1035 psw_bits(regs.psw).ia = basic->ia; in perf_push_sample()
1036 psw_bits(regs.psw).dat = basic->T; in perf_push_sample()
1037 psw_bits(regs.psw).wait = basic->W; in perf_push_sample()
1038 psw_bits(regs.psw).pstate = basic->P; in perf_push_sample()
1039 psw_bits(regs.psw).as = basic->AS; in perf_push_sample()
1045 * A non-zero guest program parameter always indicates a guest in perf_push_sample()
1052 switch (basic->CL) { in perf_push_sample()
1054 sde_regs->in_guest = 0; in perf_push_sample()
1057 sde_regs->in_guest = 1; in perf_push_sample()
1060 if (basic->gpp || basic->prim_asn != 0xffff) in perf_push_sample()
1061 sde_regs->in_guest = 1; in perf_push_sample()
1066 * Store the PID value from the sample-data-entry to be in perf_push_sample()
1069 data.tid_entry.pid = basic->hpp & LPP_PID_MASK; in perf_push_sample()
1082 local64_add(count, &event->count); in perf_event_count_update()
1085 /* hw_collect_samples() - Walk through a sample-data-block and collect samples
1087 * @sdbt: Sample-data-block table
1090 * Walks through a sample-data-block and collects sampling data entries that are
1091 * then pushed to the perf event subsystem. Depending on the sampling function,
1092 * there can be either basic-sampling or combined-sampling data entries. A
1093 * combined-sampling data entry consists of a basic- and a diagnostic-sampling
1094 * data entry. The sampling function is determined by the flags in the perf
1095 * event hardware structure. The function always works with a combined-sampling
1098 * Note that the implementation focuses on basic-sampling data entries and, if
1099 * such an entry is not valid, the entire combined-sampling data entry is
1115 if (!sample->def || sample->LS) in hw_collect_samples()
1119 perf_event_count_update(event, SAMPL_RATE(&event->hw)); in hw_collect_samples()
1122 if (sample->def == 0x0001) { in hw_collect_samples()
1129 if (sample->I == 0 && sample->W == 0) { in hw_collect_samples()
1141 * from a combined basic- and diagnostic-sampling. in hw_collect_samples()
1142 * If only basic-sampling is then active, entries are in hw_collect_samples()
1144 * This is typically the case for sample-data-blocks in hw_collect_samples()
1148 if (!te->header.f) in hw_collect_samples()
1153 sample->def = 0; in hw_collect_samples()
1158 /* hw_perf_event_update() - Process sampling buffer
1160 * @flush_all: Flag to also flush partially filled sample-data-blocks
1162 * Processes the sampling buffer and create perf event samples.
1163 * The sampling buffer position are retrieved and saved in the TEAR_REG
1166 * Only full sample-data-blocks are processed. Specify the flush_all flag
1167 * to also walk through partially filled sample-data-blocks.
1172 struct hw_perf_event *hwc = &event->hw; in hw_perf_event_update()
1179 * AUX buffer is used when in diagnostic sampling mode. in hw_perf_event_update()
1188 /* Get the trailer entry of the sample-data-block */ in hw_perf_event_update()
1193 if (!te->header.f) { in hw_perf_event_update()
1200 if (te->header.overflow) in hw_perf_event_update()
1202 * is reached, extend the sampling buffer. in hw_perf_event_update()
1205 sampl_overflow += te->header.overflow; in hw_perf_event_update()
1207 /* Collect all samples from a single sample-data-block and in hw_perf_event_update()
1214 /* Reset trailer (using compare-double-and-swap) */ in hw_perf_event_update()
1215 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in hw_perf_event_update()
1221 } while (!try_cmpxchg128(&te->header.val, &prev.val, new.val)); in hw_perf_event_update()
1223 /* Advance to next sample-data-block */ in hw_perf_event_update()
1231 /* Stop processing sample-data if all samples of the current in hw_perf_event_update()
1232 * sample-data-block were flushed even if it was not full. in hw_perf_event_update()
1258 return i % aux->sfb.num_sdb; in aux_sdb_index()
1263 return end >= start ? end - start + 1 : 0; in aux_sdb_num()
1268 return aux_sdb_num(aux->head, aux->alert_mark); in aux_sdb_num_alert()
1273 return aux_sdb_num(aux->head, aux->empty_mark); in aux_sdb_num_empty()
1285 sdb = aux->sdb_index[index]; in aux_sdb_trailer()
1290 * Finish sampling on the cpu. Called by cpumsf_pmu_del() with pmu
1309 for (i = 0, idx = aux->head; i < range_scan; i++, idx++) { in aux_output_end()
1311 if (!te->header.f) in aux_output_end()
1318 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_end()
1319 te->header.a = 0; in aux_output_end()
1323 * Start sampling on the CPU. Called by cpumsf_pmu_add() when an event
1338 if (handle->head & ~PAGE_MASK) in aux_output_begin()
1339 return -EINVAL; in aux_output_begin()
1341 aux->head = handle->head >> PAGE_SHIFT; in aux_output_begin()
1342 range = (handle->size + 1) >> PAGE_SHIFT; in aux_output_begin()
1344 return -ENOMEM; in aux_output_begin()
1347 * SDBs between aux->head and aux->empty_mark are already ready in aux_output_begin()
1351 range_scan = range - aux_sdb_num_empty(aux); in aux_output_begin()
1352 idx = aux->empty_mark + 1; in aux_output_begin()
1355 te->header.f = 0; in aux_output_begin()
1356 te->header.a = 0; in aux_output_begin()
1357 te->header.overflow = 0; in aux_output_begin()
1360 aux->empty_mark = aux->head + range - 1; in aux_output_begin()
1364 aux->alert_mark = aux->head + range/2 - 1; in aux_output_begin()
1365 te = aux_sdb_trailer(aux, aux->alert_mark); in aux_output_begin()
1366 te->header.a = 1; in aux_output_begin()
1369 head = aux_sdb_index(aux, aux->head); in aux_output_begin()
1370 base = aux->sdbt_index[head / CPUM_SF_SDB_PER_TABLE]; in aux_output_begin()
1372 cpuhw->lsctl.tear = virt_to_phys((void *)base) + offset * sizeof(unsigned long); in aux_output_begin()
1373 cpuhw->lsctl.dear = virt_to_phys((void *)aux->sdb_index[head]); in aux_output_begin()
1391 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in aux_set_alert()
1405 } while (!try_cmpxchg128(&te->header.val, &prev.val, new.val)); in aux_set_alert()
1410 * aux_reset_buffer() - Scan and setup SDBs for new samples
1412 * @range: The range of SDBs to scan started from aux->head
1415 * Set alert indicator on the SDB at index of aux->alert_mark. If this SDB is
1420 * Scan the SDBs in AUX buffer from behind aux->empty_mark. They are used
1423 * If aux->alert_mark fall in this area, just set it. Overflow count is
1426 * SDBs between aux->head and aux->empty_mark are already reset at last time.
1445 return aux_set_alert(aux, aux->alert_mark, overflow); in aux_reset_buffer()
1447 if (aux->alert_mark <= aux->empty_mark) in aux_reset_buffer()
1452 if (!aux_set_alert(aux, aux->alert_mark, overflow)) in aux_reset_buffer()
1460 range_scan = range - aux_sdb_num_empty(aux); in aux_reset_buffer()
1461 idx = aux->empty_mark + 1; in aux_reset_buffer()
1464 prev.val = READ_ONCE_ALIGNED_128(te->header.val); in aux_reset_buffer()
1470 if (idx == aux->alert_mark) in aux_reset_buffer()
1474 } while (!try_cmpxchg128(&te->header.val, &prev.val, new.val)); in aux_reset_buffer()
1479 aux->empty_mark = aux->head + range - 1; in aux_reset_buffer()
1485 * Measurement alert handler for diagnostic mode sampling.
1493 struct perf_output_handle *handle = &cpuhw->handle; in hw_collect_aux()
1506 num_sdb = aux->sfb.num_sdb; in hw_collect_aux()
1509 aux = perf_aux_output_begin(handle, cpuhw->event); in hw_collect_aux()
1510 if (handle->size == 0) { in hw_collect_aux()
1512 "diagnostic-sampling mode is full\n", in hw_collect_aux()
1520 aux->head = handle->head >> PAGE_SHIFT; in hw_collect_aux()
1521 range = (handle->size + 1) >> PAGE_SHIFT; in hw_collect_aux()
1523 aux->alert_mark = aux->head; in hw_collect_aux()
1525 aux->alert_mark = aux->head + range/2 - 1; in hw_collect_aux()
1533 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1535 "pages to overflow\n", aux->sfb.num_sdb); in hw_collect_aux()
1538 perf_aux_output_end(&cpuhw->handle, size); in hw_collect_aux()
1555 num_sdbt = aux->sfb.num_sdbt; in aux_buffer_free()
1557 free_page(aux->sdbt_index[i]); in aux_buffer_free()
1559 kfree(aux->sdbt_index); in aux_buffer_free()
1560 kfree(aux->sdb_index); in aux_buffer_free()
1571 te->clock_base = 1; in aux_sdb_init()
1572 te->progusage2 = tod_clock_base.tod; in aux_sdb_init()
1576 * aux_buffer_setup() - Setup AUX buffer for diagnostic mode sampling
1577 * @event: Event the buffer is setup for, event->cpu == -1 means current
1585 * the task among online cpus when it is a per-thread event.
1602 "maximum sampling buffer limit\n", in aux_buffer_setup()
1607 "minimum sampling buffer limit\n", in aux_buffer_setup()
1616 sfb = &aux->sfb; in aux_buffer_setup()
1620 aux->sdbt_index = kmalloc_array(n_sdbt, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1621 if (!aux->sdbt_index) in aux_buffer_setup()
1625 aux->sdb_index = kmalloc_array(nr_pages, sizeof(void *), GFP_KERNEL); in aux_buffer_setup()
1626 if (!aux->sdb_index) in aux_buffer_setup()
1630 sfb->num_sdbt = 0; in aux_buffer_setup()
1631 sfb->sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL); in aux_buffer_setup()
1632 if (!sfb->sdbt) in aux_buffer_setup()
1634 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)sfb->sdbt; in aux_buffer_setup()
1635 tail = sfb->tail = sfb->sdbt; in aux_buffer_setup()
1646 aux->sdbt_index[sfb->num_sdbt++] = (unsigned long)new; in aux_buffer_setup()
1653 aux->sdb_index[i] = (unsigned long)pages[i]; in aux_buffer_setup()
1656 sfb->num_sdb = nr_pages; in aux_buffer_setup()
1659 *tail = virt_to_phys(sfb->sdbt) + 1; in aux_buffer_setup()
1660 sfb->tail = tail; in aux_buffer_setup()
1667 aux->empty_mark = sfb->num_sdb - 1; in aux_buffer_setup()
1673 for (i = 0; i < sfb->num_sdbt; i++) in aux_buffer_setup()
1674 free_page(aux->sdbt_index[i]); in aux_buffer_setup()
1675 kfree(aux->sdb_index); in aux_buffer_setup()
1677 kfree(aux->sdbt_index); in aux_buffer_setup()
1686 /* Nothing to do ... updates are interrupt-driven */ in cpumsf_pmu_read()
1689 /* Check if the new sampling period/frequency is appropriate.
1691 * Return non-zero on error and zero on passed checks.
1700 if (event->cpu == -1) { in cpumsf_pmu_check_period()
1703 /* Event is pinned to a particular CPU, retrieve the per-CPU in cpumsf_pmu_check_period()
1704 * sampling structure for accessing the CPU-specific QSI. in cpumsf_pmu_check_period()
1706 struct cpu_hw_sf *cpuhw = &per_cpu(cpu_hw_sf, event->cpu); in cpumsf_pmu_check_period()
1708 si = cpuhw->qsi; in cpumsf_pmu_check_period()
1711 do_freq = !!SAMPL_FREQ_MODE(&event->hw); in cpumsf_pmu_check_period()
1714 return -EINVAL; in cpumsf_pmu_check_period()
1716 event->attr.sample_period = rate; in cpumsf_pmu_check_period()
1717 SAMPL_RATE(&event->hw) = rate; in cpumsf_pmu_check_period()
1718 hw_init_period(&event->hw, SAMPL_RATE(&event->hw)); in cpumsf_pmu_check_period()
1722 /* Activate sampling control.
1723 * Next call of pmu_enable() starts sampling.
1729 if (!(event->hw.state & PERF_HES_STOPPED)) in cpumsf_pmu_start()
1731 perf_pmu_disable(event->pmu); in cpumsf_pmu_start()
1732 event->hw.state = 0; in cpumsf_pmu_start()
1733 cpuhw->lsctl.cs = 1; in cpumsf_pmu_start()
1734 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_start()
1735 cpuhw->lsctl.cd = 1; in cpumsf_pmu_start()
1736 perf_pmu_enable(event->pmu); in cpumsf_pmu_start()
1739 /* Deactivate sampling control.
1740 * Next call of pmu_enable() stops sampling.
1746 if (event->hw.state & PERF_HES_STOPPED) in cpumsf_pmu_stop()
1749 perf_pmu_disable(event->pmu); in cpumsf_pmu_stop()
1750 cpuhw->lsctl.cs = 0; in cpumsf_pmu_stop()
1751 cpuhw->lsctl.cd = 0; in cpumsf_pmu_stop()
1752 event->hw.state |= PERF_HES_STOPPED; in cpumsf_pmu_stop()
1754 if ((flags & PERF_EF_UPDATE) && !(event->hw.state & PERF_HES_UPTODATE)) { in cpumsf_pmu_stop()
1756 if (cpuhw->flags & PMU_F_RESERVED) in cpumsf_pmu_stop()
1758 event->hw.state |= PERF_HES_UPTODATE; in cpumsf_pmu_stop()
1760 perf_pmu_enable(event->pmu); in cpumsf_pmu_stop()
1769 if (cpuhw->flags & PMU_F_IN_USE) in cpumsf_pmu_add()
1770 return -EAGAIN; in cpumsf_pmu_add()
1772 if (!SAMPL_DIAG_MODE(&event->hw) && !sf_buffer_available(cpuhw)) in cpumsf_pmu_add()
1773 return -EINVAL; in cpumsf_pmu_add()
1775 perf_pmu_disable(event->pmu); in cpumsf_pmu_add()
1777 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; in cpumsf_pmu_add()
1779 /* Set up sampling controls. Always program the sampling register in cpumsf_pmu_add()
1780 * using the SDB-table start. Reset TEAR_REG event hardware register in cpumsf_pmu_add()
1781 * that is used by hw_perf_event_update() to store the sampling buffer in cpumsf_pmu_add()
1784 cpuhw->lsctl.s = 0; in cpumsf_pmu_add()
1785 cpuhw->lsctl.h = 1; in cpumsf_pmu_add()
1786 cpuhw->lsctl.interval = SAMPL_RATE(&event->hw); in cpumsf_pmu_add()
1787 if (!SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1788 cpuhw->lsctl.tear = virt_to_phys(cpuhw->sfb.sdbt); in cpumsf_pmu_add()
1789 cpuhw->lsctl.dear = *(unsigned long *)cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1790 TEAR_REG(&event->hw) = (unsigned long)cpuhw->sfb.sdbt; in cpumsf_pmu_add()
1793 /* Ensure sampling functions are in the disabled state. If disabled, in cpumsf_pmu_add()
1794 * switch on sampling enable control. */ in cpumsf_pmu_add()
1795 if (WARN_ON_ONCE(cpuhw->lsctl.es == 1 || cpuhw->lsctl.ed == 1)) { in cpumsf_pmu_add()
1796 err = -EAGAIN; in cpumsf_pmu_add()
1799 if (SAMPL_DIAG_MODE(&event->hw)) { in cpumsf_pmu_add()
1800 aux = perf_aux_output_begin(&cpuhw->handle, event); in cpumsf_pmu_add()
1802 err = -EINVAL; in cpumsf_pmu_add()
1805 err = aux_output_begin(&cpuhw->handle, aux, cpuhw); in cpumsf_pmu_add()
1808 cpuhw->lsctl.ed = 1; in cpumsf_pmu_add()
1810 cpuhw->lsctl.es = 1; in cpumsf_pmu_add()
1813 cpuhw->event = event; in cpumsf_pmu_add()
1814 cpuhw->flags |= PMU_F_IN_USE; in cpumsf_pmu_add()
1820 perf_pmu_enable(event->pmu); in cpumsf_pmu_add()
1828 perf_pmu_disable(event->pmu); in cpumsf_pmu_del()
1831 cpuhw->lsctl.es = 0; in cpumsf_pmu_del()
1832 cpuhw->lsctl.ed = 0; in cpumsf_pmu_del()
1833 cpuhw->flags &= ~PMU_F_IN_USE; in cpumsf_pmu_del()
1834 cpuhw->event = NULL; in cpumsf_pmu_del()
1836 if (SAMPL_DIAG_MODE(&event->hw)) in cpumsf_pmu_del()
1837 aux_output_end(&cpuhw->handle); in cpumsf_pmu_del()
1839 perf_pmu_enable(event->pmu); in cpumsf_pmu_del()
1847 * The availablitiy depends on the CPU_MF sampling facility authorization
1849 * time by the sampling facility device driver.
1851 * also turned off for diagnostic sampling.
1854 * level for diagnostic sampling and installs the attribute
1855 * file for diagnostic sampling if necessary.
1871 PMU_FORMAT_ATTR(event, "config:0-63");
1926 if (!(cpuhw->flags & PMU_F_RESERVED)) in cpumf_measurement_alert()
1934 if (cpuhw->flags & PMU_F_IN_USE) { in cpumf_measurement_alert()
1935 if (SAMPL_DIAG_MODE(&cpuhw->event->hw)) in cpumf_measurement_alert()
1938 hw_perf_event_update(cpuhw->event, 0); in cpumf_measurement_alert()
1942 /* Report measurement alerts only for non-PRA codes */ in cpumf_measurement_alert()
1947 /* Sampling authorization change request */ in cpumf_measurement_alert()
1949 qsi(&cpuhw->qsi); in cpumf_measurement_alert()
1951 /* Loss of sample data due to high-priority machine activities */ in cpumf_measurement_alert()
1954 cpuhw->flags |= PMU_F_ERR_LSDA; in cpumf_measurement_alert()
1958 /* Invalid sampling buffer entry */ in cpumf_measurement_alert()
1960 pr_err("A sampling buffer entry is incorrect (alert=%#x)\n", in cpumf_measurement_alert()
1962 cpuhw->flags |= PMU_F_ERR_IBE; in cpumf_measurement_alert()
1994 return -ENODEV; in param_get_sfb_size()
2004 return -ENODEV; in param_set_sfb_size()
2006 return -EINVAL; in param_set_sfb_size()
2012 rc = (sscanf(val, "%lu,%lu", &min, &max) == 2) ? 0 : -EINVAL; in param_set_sfb_size()
2017 rc = -EINVAL; in param_set_sfb_size()
2022 pr_info("The sampling buffer limits have changed to: " in param_set_sfb_size()
2035 RS_INIT_FAILURE_BSDES = 2, /* Bad basic sampling size */
2042 pr_err("Sampling facility support for perf is not available: " in pr_cpumsf_err()
2052 return -ENODEV; in init_cpum_sampling_pmu()
2057 return -ENODEV; in init_cpum_sampling_pmu()
2061 return -EINVAL; in init_cpum_sampling_pmu()
2066 /* Sampling of diagnostic data authorized, in init_cpum_sampling_pmu()
2076 return -ENOMEM; in init_cpum_sampling_pmu()