1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance event support for s390x - CPU-measurement Counter Facility 4 * 5 * Copyright IBM Corp. 2012, 2023 6 * Author(s): Hendrik Brueckner <brueckner@linux.ibm.com> 7 * Thomas Richter <tmricht@linux.ibm.com> 8 */ 9 #define KMSG_COMPONENT "cpum_cf" 10 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/kernel_stat.h> 14 #include <linux/percpu.h> 15 #include <linux/notifier.h> 16 #include <linux/init.h> 17 #include <linux/export.h> 18 #include <linux/miscdevice.h> 19 #include <linux/perf_event.h> 20 21 #include <asm/cpu_mf.h> 22 #include <asm/hwctrset.h> 23 #include <asm/debug.h> 24 25 enum cpumf_ctr_set { 26 CPUMF_CTR_SET_BASIC = 0, /* Basic Counter Set */ 27 CPUMF_CTR_SET_USER = 1, /* Problem-State Counter Set */ 28 CPUMF_CTR_SET_CRYPTO = 2, /* Crypto-Activity Counter Set */ 29 CPUMF_CTR_SET_EXT = 3, /* Extended Counter Set */ 30 CPUMF_CTR_SET_MT_DIAG = 4, /* MT-diagnostic Counter Set */ 31 32 /* Maximum number of counter sets */ 33 CPUMF_CTR_SET_MAX, 34 }; 35 36 #define CPUMF_LCCTL_ENABLE_SHIFT 16 37 #define CPUMF_LCCTL_ACTCTL_SHIFT 0 38 39 static inline void ctr_set_enable(u64 *state, u64 ctrsets) 40 { 41 *state |= ctrsets << CPUMF_LCCTL_ENABLE_SHIFT; 42 } 43 44 static inline void ctr_set_disable(u64 *state, u64 ctrsets) 45 { 46 *state &= ~(ctrsets << CPUMF_LCCTL_ENABLE_SHIFT); 47 } 48 49 static inline void ctr_set_start(u64 *state, u64 ctrsets) 50 { 51 *state |= ctrsets << CPUMF_LCCTL_ACTCTL_SHIFT; 52 } 53 54 static inline void ctr_set_stop(u64 *state, u64 ctrsets) 55 { 56 *state &= ~(ctrsets << CPUMF_LCCTL_ACTCTL_SHIFT); 57 } 58 59 static inline int ctr_stcctm(enum cpumf_ctr_set set, u64 range, u64 *dest) 60 { 61 switch (set) { 62 case CPUMF_CTR_SET_BASIC: 63 return stcctm(BASIC, range, dest); 64 case CPUMF_CTR_SET_USER: 65 return stcctm(PROBLEM_STATE, range, dest); 66 case CPUMF_CTR_SET_CRYPTO: 67 return stcctm(CRYPTO_ACTIVITY, range, dest); 68 case CPUMF_CTR_SET_EXT: 69 return stcctm(EXTENDED, range, dest); 70 case CPUMF_CTR_SET_MT_DIAG: 71 return stcctm(MT_DIAG_CLEARING, range, dest); 72 case CPUMF_CTR_SET_MAX: 73 return 3; 74 } 75 return 3; 76 } 77 78 struct cpu_cf_events { 79 struct cpumf_ctr_info info; 80 atomic_t ctr_set[CPUMF_CTR_SET_MAX]; 81 u64 state; /* For perf_event_open SVC */ 82 u64 dev_state; /* For /dev/hwctr */ 83 unsigned int flags; 84 size_t used; /* Bytes used in data */ 85 size_t usedss; /* Bytes used in start/stop */ 86 unsigned char start[PAGE_SIZE]; /* Counter set at event add */ 87 unsigned char stop[PAGE_SIZE]; /* Counter set at event delete */ 88 unsigned char data[PAGE_SIZE]; /* Counter set at /dev/hwctr */ 89 unsigned int sets; /* # Counter set saved in memory */ 90 }; 91 92 /* Per-CPU event structure for the counter facility */ 93 static DEFINE_PER_CPU(struct cpu_cf_events, cpu_cf_events); 94 95 static unsigned int cfdiag_cpu_speed; /* CPU speed for CF_DIAG trailer */ 96 static debug_info_t *cf_dbg; 97 98 #define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ 99 /* interval in seconds */ 100 101 /* Counter sets are stored as data stream in a page sized memory buffer and 102 * exported to user space via raw data attached to the event sample data. 103 * Each counter set starts with an eight byte header consisting of: 104 * - a two byte eye catcher (0xfeef) 105 * - a one byte counter set number 106 * - a two byte counter set size (indicates the number of counters in this set) 107 * - a three byte reserved value (must be zero) to make the header the same 108 * size as a counter value. 109 * All counter values are eight byte in size. 110 * 111 * All counter sets are followed by a 64 byte trailer. 112 * The trailer consists of a: 113 * - flag field indicating valid fields when corresponding bit set 114 * - the counter facility first and second version number 115 * - the CPU speed if nonzero 116 * - the time stamp the counter sets have been collected 117 * - the time of day (TOD) base value 118 * - the machine type. 119 * 120 * The counter sets are saved when the process is prepared to be executed on a 121 * CPU and saved again when the process is going to be removed from a CPU. 122 * The difference of both counter sets are calculated and stored in the event 123 * sample data area. 124 */ 125 struct cf_ctrset_entry { /* CPU-M CF counter set entry (8 byte) */ 126 unsigned int def:16; /* 0-15 Data Entry Format */ 127 unsigned int set:16; /* 16-31 Counter set identifier */ 128 unsigned int ctr:16; /* 32-47 Number of stored counters */ 129 unsigned int res1:16; /* 48-63 Reserved */ 130 }; 131 132 struct cf_trailer_entry { /* CPU-M CF_DIAG trailer (64 byte) */ 133 /* 0 - 7 */ 134 union { 135 struct { 136 unsigned int clock_base:1; /* TOD clock base set */ 137 unsigned int speed:1; /* CPU speed set */ 138 /* Measurement alerts */ 139 unsigned int mtda:1; /* Loss of MT ctr. data alert */ 140 unsigned int caca:1; /* Counter auth. change alert */ 141 unsigned int lcda:1; /* Loss of counter data alert */ 142 }; 143 unsigned long flags; /* 0-63 All indicators */ 144 }; 145 /* 8 - 15 */ 146 unsigned int cfvn:16; /* 64-79 Ctr First Version */ 147 unsigned int csvn:16; /* 80-95 Ctr Second Version */ 148 unsigned int cpu_speed:32; /* 96-127 CPU speed */ 149 /* 16 - 23 */ 150 unsigned long timestamp; /* 128-191 Timestamp (TOD) */ 151 /* 24 - 55 */ 152 union { 153 struct { 154 unsigned long progusage1; 155 unsigned long progusage2; 156 unsigned long progusage3; 157 unsigned long tod_base; 158 }; 159 unsigned long progusage[4]; 160 }; 161 /* 56 - 63 */ 162 unsigned int mach_type:16; /* Machine type */ 163 unsigned int res1:16; /* Reserved */ 164 unsigned int res2:32; /* Reserved */ 165 }; 166 167 /* Create the trailer data at the end of a page. */ 168 static void cfdiag_trailer(struct cf_trailer_entry *te) 169 { 170 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 171 struct cpuid cpuid; 172 173 te->cfvn = cpuhw->info.cfvn; /* Counter version numbers */ 174 te->csvn = cpuhw->info.csvn; 175 176 get_cpu_id(&cpuid); /* Machine type */ 177 te->mach_type = cpuid.machine; 178 te->cpu_speed = cfdiag_cpu_speed; 179 if (te->cpu_speed) 180 te->speed = 1; 181 te->clock_base = 1; /* Save clock base */ 182 te->tod_base = tod_clock_base.tod; 183 te->timestamp = get_tod_clock_fast(); 184 } 185 186 /* 187 * Return the maximum possible counter set size (in number of 8 byte counters) 188 * depending on type and model number. 189 */ 190 static size_t cpum_cf_ctrset_size(enum cpumf_ctr_set ctrset, 191 struct cpumf_ctr_info *info) 192 { 193 size_t ctrset_size = 0; 194 195 switch (ctrset) { 196 case CPUMF_CTR_SET_BASIC: 197 if (info->cfvn >= 1) 198 ctrset_size = 6; 199 break; 200 case CPUMF_CTR_SET_USER: 201 if (info->cfvn == 1) 202 ctrset_size = 6; 203 else if (info->cfvn >= 3) 204 ctrset_size = 2; 205 break; 206 case CPUMF_CTR_SET_CRYPTO: 207 if (info->csvn >= 1 && info->csvn <= 5) 208 ctrset_size = 16; 209 else if (info->csvn == 6 || info->csvn == 7) 210 ctrset_size = 20; 211 break; 212 case CPUMF_CTR_SET_EXT: 213 if (info->csvn == 1) 214 ctrset_size = 32; 215 else if (info->csvn == 2) 216 ctrset_size = 48; 217 else if (info->csvn >= 3 && info->csvn <= 5) 218 ctrset_size = 128; 219 else if (info->csvn == 6 || info->csvn == 7) 220 ctrset_size = 160; 221 break; 222 case CPUMF_CTR_SET_MT_DIAG: 223 if (info->csvn > 3) 224 ctrset_size = 48; 225 break; 226 case CPUMF_CTR_SET_MAX: 227 break; 228 } 229 230 return ctrset_size; 231 } 232 233 /* Read a counter set. The counter set number determines the counter set and 234 * the CPUM-CF first and second version number determine the number of 235 * available counters in each counter set. 236 * Each counter set starts with header containing the counter set number and 237 * the number of eight byte counters. 238 * 239 * The functions returns the number of bytes occupied by this counter set 240 * including the header. 241 * If there is no counter in the counter set, this counter set is useless and 242 * zero is returned on this case. 243 * 244 * Note that the counter sets may not be enabled or active and the stcctm 245 * instruction might return error 3. Depending on error_ok value this is ok, 246 * for example when called from cpumf_pmu_start() call back function. 247 */ 248 static size_t cfdiag_getctrset(struct cf_ctrset_entry *ctrdata, int ctrset, 249 size_t room, bool error_ok) 250 { 251 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 252 size_t ctrset_size, need = 0; 253 int rc = 3; /* Assume write failure */ 254 255 ctrdata->def = CF_DIAG_CTRSET_DEF; 256 ctrdata->set = ctrset; 257 ctrdata->res1 = 0; 258 ctrset_size = cpum_cf_ctrset_size(ctrset, &cpuhw->info); 259 260 if (ctrset_size) { /* Save data */ 261 need = ctrset_size * sizeof(u64) + sizeof(*ctrdata); 262 if (need <= room) { 263 rc = ctr_stcctm(ctrset, ctrset_size, 264 (u64 *)(ctrdata + 1)); 265 } 266 if (rc != 3 || error_ok) 267 ctrdata->ctr = ctrset_size; 268 else 269 need = 0; 270 } 271 272 debug_sprintf_event(cf_dbg, 3, 273 "%s ctrset %d ctrset_size %zu cfvn %d csvn %d" 274 " need %zd rc %d\n", __func__, ctrset, ctrset_size, 275 cpuhw->info.cfvn, cpuhw->info.csvn, need, rc); 276 return need; 277 } 278 279 static const u64 cpumf_ctr_ctl[CPUMF_CTR_SET_MAX] = { 280 [CPUMF_CTR_SET_BASIC] = 0x02, 281 [CPUMF_CTR_SET_USER] = 0x04, 282 [CPUMF_CTR_SET_CRYPTO] = 0x08, 283 [CPUMF_CTR_SET_EXT] = 0x01, 284 [CPUMF_CTR_SET_MT_DIAG] = 0x20, 285 }; 286 287 /* Read out all counter sets and save them in the provided data buffer. 288 * The last 64 byte host an artificial trailer entry. 289 */ 290 static size_t cfdiag_getctr(void *data, size_t sz, unsigned long auth, 291 bool error_ok) 292 { 293 struct cf_trailer_entry *trailer; 294 size_t offset = 0, done; 295 int i; 296 297 memset(data, 0, sz); 298 sz -= sizeof(*trailer); /* Always room for trailer */ 299 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 300 struct cf_ctrset_entry *ctrdata = data + offset; 301 302 if (!(auth & cpumf_ctr_ctl[i])) 303 continue; /* Counter set not authorized */ 304 305 done = cfdiag_getctrset(ctrdata, i, sz - offset, error_ok); 306 offset += done; 307 } 308 trailer = data + offset; 309 cfdiag_trailer(trailer); 310 return offset + sizeof(*trailer); 311 } 312 313 /* Calculate the difference for each counter in a counter set. */ 314 static void cfdiag_diffctrset(u64 *pstart, u64 *pstop, int counters) 315 { 316 for (; --counters >= 0; ++pstart, ++pstop) 317 if (*pstop >= *pstart) 318 *pstop -= *pstart; 319 else 320 *pstop = *pstart - *pstop + 1; 321 } 322 323 /* Scan the counter sets and calculate the difference of each counter 324 * in each set. The result is the increment of each counter during the 325 * period the counter set has been activated. 326 * 327 * Return true on success. 328 */ 329 static int cfdiag_diffctr(struct cpu_cf_events *cpuhw, unsigned long auth) 330 { 331 struct cf_trailer_entry *trailer_start, *trailer_stop; 332 struct cf_ctrset_entry *ctrstart, *ctrstop; 333 size_t offset = 0; 334 335 auth &= (1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1; 336 do { 337 ctrstart = (struct cf_ctrset_entry *)(cpuhw->start + offset); 338 ctrstop = (struct cf_ctrset_entry *)(cpuhw->stop + offset); 339 340 if (memcmp(ctrstop, ctrstart, sizeof(*ctrstop))) { 341 pr_err_once("cpum_cf_diag counter set compare error " 342 "in set %i\n", ctrstart->set); 343 return 0; 344 } 345 auth &= ~cpumf_ctr_ctl[ctrstart->set]; 346 if (ctrstart->def == CF_DIAG_CTRSET_DEF) { 347 cfdiag_diffctrset((u64 *)(ctrstart + 1), 348 (u64 *)(ctrstop + 1), ctrstart->ctr); 349 offset += ctrstart->ctr * sizeof(u64) + 350 sizeof(*ctrstart); 351 } 352 } while (ctrstart->def && auth); 353 354 /* Save time_stamp from start of event in stop's trailer */ 355 trailer_start = (struct cf_trailer_entry *)(cpuhw->start + offset); 356 trailer_stop = (struct cf_trailer_entry *)(cpuhw->stop + offset); 357 trailer_stop->progusage[0] = trailer_start->timestamp; 358 359 return 1; 360 } 361 362 static enum cpumf_ctr_set get_counter_set(u64 event) 363 { 364 int set = CPUMF_CTR_SET_MAX; 365 366 if (event < 32) 367 set = CPUMF_CTR_SET_BASIC; 368 else if (event < 64) 369 set = CPUMF_CTR_SET_USER; 370 else if (event < 128) 371 set = CPUMF_CTR_SET_CRYPTO; 372 else if (event < 288) 373 set = CPUMF_CTR_SET_EXT; 374 else if (event >= 448 && event < 496) 375 set = CPUMF_CTR_SET_MT_DIAG; 376 377 return set; 378 } 379 380 static int validate_ctr_version(const struct hw_perf_event *hwc, 381 enum cpumf_ctr_set set) 382 { 383 struct cpu_cf_events *cpuhw; 384 int err = 0; 385 u16 mtdiag_ctl; 386 387 cpuhw = &get_cpu_var(cpu_cf_events); 388 389 /* check required version for counter sets */ 390 switch (set) { 391 case CPUMF_CTR_SET_BASIC: 392 case CPUMF_CTR_SET_USER: 393 if (cpuhw->info.cfvn < 1) 394 err = -EOPNOTSUPP; 395 break; 396 case CPUMF_CTR_SET_CRYPTO: 397 if ((cpuhw->info.csvn >= 1 && cpuhw->info.csvn <= 5 && 398 hwc->config > 79) || 399 (cpuhw->info.csvn >= 6 && hwc->config > 83)) 400 err = -EOPNOTSUPP; 401 break; 402 case CPUMF_CTR_SET_EXT: 403 if (cpuhw->info.csvn < 1) 404 err = -EOPNOTSUPP; 405 if ((cpuhw->info.csvn == 1 && hwc->config > 159) || 406 (cpuhw->info.csvn == 2 && hwc->config > 175) || 407 (cpuhw->info.csvn >= 3 && cpuhw->info.csvn <= 5 408 && hwc->config > 255) || 409 (cpuhw->info.csvn >= 6 && hwc->config > 287)) 410 err = -EOPNOTSUPP; 411 break; 412 case CPUMF_CTR_SET_MT_DIAG: 413 if (cpuhw->info.csvn <= 3) 414 err = -EOPNOTSUPP; 415 /* 416 * MT-diagnostic counters are read-only. The counter set 417 * is automatically enabled and activated on all CPUs with 418 * multithreading (SMT). Deactivation of multithreading 419 * also disables the counter set. State changes are ignored 420 * by lcctl(). Because Linux controls SMT enablement through 421 * a kernel parameter only, the counter set is either disabled 422 * or enabled and active. 423 * 424 * Thus, the counters can only be used if SMT is on and the 425 * counter set is enabled and active. 426 */ 427 mtdiag_ctl = cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG]; 428 if (!((cpuhw->info.auth_ctl & mtdiag_ctl) && 429 (cpuhw->info.enable_ctl & mtdiag_ctl) && 430 (cpuhw->info.act_ctl & mtdiag_ctl))) 431 err = -EOPNOTSUPP; 432 break; 433 case CPUMF_CTR_SET_MAX: 434 err = -EOPNOTSUPP; 435 } 436 437 put_cpu_var(cpu_cf_events); 438 return err; 439 } 440 441 static int validate_ctr_auth(const struct hw_perf_event *hwc) 442 { 443 struct cpu_cf_events *cpuhw; 444 int err = 0; 445 446 cpuhw = &get_cpu_var(cpu_cf_events); 447 448 /* Check authorization for cpu counter sets. 449 * If the particular CPU counter set is not authorized, 450 * return with -ENOENT in order to fall back to other 451 * PMUs that might suffice the event request. 452 */ 453 if (!(hwc->config_base & cpuhw->info.auth_ctl)) 454 err = -ENOENT; 455 456 put_cpu_var(cpu_cf_events); 457 return err; 458 } 459 460 /* 461 * Change the CPUMF state to active. 462 * Enable and activate the CPU-counter sets according 463 * to the per-cpu control state. 464 */ 465 static void cpumf_pmu_enable(struct pmu *pmu) 466 { 467 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 468 int err; 469 470 if (cpuhw->flags & PMU_F_ENABLED) 471 return; 472 473 err = lcctl(cpuhw->state | cpuhw->dev_state); 474 if (err) { 475 pr_err("Enabling the performance measuring unit " 476 "failed with rc=%x\n", err); 477 return; 478 } 479 480 cpuhw->flags |= PMU_F_ENABLED; 481 } 482 483 /* 484 * Change the CPUMF state to inactive. 485 * Disable and enable (inactive) the CPU-counter sets according 486 * to the per-cpu control state. 487 */ 488 static void cpumf_pmu_disable(struct pmu *pmu) 489 { 490 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 491 int err; 492 u64 inactive; 493 494 if (!(cpuhw->flags & PMU_F_ENABLED)) 495 return; 496 497 inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1); 498 inactive |= cpuhw->dev_state; 499 err = lcctl(inactive); 500 if (err) { 501 pr_err("Disabling the performance measuring unit " 502 "failed with rc=%x\n", err); 503 return; 504 } 505 506 cpuhw->flags &= ~PMU_F_ENABLED; 507 } 508 509 #define PMC_INIT 0UL 510 #define PMC_RELEASE 1UL 511 512 static void cpum_cf_setup_cpu(void *flags) 513 { 514 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 515 516 switch ((unsigned long)flags) { 517 case PMC_INIT: 518 memset(&cpuhw->info, 0, sizeof(cpuhw->info)); 519 qctri(&cpuhw->info); 520 cpuhw->flags |= PMU_F_RESERVED; 521 break; 522 523 case PMC_RELEASE: 524 cpuhw->flags &= ~PMU_F_RESERVED; 525 break; 526 } 527 528 /* Disable CPU counter sets */ 529 lcctl(0); 530 debug_sprintf_event(cf_dbg, 5, "%s flags %#x flags %#x state %#llx\n", 531 __func__, *(int *)flags, cpuhw->flags, 532 cpuhw->state); 533 } 534 535 /* Initialize the CPU-measurement counter facility */ 536 static int __kernel_cpumcf_begin(void) 537 { 538 on_each_cpu(cpum_cf_setup_cpu, (void *)PMC_INIT, 1); 539 irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); 540 541 return 0; 542 } 543 544 /* Release the CPU-measurement counter facility */ 545 static void __kernel_cpumcf_end(void) 546 { 547 on_each_cpu(cpum_cf_setup_cpu, (void *)PMC_RELEASE, 1); 548 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); 549 } 550 551 /* Number of perf events counting hardware events */ 552 static atomic_t num_events = ATOMIC_INIT(0); 553 /* Used to avoid races in calling reserve/release_cpumf_hardware */ 554 static DEFINE_MUTEX(pmc_reserve_mutex); 555 556 /* Release the PMU if event is the last perf event */ 557 static void hw_perf_event_destroy(struct perf_event *event) 558 { 559 mutex_lock(&pmc_reserve_mutex); 560 if (atomic_dec_return(&num_events) == 0) 561 __kernel_cpumcf_end(); 562 mutex_unlock(&pmc_reserve_mutex); 563 } 564 565 /* CPUMF <-> perf event mappings for kernel+userspace (basic set) */ 566 static const int cpumf_generic_events_basic[] = { 567 [PERF_COUNT_HW_CPU_CYCLES] = 0, 568 [PERF_COUNT_HW_INSTRUCTIONS] = 1, 569 [PERF_COUNT_HW_CACHE_REFERENCES] = -1, 570 [PERF_COUNT_HW_CACHE_MISSES] = -1, 571 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1, 572 [PERF_COUNT_HW_BRANCH_MISSES] = -1, 573 [PERF_COUNT_HW_BUS_CYCLES] = -1, 574 }; 575 /* CPUMF <-> perf event mappings for userspace (problem-state set) */ 576 static const int cpumf_generic_events_user[] = { 577 [PERF_COUNT_HW_CPU_CYCLES] = 32, 578 [PERF_COUNT_HW_INSTRUCTIONS] = 33, 579 [PERF_COUNT_HW_CACHE_REFERENCES] = -1, 580 [PERF_COUNT_HW_CACHE_MISSES] = -1, 581 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1, 582 [PERF_COUNT_HW_BRANCH_MISSES] = -1, 583 [PERF_COUNT_HW_BUS_CYCLES] = -1, 584 }; 585 586 static void cpumf_hw_inuse(void) 587 { 588 mutex_lock(&pmc_reserve_mutex); 589 if (atomic_inc_return(&num_events) == 1) 590 __kernel_cpumcf_begin(); 591 mutex_unlock(&pmc_reserve_mutex); 592 } 593 594 static int is_userspace_event(u64 ev) 595 { 596 return cpumf_generic_events_user[PERF_COUNT_HW_CPU_CYCLES] == ev || 597 cpumf_generic_events_user[PERF_COUNT_HW_INSTRUCTIONS] == ev; 598 } 599 600 static int __hw_perf_event_init(struct perf_event *event, unsigned int type) 601 { 602 struct perf_event_attr *attr = &event->attr; 603 struct hw_perf_event *hwc = &event->hw; 604 enum cpumf_ctr_set set; 605 int err = 0; 606 u64 ev; 607 608 switch (type) { 609 case PERF_TYPE_RAW: 610 /* Raw events are used to access counters directly, 611 * hence do not permit excludes */ 612 if (attr->exclude_kernel || attr->exclude_user || 613 attr->exclude_hv) 614 return -EOPNOTSUPP; 615 ev = attr->config; 616 break; 617 618 case PERF_TYPE_HARDWARE: 619 if (is_sampling_event(event)) /* No sampling support */ 620 return -ENOENT; 621 ev = attr->config; 622 if (!attr->exclude_user && attr->exclude_kernel) { 623 /* 624 * Count user space (problem-state) only 625 * Handle events 32 and 33 as 0:u and 1:u 626 */ 627 if (!is_userspace_event(ev)) { 628 if (ev >= ARRAY_SIZE(cpumf_generic_events_user)) 629 return -EOPNOTSUPP; 630 ev = cpumf_generic_events_user[ev]; 631 } 632 } else if (!attr->exclude_kernel && attr->exclude_user) { 633 /* No support for kernel space counters only */ 634 return -EOPNOTSUPP; 635 } else { 636 /* Count user and kernel space, incl. events 32 + 33 */ 637 if (!is_userspace_event(ev)) { 638 if (ev >= ARRAY_SIZE(cpumf_generic_events_basic)) 639 return -EOPNOTSUPP; 640 ev = cpumf_generic_events_basic[ev]; 641 } 642 } 643 break; 644 645 default: 646 return -ENOENT; 647 } 648 649 if (ev == -1) 650 return -ENOENT; 651 652 if (ev > PERF_CPUM_CF_MAX_CTR) 653 return -ENOENT; 654 655 /* Obtain the counter set to which the specified counter belongs */ 656 set = get_counter_set(ev); 657 switch (set) { 658 case CPUMF_CTR_SET_BASIC: 659 case CPUMF_CTR_SET_USER: 660 case CPUMF_CTR_SET_CRYPTO: 661 case CPUMF_CTR_SET_EXT: 662 case CPUMF_CTR_SET_MT_DIAG: 663 /* 664 * Use the hardware perf event structure to store the 665 * counter number in the 'config' member and the counter 666 * set number in the 'config_base' as bit mask. 667 * It is later used to enable/disable the counter(s). 668 */ 669 hwc->config = ev; 670 hwc->config_base = cpumf_ctr_ctl[set]; 671 break; 672 case CPUMF_CTR_SET_MAX: 673 /* The counter could not be associated to a counter set */ 674 return -EINVAL; 675 } 676 677 /* Initialize for using the CPU-measurement counter facility */ 678 cpumf_hw_inuse(); 679 event->destroy = hw_perf_event_destroy; 680 681 /* Finally, validate version and authorization of the counter set */ 682 err = validate_ctr_auth(hwc); 683 if (!err) 684 err = validate_ctr_version(hwc, set); 685 686 return err; 687 } 688 689 /* Events CPU_CYLCES and INSTRUCTIONS can be submitted with two different 690 * attribute::type values: 691 * - PERF_TYPE_HARDWARE: 692 * - pmu->type: 693 * Handle both type of invocations identical. They address the same hardware. 694 * The result is different when event modifiers exclude_kernel and/or 695 * exclude_user are also set. 696 */ 697 static int cpumf_pmu_event_type(struct perf_event *event) 698 { 699 u64 ev = event->attr.config; 700 701 if (cpumf_generic_events_basic[PERF_COUNT_HW_CPU_CYCLES] == ev || 702 cpumf_generic_events_basic[PERF_COUNT_HW_INSTRUCTIONS] == ev || 703 cpumf_generic_events_user[PERF_COUNT_HW_CPU_CYCLES] == ev || 704 cpumf_generic_events_user[PERF_COUNT_HW_INSTRUCTIONS] == ev) 705 return PERF_TYPE_HARDWARE; 706 return PERF_TYPE_RAW; 707 } 708 709 static int cpumf_pmu_event_init(struct perf_event *event) 710 { 711 unsigned int type = event->attr.type; 712 int err; 713 714 if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_RAW) 715 err = __hw_perf_event_init(event, type); 716 else if (event->pmu->type == type) 717 /* Registered as unknown PMU */ 718 err = __hw_perf_event_init(event, cpumf_pmu_event_type(event)); 719 else 720 return -ENOENT; 721 722 if (unlikely(err) && event->destroy) 723 event->destroy(event); 724 725 return err; 726 } 727 728 static int hw_perf_event_reset(struct perf_event *event) 729 { 730 u64 prev, new; 731 int err; 732 733 do { 734 prev = local64_read(&event->hw.prev_count); 735 err = ecctr(event->hw.config, &new); 736 if (err) { 737 if (err != 3) 738 break; 739 /* The counter is not (yet) available. This 740 * might happen if the counter set to which 741 * this counter belongs is in the disabled 742 * state. 743 */ 744 new = 0; 745 } 746 } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev); 747 748 return err; 749 } 750 751 static void hw_perf_event_update(struct perf_event *event) 752 { 753 u64 prev, new, delta; 754 int err; 755 756 do { 757 prev = local64_read(&event->hw.prev_count); 758 err = ecctr(event->hw.config, &new); 759 if (err) 760 return; 761 } while (local64_cmpxchg(&event->hw.prev_count, prev, new) != prev); 762 763 delta = (prev <= new) ? new - prev 764 : (-1ULL - prev) + new + 1; /* overflow */ 765 local64_add(delta, &event->count); 766 } 767 768 static void cpumf_pmu_read(struct perf_event *event) 769 { 770 if (event->hw.state & PERF_HES_STOPPED) 771 return; 772 773 hw_perf_event_update(event); 774 } 775 776 static void cpumf_pmu_start(struct perf_event *event, int flags) 777 { 778 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 779 struct hw_perf_event *hwc = &event->hw; 780 int i; 781 782 if (!(hwc->state & PERF_HES_STOPPED)) 783 return; 784 785 hwc->state = 0; 786 787 /* (Re-)enable and activate the counter set */ 788 ctr_set_enable(&cpuhw->state, hwc->config_base); 789 ctr_set_start(&cpuhw->state, hwc->config_base); 790 791 /* The counter set to which this counter belongs can be already active. 792 * Because all counters in a set are active, the event->hw.prev_count 793 * needs to be synchronized. At this point, the counter set can be in 794 * the inactive or disabled state. 795 */ 796 if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) { 797 cpuhw->usedss = cfdiag_getctr(cpuhw->start, 798 sizeof(cpuhw->start), 799 hwc->config_base, true); 800 } else { 801 hw_perf_event_reset(event); 802 } 803 804 /* Increment refcount for counter sets */ 805 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) 806 if ((hwc->config_base & cpumf_ctr_ctl[i])) 807 atomic_inc(&cpuhw->ctr_set[i]); 808 } 809 810 /* Create perf event sample with the counter sets as raw data. The sample 811 * is then pushed to the event subsystem and the function checks for 812 * possible event overflows. If an event overflow occurs, the PMU is 813 * stopped. 814 * 815 * Return non-zero if an event overflow occurred. 816 */ 817 static int cfdiag_push_sample(struct perf_event *event, 818 struct cpu_cf_events *cpuhw) 819 { 820 struct perf_sample_data data; 821 struct perf_raw_record raw; 822 struct pt_regs regs; 823 int overflow; 824 825 /* Setup perf sample */ 826 perf_sample_data_init(&data, 0, event->hw.last_period); 827 memset(®s, 0, sizeof(regs)); 828 memset(&raw, 0, sizeof(raw)); 829 830 if (event->attr.sample_type & PERF_SAMPLE_CPU) 831 data.cpu_entry.cpu = event->cpu; 832 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 833 raw.frag.size = cpuhw->usedss; 834 raw.frag.data = cpuhw->stop; 835 perf_sample_save_raw_data(&data, &raw); 836 } 837 838 overflow = perf_event_overflow(event, &data, ®s); 839 debug_sprintf_event(cf_dbg, 3, 840 "%s event %#llx sample_type %#llx raw %d ov %d\n", 841 __func__, event->hw.config, 842 event->attr.sample_type, raw.size, overflow); 843 if (overflow) 844 event->pmu->stop(event, 0); 845 846 perf_event_update_userpage(event); 847 return overflow; 848 } 849 850 static void cpumf_pmu_stop(struct perf_event *event, int flags) 851 { 852 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 853 struct hw_perf_event *hwc = &event->hw; 854 int i; 855 856 if (!(hwc->state & PERF_HES_STOPPED)) { 857 /* Decrement reference count for this counter set and if this 858 * is the last used counter in the set, clear activation 859 * control and set the counter set state to inactive. 860 */ 861 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 862 if (!(hwc->config_base & cpumf_ctr_ctl[i])) 863 continue; 864 if (!atomic_dec_return(&cpuhw->ctr_set[i])) 865 ctr_set_stop(&cpuhw->state, cpumf_ctr_ctl[i]); 866 } 867 hwc->state |= PERF_HES_STOPPED; 868 } 869 870 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 871 if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) { 872 local64_inc(&event->count); 873 cpuhw->usedss = cfdiag_getctr(cpuhw->stop, 874 sizeof(cpuhw->stop), 875 event->hw.config_base, 876 false); 877 if (cfdiag_diffctr(cpuhw, event->hw.config_base)) 878 cfdiag_push_sample(event, cpuhw); 879 } else if (cpuhw->flags & PMU_F_RESERVED) { 880 /* Only update when PMU not hotplugged off */ 881 hw_perf_event_update(event); 882 } 883 hwc->state |= PERF_HES_UPTODATE; 884 } 885 } 886 887 static int cpumf_pmu_add(struct perf_event *event, int flags) 888 { 889 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 890 891 ctr_set_enable(&cpuhw->state, event->hw.config_base); 892 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 893 894 if (flags & PERF_EF_START) 895 cpumf_pmu_start(event, PERF_EF_RELOAD); 896 897 return 0; 898 } 899 900 static void cpumf_pmu_del(struct perf_event *event, int flags) 901 { 902 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 903 int i; 904 905 cpumf_pmu_stop(event, PERF_EF_UPDATE); 906 907 /* Check if any counter in the counter set is still used. If not used, 908 * change the counter set to the disabled state. This also clears the 909 * content of all counters in the set. 910 * 911 * When a new perf event has been added but not yet started, this can 912 * clear enable control and resets all counters in a set. Therefore, 913 * cpumf_pmu_start() always has to reenable a counter set. 914 */ 915 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) 916 if (!atomic_read(&cpuhw->ctr_set[i])) 917 ctr_set_disable(&cpuhw->state, cpumf_ctr_ctl[i]); 918 } 919 920 /* Performance monitoring unit for s390x */ 921 static struct pmu cpumf_pmu = { 922 .task_ctx_nr = perf_sw_context, 923 .capabilities = PERF_PMU_CAP_NO_INTERRUPT, 924 .pmu_enable = cpumf_pmu_enable, 925 .pmu_disable = cpumf_pmu_disable, 926 .event_init = cpumf_pmu_event_init, 927 .add = cpumf_pmu_add, 928 .del = cpumf_pmu_del, 929 .start = cpumf_pmu_start, 930 .stop = cpumf_pmu_stop, 931 .read = cpumf_pmu_read, 932 }; 933 934 static int cpum_cf_setup(unsigned int cpu, unsigned long flags) 935 { 936 local_irq_disable(); 937 cpum_cf_setup_cpu((void *)flags); 938 local_irq_enable(); 939 return 0; 940 } 941 942 static int cfset_online_cpu(unsigned int cpu); 943 static int cpum_cf_online_cpu(unsigned int cpu) 944 { 945 debug_sprintf_event(cf_dbg, 4, "%s cpu %d in_irq %ld\n", __func__, 946 cpu, in_interrupt()); 947 cpum_cf_setup(cpu, PMC_INIT); 948 return cfset_online_cpu(cpu); 949 } 950 951 static int cfset_offline_cpu(unsigned int cpu); 952 static int cpum_cf_offline_cpu(unsigned int cpu) 953 { 954 debug_sprintf_event(cf_dbg, 4, "%s cpu %d\n", __func__, cpu); 955 cfset_offline_cpu(cpu); 956 return cpum_cf_setup(cpu, PMC_RELEASE); 957 } 958 959 /* Return true if store counter set multiple instruction is available */ 960 static inline int stccm_avail(void) 961 { 962 return test_facility(142); 963 } 964 965 /* CPU-measurement alerts for the counter facility */ 966 static void cpumf_measurement_alert(struct ext_code ext_code, 967 unsigned int alert, unsigned long unused) 968 { 969 struct cpu_cf_events *cpuhw; 970 971 if (!(alert & CPU_MF_INT_CF_MASK)) 972 return; 973 974 inc_irq_stat(IRQEXT_CMC); 975 cpuhw = this_cpu_ptr(&cpu_cf_events); 976 977 /* 978 * Measurement alerts are shared and might happen when the PMU 979 * is not reserved. Ignore these alerts in this case. 980 */ 981 if (!(cpuhw->flags & PMU_F_RESERVED)) 982 return; 983 984 /* counter authorization change alert */ 985 if (alert & CPU_MF_INT_CF_CACA) 986 qctri(&cpuhw->info); 987 988 /* loss of counter data alert */ 989 if (alert & CPU_MF_INT_CF_LCDA) 990 pr_err("CPU[%i] Counter data was lost\n", smp_processor_id()); 991 992 /* loss of MT counter data alert */ 993 if (alert & CPU_MF_INT_CF_MTDA) 994 pr_warn("CPU[%i] MT counter data was lost\n", 995 smp_processor_id()); 996 } 997 998 static int cfset_init(void); 999 static int __init cpumf_pmu_init(void) 1000 { 1001 int rc; 1002 1003 if (!cpum_cf_avail()) 1004 return -ENODEV; 1005 1006 /* 1007 * Clear bit 15 of cr0 to unauthorize problem-state to 1008 * extract measurement counters 1009 */ 1010 ctl_clear_bit(0, 48); 1011 1012 /* register handler for measurement-alert interruptions */ 1013 rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, 1014 cpumf_measurement_alert); 1015 if (rc) { 1016 pr_err("Registering for CPU-measurement alerts failed with rc=%i\n", rc); 1017 return rc; 1018 } 1019 1020 /* Setup s390dbf facility */ 1021 cf_dbg = debug_register(KMSG_COMPONENT, 2, 1, 128); 1022 if (!cf_dbg) { 1023 pr_err("Registration of s390dbf(cpum_cf) failed\n"); 1024 rc = -ENOMEM; 1025 goto out1; 1026 } 1027 debug_register_view(cf_dbg, &debug_sprintf_view); 1028 1029 cpumf_pmu.attr_groups = cpumf_cf_event_group(); 1030 rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", -1); 1031 if (rc) { 1032 pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc); 1033 goto out2; 1034 } else if (stccm_avail()) { /* Setup counter set device */ 1035 cfset_init(); 1036 } 1037 1038 rc = cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE, 1039 "perf/s390/cf:online", 1040 cpum_cf_online_cpu, cpum_cf_offline_cpu); 1041 return rc; 1042 1043 out2: 1044 debug_unregister_view(cf_dbg, &debug_sprintf_view); 1045 debug_unregister(cf_dbg); 1046 out1: 1047 unregister_external_irq(EXT_IRQ_MEASURE_ALERT, cpumf_measurement_alert); 1048 return rc; 1049 } 1050 1051 /* Support for the CPU Measurement Facility counter set extraction using 1052 * device /dev/hwctr. This allows user space programs to extract complete 1053 * counter set via normal file operations. 1054 */ 1055 1056 static atomic_t cfset_opencnt = ATOMIC_INIT(0); /* Access count */ 1057 static DEFINE_MUTEX(cfset_ctrset_mutex);/* Synchronize access to hardware */ 1058 struct cfset_call_on_cpu_parm { /* Parm struct for smp_call_on_cpu */ 1059 unsigned int sets; /* Counter set bit mask */ 1060 atomic_t cpus_ack; /* # CPUs successfully executed func */ 1061 }; 1062 1063 static struct cfset_session { /* CPUs and counter set bit mask */ 1064 struct list_head head; /* Head of list of active processes */ 1065 } cfset_session = { 1066 .head = LIST_HEAD_INIT(cfset_session.head) 1067 }; 1068 1069 struct cfset_request { /* CPUs and counter set bit mask */ 1070 unsigned long ctrset; /* Bit mask of counter set to read */ 1071 cpumask_t mask; /* CPU mask to read from */ 1072 struct list_head node; /* Chain to cfset_session.head */ 1073 }; 1074 1075 static void cfset_session_init(void) 1076 { 1077 INIT_LIST_HEAD(&cfset_session.head); 1078 } 1079 1080 /* Remove current request from global bookkeeping. Maintain a counter set bit 1081 * mask on a per CPU basis. 1082 * Done in process context under mutex protection. 1083 */ 1084 static void cfset_session_del(struct cfset_request *p) 1085 { 1086 list_del(&p->node); 1087 } 1088 1089 /* Add current request to global bookkeeping. Maintain a counter set bit mask 1090 * on a per CPU basis. 1091 * Done in process context under mutex protection. 1092 */ 1093 static void cfset_session_add(struct cfset_request *p) 1094 { 1095 list_add(&p->node, &cfset_session.head); 1096 } 1097 1098 /* The /dev/hwctr device access uses PMU_F_IN_USE to mark the device access 1099 * path is currently used. 1100 * The cpu_cf_events::dev_state is used to denote counter sets in use by this 1101 * interface. It is always or'ed in. If this interface is not active, its 1102 * value is zero and no additional counter sets will be included. 1103 * 1104 * The cpu_cf_events::state is used by the perf_event_open SVC and remains 1105 * unchanged. 1106 * 1107 * perf_pmu_enable() and perf_pmu_enable() and its call backs 1108 * cpumf_pmu_enable() and cpumf_pmu_disable() are called by the 1109 * performance measurement subsystem to enable per process 1110 * CPU Measurement counter facility. 1111 * The XXX_enable() and XXX_disable functions are used to turn off 1112 * x86 performance monitoring interrupt (PMI) during scheduling. 1113 * s390 uses these calls to temporarily stop and resume the active CPU 1114 * counters sets during scheduling. 1115 * 1116 * We do allow concurrent access of perf_event_open() SVC and /dev/hwctr 1117 * device access. The perf_event_open() SVC interface makes a lot of effort 1118 * to only run the counters while the calling process is actively scheduled 1119 * to run. 1120 * When /dev/hwctr interface is also used at the same time, the counter sets 1121 * will keep running, even when the process is scheduled off a CPU. 1122 * However this is not a problem and does not lead to wrong counter values 1123 * for the perf_event_open() SVC. The current counter value will be recorded 1124 * during schedule-in. At schedule-out time the current counter value is 1125 * extracted again and the delta is calculated and added to the event. 1126 */ 1127 /* Stop all counter sets via ioctl interface */ 1128 static void cfset_ioctl_off(void *parm) 1129 { 1130 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 1131 struct cfset_call_on_cpu_parm *p = parm; 1132 int rc; 1133 1134 /* Check if any counter set used by /dev/hwc */ 1135 for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc) 1136 if ((p->sets & cpumf_ctr_ctl[rc])) { 1137 if (!atomic_dec_return(&cpuhw->ctr_set[rc])) { 1138 ctr_set_disable(&cpuhw->dev_state, 1139 cpumf_ctr_ctl[rc]); 1140 ctr_set_stop(&cpuhw->dev_state, 1141 cpumf_ctr_ctl[rc]); 1142 } 1143 } 1144 /* Keep perf_event_open counter sets */ 1145 rc = lcctl(cpuhw->dev_state | cpuhw->state); 1146 if (rc) 1147 pr_err("Counter set stop %#llx of /dev/%s failed rc=%i\n", 1148 cpuhw->state, S390_HWCTR_DEVICE, rc); 1149 if (!cpuhw->dev_state) 1150 cpuhw->flags &= ~PMU_F_IN_USE; 1151 debug_sprintf_event(cf_dbg, 4, "%s rc %d state %#llx dev_state %#llx\n", 1152 __func__, rc, cpuhw->state, cpuhw->dev_state); 1153 } 1154 1155 /* Start counter sets on particular CPU */ 1156 static void cfset_ioctl_on(void *parm) 1157 { 1158 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 1159 struct cfset_call_on_cpu_parm *p = parm; 1160 int rc; 1161 1162 cpuhw->flags |= PMU_F_IN_USE; 1163 ctr_set_enable(&cpuhw->dev_state, p->sets); 1164 ctr_set_start(&cpuhw->dev_state, p->sets); 1165 for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc) 1166 if ((p->sets & cpumf_ctr_ctl[rc])) 1167 atomic_inc(&cpuhw->ctr_set[rc]); 1168 rc = lcctl(cpuhw->dev_state | cpuhw->state); /* Start counter sets */ 1169 if (!rc) 1170 atomic_inc(&p->cpus_ack); 1171 else 1172 pr_err("Counter set start %#llx of /dev/%s failed rc=%i\n", 1173 cpuhw->dev_state | cpuhw->state, S390_HWCTR_DEVICE, rc); 1174 debug_sprintf_event(cf_dbg, 4, "%s rc %d state %#llx dev_state %#llx\n", 1175 __func__, rc, cpuhw->state, cpuhw->dev_state); 1176 } 1177 1178 static void cfset_release_cpu(void *p) 1179 { 1180 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 1181 int rc; 1182 1183 debug_sprintf_event(cf_dbg, 4, "%s state %#llx dev_state %#llx\n", 1184 __func__, cpuhw->state, cpuhw->dev_state); 1185 cpuhw->dev_state = 0; 1186 rc = lcctl(cpuhw->state); /* Keep perf_event_open counter sets */ 1187 if (rc) 1188 pr_err("Counter set release %#llx of /dev/%s failed rc=%i\n", 1189 cpuhw->state, S390_HWCTR_DEVICE, rc); 1190 } 1191 1192 /* This modifies the process CPU mask to adopt it to the currently online 1193 * CPUs. Offline CPUs can not be addresses. This call terminates the access 1194 * and is usually followed by close() or a new iotcl(..., START, ...) which 1195 * creates a new request structure. 1196 */ 1197 static void cfset_all_stop(struct cfset_request *req) 1198 { 1199 struct cfset_call_on_cpu_parm p = { 1200 .sets = req->ctrset, 1201 }; 1202 1203 cpumask_and(&req->mask, &req->mask, cpu_online_mask); 1204 on_each_cpu_mask(&req->mask, cfset_ioctl_off, &p, 1); 1205 } 1206 1207 /* Release function is also called when application gets terminated without 1208 * doing a proper ioctl(..., S390_HWCTR_STOP, ...) command. 1209 */ 1210 static int cfset_release(struct inode *inode, struct file *file) 1211 { 1212 mutex_lock(&cfset_ctrset_mutex); 1213 /* Open followed by close/exit has no private_data */ 1214 if (file->private_data) { 1215 cfset_all_stop(file->private_data); 1216 cfset_session_del(file->private_data); 1217 kfree(file->private_data); 1218 file->private_data = NULL; 1219 } 1220 if (!atomic_dec_return(&cfset_opencnt)) 1221 on_each_cpu(cfset_release_cpu, NULL, 1); 1222 mutex_unlock(&cfset_ctrset_mutex); 1223 1224 hw_perf_event_destroy(NULL); 1225 return 0; 1226 } 1227 1228 static int cfset_open(struct inode *inode, struct file *file) 1229 { 1230 if (!capable(CAP_SYS_ADMIN)) 1231 return -EPERM; 1232 mutex_lock(&cfset_ctrset_mutex); 1233 if (atomic_inc_return(&cfset_opencnt) == 1) 1234 cfset_session_init(); 1235 mutex_unlock(&cfset_ctrset_mutex); 1236 1237 cpumf_hw_inuse(); 1238 file->private_data = NULL; 1239 /* nonseekable_open() never fails */ 1240 return nonseekable_open(inode, file); 1241 } 1242 1243 static int cfset_all_start(struct cfset_request *req) 1244 { 1245 struct cfset_call_on_cpu_parm p = { 1246 .sets = req->ctrset, 1247 .cpus_ack = ATOMIC_INIT(0), 1248 }; 1249 cpumask_var_t mask; 1250 int rc = 0; 1251 1252 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 1253 return -ENOMEM; 1254 cpumask_and(mask, &req->mask, cpu_online_mask); 1255 on_each_cpu_mask(mask, cfset_ioctl_on, &p, 1); 1256 if (atomic_read(&p.cpus_ack) != cpumask_weight(mask)) { 1257 on_each_cpu_mask(mask, cfset_ioctl_off, &p, 1); 1258 rc = -EIO; 1259 debug_sprintf_event(cf_dbg, 4, "%s CPUs missing", __func__); 1260 } 1261 free_cpumask_var(mask); 1262 return rc; 1263 } 1264 1265 /* Return the maximum required space for all possible CPUs in case one 1266 * CPU will be onlined during the START, READ, STOP cycles. 1267 * To find out the size of the counter sets, any one CPU will do. They 1268 * all have the same counter sets. 1269 */ 1270 static size_t cfset_needspace(unsigned int sets) 1271 { 1272 struct cpu_cf_events *cpuhw = get_cpu_ptr(&cpu_cf_events); 1273 size_t bytes = 0; 1274 int i; 1275 1276 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 1277 if (!(sets & cpumf_ctr_ctl[i])) 1278 continue; 1279 bytes += cpum_cf_ctrset_size(i, &cpuhw->info) * sizeof(u64) + 1280 sizeof(((struct s390_ctrset_setdata *)0)->set) + 1281 sizeof(((struct s390_ctrset_setdata *)0)->no_cnts); 1282 } 1283 bytes = sizeof(((struct s390_ctrset_read *)0)->no_cpus) + nr_cpu_ids * 1284 (bytes + sizeof(((struct s390_ctrset_cpudata *)0)->cpu_nr) + 1285 sizeof(((struct s390_ctrset_cpudata *)0)->no_sets)); 1286 put_cpu_ptr(&cpu_cf_events); 1287 return bytes; 1288 } 1289 1290 static int cfset_all_copy(unsigned long arg, cpumask_t *mask) 1291 { 1292 struct s390_ctrset_read __user *ctrset_read; 1293 unsigned int cpu, cpus, rc; 1294 void __user *uptr; 1295 1296 ctrset_read = (struct s390_ctrset_read __user *)arg; 1297 uptr = ctrset_read->data; 1298 for_each_cpu(cpu, mask) { 1299 struct cpu_cf_events *cpuhw = per_cpu_ptr(&cpu_cf_events, cpu); 1300 struct s390_ctrset_cpudata __user *ctrset_cpudata; 1301 1302 ctrset_cpudata = uptr; 1303 rc = put_user(cpu, &ctrset_cpudata->cpu_nr); 1304 rc |= put_user(cpuhw->sets, &ctrset_cpudata->no_sets); 1305 rc |= copy_to_user(ctrset_cpudata->data, cpuhw->data, 1306 cpuhw->used); 1307 if (rc) 1308 return -EFAULT; 1309 uptr += sizeof(struct s390_ctrset_cpudata) + cpuhw->used; 1310 cond_resched(); 1311 } 1312 cpus = cpumask_weight(mask); 1313 if (put_user(cpus, &ctrset_read->no_cpus)) 1314 return -EFAULT; 1315 debug_sprintf_event(cf_dbg, 4, "%s copied %ld\n", __func__, 1316 uptr - (void __user *)ctrset_read->data); 1317 return 0; 1318 } 1319 1320 static size_t cfset_cpuset_read(struct s390_ctrset_setdata *p, int ctrset, 1321 int ctrset_size, size_t room) 1322 { 1323 size_t need = 0; 1324 int rc = -1; 1325 1326 need = sizeof(*p) + sizeof(u64) * ctrset_size; 1327 if (need <= room) { 1328 p->set = cpumf_ctr_ctl[ctrset]; 1329 p->no_cnts = ctrset_size; 1330 rc = ctr_stcctm(ctrset, ctrset_size, (u64 *)p->cv); 1331 if (rc == 3) /* Nothing stored */ 1332 need = 0; 1333 } 1334 return need; 1335 } 1336 1337 /* Read all counter sets. */ 1338 static void cfset_cpu_read(void *parm) 1339 { 1340 struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events); 1341 struct cfset_call_on_cpu_parm *p = parm; 1342 int set, set_size; 1343 size_t space; 1344 1345 /* No data saved yet */ 1346 cpuhw->used = 0; 1347 cpuhw->sets = 0; 1348 memset(cpuhw->data, 0, sizeof(cpuhw->data)); 1349 1350 /* Scan the counter sets */ 1351 for (set = CPUMF_CTR_SET_BASIC; set < CPUMF_CTR_SET_MAX; ++set) { 1352 struct s390_ctrset_setdata *sp = (void *)cpuhw->data + 1353 cpuhw->used; 1354 1355 if (!(p->sets & cpumf_ctr_ctl[set])) 1356 continue; /* Counter set not in list */ 1357 set_size = cpum_cf_ctrset_size(set, &cpuhw->info); 1358 space = sizeof(cpuhw->data) - cpuhw->used; 1359 space = cfset_cpuset_read(sp, set, set_size, space); 1360 if (space) { 1361 cpuhw->used += space; 1362 cpuhw->sets += 1; 1363 } 1364 } 1365 debug_sprintf_event(cf_dbg, 4, "%s sets %d used %zd\n", __func__, 1366 cpuhw->sets, cpuhw->used); 1367 } 1368 1369 static int cfset_all_read(unsigned long arg, struct cfset_request *req) 1370 { 1371 struct cfset_call_on_cpu_parm p; 1372 cpumask_var_t mask; 1373 int rc; 1374 1375 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 1376 return -ENOMEM; 1377 1378 p.sets = req->ctrset; 1379 cpumask_and(mask, &req->mask, cpu_online_mask); 1380 on_each_cpu_mask(mask, cfset_cpu_read, &p, 1); 1381 rc = cfset_all_copy(arg, mask); 1382 free_cpumask_var(mask); 1383 return rc; 1384 } 1385 1386 static long cfset_ioctl_read(unsigned long arg, struct cfset_request *req) 1387 { 1388 struct s390_ctrset_read read; 1389 int ret = -ENODATA; 1390 1391 if (req && req->ctrset) { 1392 if (copy_from_user(&read, (char __user *)arg, sizeof(read))) 1393 return -EFAULT; 1394 ret = cfset_all_read(arg, req); 1395 } 1396 return ret; 1397 } 1398 1399 static long cfset_ioctl_stop(struct file *file) 1400 { 1401 struct cfset_request *req = file->private_data; 1402 int ret = -ENXIO; 1403 1404 if (req) { 1405 cfset_all_stop(req); 1406 cfset_session_del(req); 1407 kfree(req); 1408 file->private_data = NULL; 1409 ret = 0; 1410 } 1411 return ret; 1412 } 1413 1414 static long cfset_ioctl_start(unsigned long arg, struct file *file) 1415 { 1416 struct s390_ctrset_start __user *ustart; 1417 struct s390_ctrset_start start; 1418 struct cfset_request *preq; 1419 void __user *umask; 1420 unsigned int len; 1421 int ret = 0; 1422 size_t need; 1423 1424 if (file->private_data) 1425 return -EBUSY; 1426 ustart = (struct s390_ctrset_start __user *)arg; 1427 if (copy_from_user(&start, ustart, sizeof(start))) 1428 return -EFAULT; 1429 if (start.version != S390_HWCTR_START_VERSION) 1430 return -EINVAL; 1431 if (start.counter_sets & ~(cpumf_ctr_ctl[CPUMF_CTR_SET_BASIC] | 1432 cpumf_ctr_ctl[CPUMF_CTR_SET_USER] | 1433 cpumf_ctr_ctl[CPUMF_CTR_SET_CRYPTO] | 1434 cpumf_ctr_ctl[CPUMF_CTR_SET_EXT] | 1435 cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG])) 1436 return -EINVAL; /* Invalid counter set */ 1437 if (!start.counter_sets) 1438 return -EINVAL; /* No counter set at all? */ 1439 1440 preq = kzalloc(sizeof(*preq), GFP_KERNEL); 1441 if (!preq) 1442 return -ENOMEM; 1443 cpumask_clear(&preq->mask); 1444 len = min_t(u64, start.cpumask_len, cpumask_size()); 1445 umask = (void __user *)start.cpumask; 1446 if (copy_from_user(&preq->mask, umask, len)) { 1447 kfree(preq); 1448 return -EFAULT; 1449 } 1450 if (cpumask_empty(&preq->mask)) { 1451 kfree(preq); 1452 return -EINVAL; 1453 } 1454 need = cfset_needspace(start.counter_sets); 1455 if (put_user(need, &ustart->data_bytes)) { 1456 kfree(preq); 1457 return -EFAULT; 1458 } 1459 preq->ctrset = start.counter_sets; 1460 ret = cfset_all_start(preq); 1461 if (!ret) { 1462 cfset_session_add(preq); 1463 file->private_data = preq; 1464 debug_sprintf_event(cf_dbg, 4, "%s set %#lx need %ld ret %d\n", 1465 __func__, preq->ctrset, need, ret); 1466 } else { 1467 kfree(preq); 1468 } 1469 return ret; 1470 } 1471 1472 /* Entry point to the /dev/hwctr device interface. 1473 * The ioctl system call supports three subcommands: 1474 * S390_HWCTR_START: Start the specified counter sets on a CPU list. The 1475 * counter set keeps running until explicitly stopped. Returns the number 1476 * of bytes needed to store the counter values. If another S390_HWCTR_START 1477 * ioctl subcommand is called without a previous S390_HWCTR_STOP stop 1478 * command on the same file descriptor, -EBUSY is returned. 1479 * S390_HWCTR_READ: Read the counter set values from specified CPU list given 1480 * with the S390_HWCTR_START command. 1481 * S390_HWCTR_STOP: Stops the counter sets on the CPU list given with the 1482 * previous S390_HWCTR_START subcommand. 1483 */ 1484 static long cfset_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 1485 { 1486 int ret; 1487 1488 cpus_read_lock(); 1489 mutex_lock(&cfset_ctrset_mutex); 1490 switch (cmd) { 1491 case S390_HWCTR_START: 1492 ret = cfset_ioctl_start(arg, file); 1493 break; 1494 case S390_HWCTR_STOP: 1495 ret = cfset_ioctl_stop(file); 1496 break; 1497 case S390_HWCTR_READ: 1498 ret = cfset_ioctl_read(arg, file->private_data); 1499 break; 1500 default: 1501 ret = -ENOTTY; 1502 break; 1503 } 1504 mutex_unlock(&cfset_ctrset_mutex); 1505 cpus_read_unlock(); 1506 return ret; 1507 } 1508 1509 static const struct file_operations cfset_fops = { 1510 .owner = THIS_MODULE, 1511 .open = cfset_open, 1512 .release = cfset_release, 1513 .unlocked_ioctl = cfset_ioctl, 1514 .compat_ioctl = cfset_ioctl, 1515 .llseek = no_llseek 1516 }; 1517 1518 static struct miscdevice cfset_dev = { 1519 .name = S390_HWCTR_DEVICE, 1520 .minor = MISC_DYNAMIC_MINOR, 1521 .fops = &cfset_fops, 1522 }; 1523 1524 /* Hotplug add of a CPU. Scan through all active processes and add 1525 * that CPU to the list of CPUs supplied with ioctl(..., START, ...). 1526 */ 1527 static int cfset_online_cpu(unsigned int cpu) 1528 { 1529 struct cfset_call_on_cpu_parm p; 1530 struct cfset_request *rp; 1531 1532 mutex_lock(&cfset_ctrset_mutex); 1533 if (!list_empty(&cfset_session.head)) { 1534 list_for_each_entry(rp, &cfset_session.head, node) { 1535 p.sets = rp->ctrset; 1536 cfset_ioctl_on(&p); 1537 cpumask_set_cpu(cpu, &rp->mask); 1538 } 1539 } 1540 mutex_unlock(&cfset_ctrset_mutex); 1541 return 0; 1542 } 1543 1544 /* Hotplug remove of a CPU. Scan through all active processes and clear 1545 * that CPU from the list of CPUs supplied with ioctl(..., START, ...). 1546 */ 1547 static int cfset_offline_cpu(unsigned int cpu) 1548 { 1549 struct cfset_call_on_cpu_parm p; 1550 struct cfset_request *rp; 1551 1552 mutex_lock(&cfset_ctrset_mutex); 1553 if (!list_empty(&cfset_session.head)) { 1554 list_for_each_entry(rp, &cfset_session.head, node) { 1555 p.sets = rp->ctrset; 1556 cfset_ioctl_off(&p); 1557 cpumask_clear_cpu(cpu, &rp->mask); 1558 } 1559 } 1560 mutex_unlock(&cfset_ctrset_mutex); 1561 return 0; 1562 } 1563 1564 static void cfdiag_read(struct perf_event *event) 1565 { 1566 debug_sprintf_event(cf_dbg, 3, "%s event %#llx count %ld\n", __func__, 1567 event->attr.config, local64_read(&event->count)); 1568 } 1569 1570 static int get_authctrsets(void) 1571 { 1572 struct cpu_cf_events *cpuhw; 1573 unsigned long auth = 0; 1574 enum cpumf_ctr_set i; 1575 1576 cpuhw = &get_cpu_var(cpu_cf_events); 1577 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 1578 if (cpuhw->info.auth_ctl & cpumf_ctr_ctl[i]) 1579 auth |= cpumf_ctr_ctl[i]; 1580 } 1581 put_cpu_var(cpu_cf_events); 1582 return auth; 1583 } 1584 1585 /* Setup the event. Test for authorized counter sets and only include counter 1586 * sets which are authorized at the time of the setup. Including unauthorized 1587 * counter sets result in specification exception (and panic). 1588 */ 1589 static int cfdiag_event_init2(struct perf_event *event) 1590 { 1591 struct perf_event_attr *attr = &event->attr; 1592 int err = 0; 1593 1594 /* Set sample_period to indicate sampling */ 1595 event->hw.config = attr->config; 1596 event->hw.sample_period = attr->sample_period; 1597 local64_set(&event->hw.period_left, event->hw.sample_period); 1598 local64_set(&event->count, 0); 1599 event->hw.last_period = event->hw.sample_period; 1600 1601 /* Add all authorized counter sets to config_base. The 1602 * the hardware init function is either called per-cpu or just once 1603 * for all CPUS (event->cpu == -1). This depends on the whether 1604 * counting is started for all CPUs or on a per workload base where 1605 * the perf event moves from one CPU to another CPU. 1606 * Checking the authorization on any CPU is fine as the hardware 1607 * applies the same authorization settings to all CPUs. 1608 */ 1609 event->hw.config_base = get_authctrsets(); 1610 1611 /* No authorized counter sets, nothing to count/sample */ 1612 if (!event->hw.config_base) 1613 err = -EINVAL; 1614 1615 debug_sprintf_event(cf_dbg, 5, "%s err %d config_base %#lx\n", 1616 __func__, err, event->hw.config_base); 1617 return err; 1618 } 1619 1620 static int cfdiag_event_init(struct perf_event *event) 1621 { 1622 struct perf_event_attr *attr = &event->attr; 1623 int err = -ENOENT; 1624 1625 if (event->attr.config != PERF_EVENT_CPUM_CF_DIAG || 1626 event->attr.type != event->pmu->type) 1627 goto out; 1628 1629 /* Raw events are used to access counters directly, 1630 * hence do not permit excludes. 1631 * This event is useless without PERF_SAMPLE_RAW to return counter set 1632 * values as raw data. 1633 */ 1634 if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv || 1635 !(attr->sample_type & (PERF_SAMPLE_CPU | PERF_SAMPLE_RAW))) { 1636 err = -EOPNOTSUPP; 1637 goto out; 1638 } 1639 1640 /* Initialize for using the CPU-measurement counter facility */ 1641 cpumf_hw_inuse(); 1642 event->destroy = hw_perf_event_destroy; 1643 1644 err = cfdiag_event_init2(event); 1645 if (unlikely(err)) 1646 event->destroy(event); 1647 out: 1648 return err; 1649 } 1650 1651 /* Create cf_diag/events/CF_DIAG event sysfs file. This counter is used 1652 * to collect the complete counter sets for a scheduled process. Target 1653 * are complete counter sets attached as raw data to the artificial event. 1654 * This results in complete counter sets available when a process is 1655 * scheduled. Contains the delta of every counter while the process was 1656 * running. 1657 */ 1658 CPUMF_EVENT_ATTR(CF_DIAG, CF_DIAG, PERF_EVENT_CPUM_CF_DIAG); 1659 1660 static struct attribute *cfdiag_events_attr[] = { 1661 CPUMF_EVENT_PTR(CF_DIAG, CF_DIAG), 1662 NULL, 1663 }; 1664 1665 PMU_FORMAT_ATTR(event, "config:0-63"); 1666 1667 static struct attribute *cfdiag_format_attr[] = { 1668 &format_attr_event.attr, 1669 NULL, 1670 }; 1671 1672 static struct attribute_group cfdiag_events_group = { 1673 .name = "events", 1674 .attrs = cfdiag_events_attr, 1675 }; 1676 static struct attribute_group cfdiag_format_group = { 1677 .name = "format", 1678 .attrs = cfdiag_format_attr, 1679 }; 1680 static const struct attribute_group *cfdiag_attr_groups[] = { 1681 &cfdiag_events_group, 1682 &cfdiag_format_group, 1683 NULL, 1684 }; 1685 1686 /* Performance monitoring unit for event CF_DIAG. Since this event 1687 * is also started and stopped via the perf_event_open() system call, use 1688 * the same event enable/disable call back functions. They do not 1689 * have a pointer to the perf_event strcture as first parameter. 1690 * 1691 * The functions XXX_add, XXX_del, XXX_start and XXX_stop are also common. 1692 * Reuse them and distinguish the event (always first parameter) via 1693 * 'config' member. 1694 */ 1695 static struct pmu cf_diag = { 1696 .task_ctx_nr = perf_sw_context, 1697 .event_init = cfdiag_event_init, 1698 .pmu_enable = cpumf_pmu_enable, 1699 .pmu_disable = cpumf_pmu_disable, 1700 .add = cpumf_pmu_add, 1701 .del = cpumf_pmu_del, 1702 .start = cpumf_pmu_start, 1703 .stop = cpumf_pmu_stop, 1704 .read = cfdiag_read, 1705 1706 .attr_groups = cfdiag_attr_groups 1707 }; 1708 1709 /* Calculate memory needed to store all counter sets together with header and 1710 * trailer data. This is independent of the counter set authorization which 1711 * can vary depending on the configuration. 1712 */ 1713 static size_t cfdiag_maxsize(struct cpumf_ctr_info *info) 1714 { 1715 size_t max_size = sizeof(struct cf_trailer_entry); 1716 enum cpumf_ctr_set i; 1717 1718 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 1719 size_t size = cpum_cf_ctrset_size(i, info); 1720 1721 if (size) 1722 max_size += size * sizeof(u64) + 1723 sizeof(struct cf_ctrset_entry); 1724 } 1725 return max_size; 1726 } 1727 1728 /* Get the CPU speed, try sampling facility first and CPU attributes second. */ 1729 static void cfdiag_get_cpu_speed(void) 1730 { 1731 unsigned long mhz; 1732 1733 if (cpum_sf_avail()) { /* Sampling facility first */ 1734 struct hws_qsi_info_block si; 1735 1736 memset(&si, 0, sizeof(si)); 1737 if (!qsi(&si)) { 1738 cfdiag_cpu_speed = si.cpu_speed; 1739 return; 1740 } 1741 } 1742 1743 /* Fallback: CPU speed extract static part. Used in case 1744 * CPU Measurement Sampling Facility is turned off. 1745 */ 1746 mhz = __ecag(ECAG_CPU_ATTRIBUTE, 0); 1747 if (mhz != -1UL) 1748 cfdiag_cpu_speed = mhz & 0xffffffff; 1749 } 1750 1751 static int cfset_init(void) 1752 { 1753 struct cpumf_ctr_info info; 1754 size_t need; 1755 int rc; 1756 1757 if (qctri(&info)) 1758 return -ENODEV; 1759 1760 cfdiag_get_cpu_speed(); 1761 /* Make sure the counter set data fits into predefined buffer. */ 1762 need = cfdiag_maxsize(&info); 1763 if (need > sizeof(((struct cpu_cf_events *)0)->start)) { 1764 pr_err("Insufficient memory for PMU(cpum_cf_diag) need=%zu\n", 1765 need); 1766 return -ENOMEM; 1767 } 1768 1769 rc = misc_register(&cfset_dev); 1770 if (rc) { 1771 pr_err("Registration of /dev/%s failed rc=%i\n", 1772 cfset_dev.name, rc); 1773 goto out; 1774 } 1775 1776 rc = perf_pmu_register(&cf_diag, "cpum_cf_diag", -1); 1777 if (rc) { 1778 misc_deregister(&cfset_dev); 1779 pr_err("Registration of PMU(cpum_cf_diag) failed with rc=%i\n", 1780 rc); 1781 } 1782 out: 1783 return rc; 1784 } 1785 1786 device_initcall(cpumf_pmu_init); 1787