1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Performance event support for s390x - CPU-measurement Counter Facility 4 * 5 * Copyright IBM Corp. 2012, 2023 6 * Author(s): Hendrik Brueckner <brueckner@linux.ibm.com> 7 * Thomas Richter <tmricht@linux.ibm.com> 8 */ 9 #define KMSG_COMPONENT "cpum_cf" 10 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 11 12 #include <linux/kernel.h> 13 #include <linux/kernel_stat.h> 14 #include <linux/percpu.h> 15 #include <linux/notifier.h> 16 #include <linux/init.h> 17 #include <linux/miscdevice.h> 18 #include <linux/perf_event.h> 19 20 #include <asm/cpu_mf.h> 21 #include <asm/hwctrset.h> 22 #include <asm/debug.h> 23 24 /* Perf PMU definitions for the counter facility */ 25 #define PERF_CPUM_CF_MAX_CTR 0xffffUL /* Max ctr for ECCTR */ 26 #define PERF_EVENT_CPUM_CF_DIAG 0xBC000UL /* Event: Counter sets */ 27 28 enum cpumf_ctr_set { 29 CPUMF_CTR_SET_BASIC = 0, /* Basic Counter Set */ 30 CPUMF_CTR_SET_USER = 1, /* Problem-State Counter Set */ 31 CPUMF_CTR_SET_CRYPTO = 2, /* Crypto-Activity Counter Set */ 32 CPUMF_CTR_SET_EXT = 3, /* Extended Counter Set */ 33 CPUMF_CTR_SET_MT_DIAG = 4, /* MT-diagnostic Counter Set */ 34 35 /* Maximum number of counter sets */ 36 CPUMF_CTR_SET_MAX, 37 }; 38 39 #define CPUMF_LCCTL_ENABLE_SHIFT 16 40 #define CPUMF_LCCTL_ACTCTL_SHIFT 0 41 42 static inline void ctr_set_enable(u64 *state, u64 ctrsets) 43 { 44 *state |= ctrsets << CPUMF_LCCTL_ENABLE_SHIFT; 45 } 46 47 static inline void ctr_set_disable(u64 *state, u64 ctrsets) 48 { 49 *state &= ~(ctrsets << CPUMF_LCCTL_ENABLE_SHIFT); 50 } 51 52 static inline void ctr_set_start(u64 *state, u64 ctrsets) 53 { 54 *state |= ctrsets << CPUMF_LCCTL_ACTCTL_SHIFT; 55 } 56 57 static inline void ctr_set_stop(u64 *state, u64 ctrsets) 58 { 59 *state &= ~(ctrsets << CPUMF_LCCTL_ACTCTL_SHIFT); 60 } 61 62 static inline int ctr_stcctm(enum cpumf_ctr_set set, u64 range, u64 *dest) 63 { 64 switch (set) { 65 case CPUMF_CTR_SET_BASIC: 66 return stcctm(BASIC, range, dest); 67 case CPUMF_CTR_SET_USER: 68 return stcctm(PROBLEM_STATE, range, dest); 69 case CPUMF_CTR_SET_CRYPTO: 70 return stcctm(CRYPTO_ACTIVITY, range, dest); 71 case CPUMF_CTR_SET_EXT: 72 return stcctm(EXTENDED, range, dest); 73 case CPUMF_CTR_SET_MT_DIAG: 74 return stcctm(MT_DIAG_CLEARING, range, dest); 75 case CPUMF_CTR_SET_MAX: 76 return 3; 77 } 78 return 3; 79 } 80 81 struct cpu_cf_events { 82 refcount_t refcnt; /* Reference count */ 83 atomic_t ctr_set[CPUMF_CTR_SET_MAX]; 84 u64 state; /* For perf_event_open SVC */ 85 u64 dev_state; /* For /dev/hwctr */ 86 unsigned int flags; 87 size_t used; /* Bytes used in data */ 88 size_t usedss; /* Bytes used in start/stop */ 89 unsigned char start[PAGE_SIZE]; /* Counter set at event add */ 90 unsigned char stop[PAGE_SIZE]; /* Counter set at event delete */ 91 unsigned char data[PAGE_SIZE]; /* Counter set at /dev/hwctr */ 92 unsigned int sets; /* # Counter set saved in memory */ 93 }; 94 95 static unsigned int cfdiag_cpu_speed; /* CPU speed for CF_DIAG trailer */ 96 static debug_info_t *cf_dbg; 97 98 /* 99 * The CPU Measurement query counter information instruction contains 100 * information which varies per machine generation, but is constant and 101 * does not change when running on a particular machine, such as counter 102 * first and second version number. This is needed to determine the size 103 * of counter sets. Extract this information at device driver initialization. 104 */ 105 static struct cpumf_ctr_info cpumf_ctr_info; 106 107 struct cpu_cf_ptr { 108 struct cpu_cf_events *cpucf; 109 }; 110 111 static struct cpu_cf_root { /* Anchor to per CPU data */ 112 refcount_t refcnt; /* Overall active events */ 113 struct cpu_cf_ptr __percpu *cfptr; 114 } cpu_cf_root; 115 116 /* 117 * Serialize event initialization and event removal. Both are called from 118 * user space in task context with perf_event_open() and close() 119 * system calls. 120 * 121 * This mutex serializes functions cpum_cf_alloc_cpu() called at event 122 * initialization via cpumf_pmu_event_init() and function cpum_cf_free_cpu() 123 * called at event removal via call back function hw_perf_event_destroy() 124 * when the event is deleted. They are serialized to enforce correct 125 * bookkeeping of pointer and reference counts anchored by 126 * struct cpu_cf_root and the access to cpu_cf_root::refcnt and the 127 * per CPU pointers stored in cpu_cf_root::cfptr. 128 */ 129 static DEFINE_MUTEX(pmc_reserve_mutex); 130 131 /* 132 * Get pointer to per-cpu structure. 133 * 134 * Function get_cpu_cfhw() is called from 135 * - cfset_copy_all(): This function is protected by cpus_read_lock(), so 136 * CPU hot plug remove can not happen. Event removal requires a close() 137 * first. 138 * 139 * Function this_cpu_cfhw() is called from perf common code functions: 140 * - pmu_{en|dis}able(), pmu_{add|del}()and pmu_{start|stop}(): 141 * All functions execute with interrupts disabled on that particular CPU. 142 * - cfset_ioctl_{on|off}, cfset_cpu_read(): see comment cfset_copy_all(). 143 * 144 * Therefore it is safe to access the CPU specific pointer to the event. 145 */ 146 static struct cpu_cf_events *get_cpu_cfhw(int cpu) 147 { 148 struct cpu_cf_ptr __percpu *p = cpu_cf_root.cfptr; 149 150 if (p) { 151 struct cpu_cf_ptr *q = per_cpu_ptr(p, cpu); 152 153 return q->cpucf; 154 } 155 return NULL; 156 } 157 158 static struct cpu_cf_events *this_cpu_cfhw(void) 159 { 160 return get_cpu_cfhw(smp_processor_id()); 161 } 162 163 /* Disable counter sets on dedicated CPU */ 164 static void cpum_cf_reset_cpu(void *flags) 165 { 166 lcctl(0); 167 } 168 169 /* Free per CPU data when the last event is removed. */ 170 static void cpum_cf_free_root(void) 171 { 172 if (!refcount_dec_and_test(&cpu_cf_root.refcnt)) 173 return; 174 free_percpu(cpu_cf_root.cfptr); 175 cpu_cf_root.cfptr = NULL; 176 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT); 177 on_each_cpu(cpum_cf_reset_cpu, NULL, 1); 178 debug_sprintf_event(cf_dbg, 4, "%s root.refcnt %u cfptr %d\n", 179 __func__, refcount_read(&cpu_cf_root.refcnt), 180 !cpu_cf_root.cfptr); 181 } 182 183 /* 184 * On initialization of first event also allocate per CPU data dynamically. 185 * Start with an array of pointers, the array size is the maximum number of 186 * CPUs possible, which might be larger than the number of CPUs currently 187 * online. 188 */ 189 static int cpum_cf_alloc_root(void) 190 { 191 int rc = 0; 192 193 if (refcount_inc_not_zero(&cpu_cf_root.refcnt)) 194 return rc; 195 196 /* The memory is already zeroed. */ 197 cpu_cf_root.cfptr = alloc_percpu(struct cpu_cf_ptr); 198 if (cpu_cf_root.cfptr) { 199 refcount_set(&cpu_cf_root.refcnt, 1); 200 on_each_cpu(cpum_cf_reset_cpu, NULL, 1); 201 irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT); 202 } else { 203 rc = -ENOMEM; 204 } 205 206 return rc; 207 } 208 209 /* Free CPU counter data structure for a PMU */ 210 static void cpum_cf_free_cpu(int cpu) 211 { 212 struct cpu_cf_events *cpuhw; 213 struct cpu_cf_ptr *p; 214 215 mutex_lock(&pmc_reserve_mutex); 216 /* 217 * When invoked via CPU hotplug handler, there might be no events 218 * installed or that particular CPU might not have an 219 * event installed. This anchor pointer can be NULL! 220 */ 221 if (!cpu_cf_root.cfptr) 222 goto out; 223 p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); 224 cpuhw = p->cpucf; 225 /* 226 * Might be zero when called from CPU hotplug handler and no event 227 * installed on that CPU, but on different CPUs. 228 */ 229 if (!cpuhw) 230 goto out; 231 232 if (refcount_dec_and_test(&cpuhw->refcnt)) { 233 kfree(cpuhw); 234 p->cpucf = NULL; 235 } 236 cpum_cf_free_root(); 237 out: 238 mutex_unlock(&pmc_reserve_mutex); 239 } 240 241 /* Allocate CPU counter data structure for a PMU. Called under mutex lock. */ 242 static int cpum_cf_alloc_cpu(int cpu) 243 { 244 struct cpu_cf_events *cpuhw; 245 struct cpu_cf_ptr *p; 246 int rc; 247 248 mutex_lock(&pmc_reserve_mutex); 249 rc = cpum_cf_alloc_root(); 250 if (rc) 251 goto unlock; 252 p = per_cpu_ptr(cpu_cf_root.cfptr, cpu); 253 cpuhw = p->cpucf; 254 255 if (!cpuhw) { 256 cpuhw = kzalloc(sizeof(*cpuhw), GFP_KERNEL); 257 if (cpuhw) { 258 p->cpucf = cpuhw; 259 refcount_set(&cpuhw->refcnt, 1); 260 } else { 261 rc = -ENOMEM; 262 } 263 } else { 264 refcount_inc(&cpuhw->refcnt); 265 } 266 if (rc) { 267 /* 268 * Error in allocation of event, decrement anchor. Since 269 * cpu_cf_event in not created, its destroy() function is not 270 * invoked. Adjust the reference counter for the anchor. 271 */ 272 cpum_cf_free_root(); 273 } 274 unlock: 275 mutex_unlock(&pmc_reserve_mutex); 276 return rc; 277 } 278 279 /* 280 * Create/delete per CPU data structures for /dev/hwctr interface and events 281 * created by perf_event_open(). 282 * If cpu is -1, track task on all available CPUs. This requires 283 * allocation of hardware data structures for all CPUs. This setup handles 284 * perf_event_open() with task context and /dev/hwctr interface. 285 * If cpu is non-zero install event on this CPU only. This setup handles 286 * perf_event_open() with CPU context. 287 */ 288 static int cpum_cf_alloc(int cpu) 289 { 290 cpumask_var_t mask; 291 int rc; 292 293 if (cpu == -1) { 294 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) 295 return -ENOMEM; 296 for_each_online_cpu(cpu) { 297 rc = cpum_cf_alloc_cpu(cpu); 298 if (rc) { 299 for_each_cpu(cpu, mask) 300 cpum_cf_free_cpu(cpu); 301 break; 302 } 303 cpumask_set_cpu(cpu, mask); 304 } 305 free_cpumask_var(mask); 306 } else { 307 rc = cpum_cf_alloc_cpu(cpu); 308 } 309 return rc; 310 } 311 312 static void cpum_cf_free(int cpu) 313 { 314 if (cpu == -1) { 315 for_each_online_cpu(cpu) 316 cpum_cf_free_cpu(cpu); 317 } else { 318 cpum_cf_free_cpu(cpu); 319 } 320 } 321 322 #define CF_DIAG_CTRSET_DEF 0xfeef /* Counter set header mark */ 323 /* interval in seconds */ 324 325 /* Counter sets are stored as data stream in a page sized memory buffer and 326 * exported to user space via raw data attached to the event sample data. 327 * Each counter set starts with an eight byte header consisting of: 328 * - a two byte eye catcher (0xfeef) 329 * - a one byte counter set number 330 * - a two byte counter set size (indicates the number of counters in this set) 331 * - a three byte reserved value (must be zero) to make the header the same 332 * size as a counter value. 333 * All counter values are eight byte in size. 334 * 335 * All counter sets are followed by a 64 byte trailer. 336 * The trailer consists of a: 337 * - flag field indicating valid fields when corresponding bit set 338 * - the counter facility first and second version number 339 * - the CPU speed if nonzero 340 * - the time stamp the counter sets have been collected 341 * - the time of day (TOD) base value 342 * - the machine type. 343 * 344 * The counter sets are saved when the process is prepared to be executed on a 345 * CPU and saved again when the process is going to be removed from a CPU. 346 * The difference of both counter sets are calculated and stored in the event 347 * sample data area. 348 */ 349 struct cf_ctrset_entry { /* CPU-M CF counter set entry (8 byte) */ 350 unsigned int def:16; /* 0-15 Data Entry Format */ 351 unsigned int set:16; /* 16-31 Counter set identifier */ 352 unsigned int ctr:16; /* 32-47 Number of stored counters */ 353 unsigned int res1:16; /* 48-63 Reserved */ 354 }; 355 356 struct cf_trailer_entry { /* CPU-M CF_DIAG trailer (64 byte) */ 357 /* 0 - 7 */ 358 union { 359 struct { 360 unsigned int clock_base:1; /* TOD clock base set */ 361 unsigned int speed:1; /* CPU speed set */ 362 /* Measurement alerts */ 363 unsigned int mtda:1; /* Loss of MT ctr. data alert */ 364 unsigned int caca:1; /* Counter auth. change alert */ 365 unsigned int lcda:1; /* Loss of counter data alert */ 366 }; 367 unsigned long flags; /* 0-63 All indicators */ 368 }; 369 /* 8 - 15 */ 370 unsigned int cfvn:16; /* 64-79 Ctr First Version */ 371 unsigned int csvn:16; /* 80-95 Ctr Second Version */ 372 unsigned int cpu_speed:32; /* 96-127 CPU speed */ 373 /* 16 - 23 */ 374 unsigned long timestamp; /* 128-191 Timestamp (TOD) */ 375 /* 24 - 55 */ 376 union { 377 struct { 378 unsigned long progusage1; 379 unsigned long progusage2; 380 unsigned long progusage3; 381 unsigned long tod_base; 382 }; 383 unsigned long progusage[4]; 384 }; 385 /* 56 - 63 */ 386 unsigned int mach_type:16; /* Machine type */ 387 unsigned int res1:16; /* Reserved */ 388 unsigned int res2:32; /* Reserved */ 389 }; 390 391 /* Create the trailer data at the end of a page. */ 392 static void cfdiag_trailer(struct cf_trailer_entry *te) 393 { 394 struct cpuid cpuid; 395 396 te->cfvn = cpumf_ctr_info.cfvn; /* Counter version numbers */ 397 te->csvn = cpumf_ctr_info.csvn; 398 399 get_cpu_id(&cpuid); /* Machine type */ 400 te->mach_type = cpuid.machine; 401 te->cpu_speed = cfdiag_cpu_speed; 402 if (te->cpu_speed) 403 te->speed = 1; 404 te->clock_base = 1; /* Save clock base */ 405 te->tod_base = tod_clock_base.tod; 406 te->timestamp = get_tod_clock_fast(); 407 } 408 409 /* 410 * The number of counters per counter set varies between machine generations, 411 * but is constant when running on a particular machine generation. 412 * Determine each counter set size at device driver initialization and 413 * retrieve it later. 414 */ 415 static size_t cpumf_ctr_setsizes[CPUMF_CTR_SET_MAX]; 416 static void cpum_cf_make_setsize(enum cpumf_ctr_set ctrset) 417 { 418 size_t ctrset_size = 0; 419 420 switch (ctrset) { 421 case CPUMF_CTR_SET_BASIC: 422 if (cpumf_ctr_info.cfvn >= 1) 423 ctrset_size = 6; 424 break; 425 case CPUMF_CTR_SET_USER: 426 if (cpumf_ctr_info.cfvn == 1) 427 ctrset_size = 6; 428 else if (cpumf_ctr_info.cfvn >= 3) 429 ctrset_size = 2; 430 break; 431 case CPUMF_CTR_SET_CRYPTO: 432 if (cpumf_ctr_info.csvn >= 1 && cpumf_ctr_info.csvn <= 5) 433 ctrset_size = 16; 434 else if (cpumf_ctr_info.csvn >= 6) 435 ctrset_size = 20; 436 break; 437 case CPUMF_CTR_SET_EXT: 438 if (cpumf_ctr_info.csvn == 1) 439 ctrset_size = 32; 440 else if (cpumf_ctr_info.csvn == 2) 441 ctrset_size = 48; 442 else if (cpumf_ctr_info.csvn >= 3 && cpumf_ctr_info.csvn <= 5) 443 ctrset_size = 128; 444 else if (cpumf_ctr_info.csvn >= 6 && cpumf_ctr_info.csvn <= 8) 445 ctrset_size = 160; 446 break; 447 case CPUMF_CTR_SET_MT_DIAG: 448 if (cpumf_ctr_info.csvn > 3) 449 ctrset_size = 48; 450 break; 451 case CPUMF_CTR_SET_MAX: 452 break; 453 } 454 cpumf_ctr_setsizes[ctrset] = ctrset_size; 455 } 456 457 /* 458 * Return the maximum possible counter set size (in number of 8 byte counters) 459 * depending on type and model number. 460 */ 461 static size_t cpum_cf_read_setsize(enum cpumf_ctr_set ctrset) 462 { 463 return cpumf_ctr_setsizes[ctrset]; 464 } 465 466 /* Read a counter set. The counter set number determines the counter set and 467 * the CPUM-CF first and second version number determine the number of 468 * available counters in each counter set. 469 * Each counter set starts with header containing the counter set number and 470 * the number of eight byte counters. 471 * 472 * The functions returns the number of bytes occupied by this counter set 473 * including the header. 474 * If there is no counter in the counter set, this counter set is useless and 475 * zero is returned on this case. 476 * 477 * Note that the counter sets may not be enabled or active and the stcctm 478 * instruction might return error 3. Depending on error_ok value this is ok, 479 * for example when called from cpumf_pmu_start() call back function. 480 */ 481 static size_t cfdiag_getctrset(struct cf_ctrset_entry *ctrdata, int ctrset, 482 size_t room, bool error_ok) 483 { 484 size_t ctrset_size, need = 0; 485 int rc = 3; /* Assume write failure */ 486 487 ctrdata->def = CF_DIAG_CTRSET_DEF; 488 ctrdata->set = ctrset; 489 ctrdata->res1 = 0; 490 ctrset_size = cpum_cf_read_setsize(ctrset); 491 492 if (ctrset_size) { /* Save data */ 493 need = ctrset_size * sizeof(u64) + sizeof(*ctrdata); 494 if (need <= room) { 495 rc = ctr_stcctm(ctrset, ctrset_size, 496 (u64 *)(ctrdata + 1)); 497 } 498 if (rc != 3 || error_ok) 499 ctrdata->ctr = ctrset_size; 500 else 501 need = 0; 502 } 503 504 return need; 505 } 506 507 static const u64 cpumf_ctr_ctl[CPUMF_CTR_SET_MAX] = { 508 [CPUMF_CTR_SET_BASIC] = 0x02, 509 [CPUMF_CTR_SET_USER] = 0x04, 510 [CPUMF_CTR_SET_CRYPTO] = 0x08, 511 [CPUMF_CTR_SET_EXT] = 0x01, 512 [CPUMF_CTR_SET_MT_DIAG] = 0x20, 513 }; 514 515 /* Read out all counter sets and save them in the provided data buffer. 516 * The last 64 byte host an artificial trailer entry. 517 */ 518 static size_t cfdiag_getctr(void *data, size_t sz, unsigned long auth, 519 bool error_ok) 520 { 521 struct cf_trailer_entry *trailer; 522 size_t offset = 0, done; 523 int i; 524 525 memset(data, 0, sz); 526 sz -= sizeof(*trailer); /* Always room for trailer */ 527 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 528 struct cf_ctrset_entry *ctrdata = data + offset; 529 530 if (!(auth & cpumf_ctr_ctl[i])) 531 continue; /* Counter set not authorized */ 532 533 done = cfdiag_getctrset(ctrdata, i, sz - offset, error_ok); 534 offset += done; 535 } 536 trailer = data + offset; 537 cfdiag_trailer(trailer); 538 return offset + sizeof(*trailer); 539 } 540 541 /* Calculate the difference for each counter in a counter set. */ 542 static void cfdiag_diffctrset(u64 *pstart, u64 *pstop, int counters) 543 { 544 for (; --counters >= 0; ++pstart, ++pstop) 545 if (*pstop >= *pstart) 546 *pstop -= *pstart; 547 else 548 *pstop = *pstart - *pstop + 1; 549 } 550 551 /* Scan the counter sets and calculate the difference of each counter 552 * in each set. The result is the increment of each counter during the 553 * period the counter set has been activated. 554 * 555 * Return true on success. 556 */ 557 static int cfdiag_diffctr(struct cpu_cf_events *cpuhw, unsigned long auth) 558 { 559 struct cf_trailer_entry *trailer_start, *trailer_stop; 560 struct cf_ctrset_entry *ctrstart, *ctrstop; 561 size_t offset = 0; 562 int i; 563 564 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 565 ctrstart = (struct cf_ctrset_entry *)(cpuhw->start + offset); 566 ctrstop = (struct cf_ctrset_entry *)(cpuhw->stop + offset); 567 568 /* Counter set not authorized */ 569 if (!(auth & cpumf_ctr_ctl[i])) 570 continue; 571 /* Counter set size zero was not saved */ 572 if (!cpum_cf_read_setsize(i)) 573 continue; 574 575 if (memcmp(ctrstop, ctrstart, sizeof(*ctrstop))) { 576 pr_err_once("cpum_cf_diag counter set compare error " 577 "in set %i\n", ctrstart->set); 578 return 0; 579 } 580 if (ctrstart->def == CF_DIAG_CTRSET_DEF) { 581 cfdiag_diffctrset((u64 *)(ctrstart + 1), 582 (u64 *)(ctrstop + 1), ctrstart->ctr); 583 offset += ctrstart->ctr * sizeof(u64) + 584 sizeof(*ctrstart); 585 } 586 } 587 588 /* Save time_stamp from start of event in stop's trailer */ 589 trailer_start = (struct cf_trailer_entry *)(cpuhw->start + offset); 590 trailer_stop = (struct cf_trailer_entry *)(cpuhw->stop + offset); 591 trailer_stop->progusage[0] = trailer_start->timestamp; 592 593 return 1; 594 } 595 596 static enum cpumf_ctr_set get_counter_set(u64 event) 597 { 598 int set = CPUMF_CTR_SET_MAX; 599 600 if (event < 32) 601 set = CPUMF_CTR_SET_BASIC; 602 else if (event < 64) 603 set = CPUMF_CTR_SET_USER; 604 else if (event < 128) 605 set = CPUMF_CTR_SET_CRYPTO; 606 else if (event < 288) 607 set = CPUMF_CTR_SET_EXT; 608 else if (event >= 448 && event < 496) 609 set = CPUMF_CTR_SET_MT_DIAG; 610 611 return set; 612 } 613 614 static int validate_ctr_version(const u64 config, enum cpumf_ctr_set set) 615 { 616 u16 mtdiag_ctl; 617 int err = 0; 618 619 /* check required version for counter sets */ 620 switch (set) { 621 case CPUMF_CTR_SET_BASIC: 622 case CPUMF_CTR_SET_USER: 623 if (cpumf_ctr_info.cfvn < 1) 624 err = -EOPNOTSUPP; 625 break; 626 case CPUMF_CTR_SET_CRYPTO: 627 if ((cpumf_ctr_info.csvn >= 1 && cpumf_ctr_info.csvn <= 5 && 628 config > 79) || (cpumf_ctr_info.csvn >= 6 && config > 83)) 629 err = -EOPNOTSUPP; 630 break; 631 case CPUMF_CTR_SET_EXT: 632 if (cpumf_ctr_info.csvn < 1) 633 err = -EOPNOTSUPP; 634 if ((cpumf_ctr_info.csvn == 1 && config > 159) || 635 (cpumf_ctr_info.csvn == 2 && config > 175) || 636 (cpumf_ctr_info.csvn >= 3 && cpumf_ctr_info.csvn <= 5 && 637 config > 255) || 638 (cpumf_ctr_info.csvn >= 6 && config > 287)) 639 err = -EOPNOTSUPP; 640 break; 641 case CPUMF_CTR_SET_MT_DIAG: 642 if (cpumf_ctr_info.csvn <= 3) 643 err = -EOPNOTSUPP; 644 /* 645 * MT-diagnostic counters are read-only. The counter set 646 * is automatically enabled and activated on all CPUs with 647 * multithreading (SMT). Deactivation of multithreading 648 * also disables the counter set. State changes are ignored 649 * by lcctl(). Because Linux controls SMT enablement through 650 * a kernel parameter only, the counter set is either disabled 651 * or enabled and active. 652 * 653 * Thus, the counters can only be used if SMT is on and the 654 * counter set is enabled and active. 655 */ 656 mtdiag_ctl = cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG]; 657 if (!((cpumf_ctr_info.auth_ctl & mtdiag_ctl) && 658 (cpumf_ctr_info.enable_ctl & mtdiag_ctl) && 659 (cpumf_ctr_info.act_ctl & mtdiag_ctl))) 660 err = -EOPNOTSUPP; 661 break; 662 case CPUMF_CTR_SET_MAX: 663 err = -EOPNOTSUPP; 664 } 665 666 return err; 667 } 668 669 /* 670 * Change the CPUMF state to active. 671 * Enable and activate the CPU-counter sets according 672 * to the per-cpu control state. 673 */ 674 static void cpumf_pmu_enable(struct pmu *pmu) 675 { 676 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 677 int err; 678 679 if (!cpuhw || (cpuhw->flags & PMU_F_ENABLED)) 680 return; 681 682 err = lcctl(cpuhw->state | cpuhw->dev_state); 683 if (err) 684 pr_err("Enabling the performance measuring unit failed with rc=%x\n", err); 685 else 686 cpuhw->flags |= PMU_F_ENABLED; 687 } 688 689 /* 690 * Change the CPUMF state to inactive. 691 * Disable and enable (inactive) the CPU-counter sets according 692 * to the per-cpu control state. 693 */ 694 static void cpumf_pmu_disable(struct pmu *pmu) 695 { 696 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 697 u64 inactive; 698 int err; 699 700 if (!cpuhw || !(cpuhw->flags & PMU_F_ENABLED)) 701 return; 702 703 inactive = cpuhw->state & ~((1 << CPUMF_LCCTL_ENABLE_SHIFT) - 1); 704 inactive |= cpuhw->dev_state; 705 err = lcctl(inactive); 706 if (err) 707 pr_err("Disabling the performance measuring unit failed with rc=%x\n", err); 708 else 709 cpuhw->flags &= ~PMU_F_ENABLED; 710 } 711 712 /* Release the PMU if event is the last perf event */ 713 static void hw_perf_event_destroy(struct perf_event *event) 714 { 715 cpum_cf_free(event->cpu); 716 } 717 718 /* CPUMF <-> perf event mappings for kernel+userspace (basic set) */ 719 static const int cpumf_generic_events_basic[] = { 720 [PERF_COUNT_HW_CPU_CYCLES] = 0, 721 [PERF_COUNT_HW_INSTRUCTIONS] = 1, 722 [PERF_COUNT_HW_CACHE_REFERENCES] = -1, 723 [PERF_COUNT_HW_CACHE_MISSES] = -1, 724 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1, 725 [PERF_COUNT_HW_BRANCH_MISSES] = -1, 726 [PERF_COUNT_HW_BUS_CYCLES] = -1, 727 }; 728 /* CPUMF <-> perf event mappings for userspace (problem-state set) */ 729 static const int cpumf_generic_events_user[] = { 730 [PERF_COUNT_HW_CPU_CYCLES] = 32, 731 [PERF_COUNT_HW_INSTRUCTIONS] = 33, 732 [PERF_COUNT_HW_CACHE_REFERENCES] = -1, 733 [PERF_COUNT_HW_CACHE_MISSES] = -1, 734 [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = -1, 735 [PERF_COUNT_HW_BRANCH_MISSES] = -1, 736 [PERF_COUNT_HW_BUS_CYCLES] = -1, 737 }; 738 739 static int is_userspace_event(u64 ev) 740 { 741 return cpumf_generic_events_user[PERF_COUNT_HW_CPU_CYCLES] == ev || 742 cpumf_generic_events_user[PERF_COUNT_HW_INSTRUCTIONS] == ev; 743 } 744 745 static int __hw_perf_event_init(struct perf_event *event, unsigned int type) 746 { 747 struct perf_event_attr *attr = &event->attr; 748 struct hw_perf_event *hwc = &event->hw; 749 enum cpumf_ctr_set set; 750 u64 ev; 751 752 switch (type) { 753 case PERF_TYPE_RAW: 754 /* Raw events are used to access counters directly, 755 * hence do not permit excludes */ 756 if (attr->exclude_kernel || attr->exclude_user || 757 attr->exclude_hv) 758 return -EOPNOTSUPP; 759 ev = attr->config; 760 break; 761 762 case PERF_TYPE_HARDWARE: 763 if (is_sampling_event(event)) /* No sampling support */ 764 return -ENOENT; 765 ev = attr->config; 766 if (!attr->exclude_user && attr->exclude_kernel) { 767 /* 768 * Count user space (problem-state) only 769 * Handle events 32 and 33 as 0:u and 1:u 770 */ 771 if (!is_userspace_event(ev)) { 772 if (ev >= ARRAY_SIZE(cpumf_generic_events_user)) 773 return -EOPNOTSUPP; 774 ev = cpumf_generic_events_user[ev]; 775 } 776 } else if (!attr->exclude_kernel && attr->exclude_user) { 777 /* No support for kernel space counters only */ 778 return -EOPNOTSUPP; 779 } else { 780 /* Count user and kernel space, incl. events 32 + 33 */ 781 if (!is_userspace_event(ev)) { 782 if (ev >= ARRAY_SIZE(cpumf_generic_events_basic)) 783 return -EOPNOTSUPP; 784 ev = cpumf_generic_events_basic[ev]; 785 } 786 } 787 break; 788 789 default: 790 return -ENOENT; 791 } 792 793 if (ev == -1) 794 return -ENOENT; 795 796 if (ev > PERF_CPUM_CF_MAX_CTR) 797 return -ENOENT; 798 799 /* Obtain the counter set to which the specified counter belongs */ 800 set = get_counter_set(ev); 801 switch (set) { 802 case CPUMF_CTR_SET_BASIC: 803 case CPUMF_CTR_SET_USER: 804 case CPUMF_CTR_SET_CRYPTO: 805 case CPUMF_CTR_SET_EXT: 806 case CPUMF_CTR_SET_MT_DIAG: 807 /* 808 * Use the hardware perf event structure to store the 809 * counter number in the 'config' member and the counter 810 * set number in the 'config_base' as bit mask. 811 * It is later used to enable/disable the counter(s). 812 */ 813 hwc->config = ev; 814 hwc->config_base = cpumf_ctr_ctl[set]; 815 break; 816 case CPUMF_CTR_SET_MAX: 817 /* The counter could not be associated to a counter set */ 818 return -EINVAL; 819 } 820 821 /* Initialize for using the CPU-measurement counter facility */ 822 if (cpum_cf_alloc(event->cpu)) 823 return -ENOMEM; 824 event->destroy = hw_perf_event_destroy; 825 826 /* 827 * Finally, validate version and authorization of the counter set. 828 * If the particular CPU counter set is not authorized, 829 * return with -ENOENT in order to fall back to other 830 * PMUs that might suffice the event request. 831 */ 832 if (!(hwc->config_base & cpumf_ctr_info.auth_ctl)) 833 return -ENOENT; 834 return validate_ctr_version(hwc->config, set); 835 } 836 837 /* Events CPU_CYCLES and INSTRUCTIONS can be submitted with two different 838 * attribute::type values: 839 * - PERF_TYPE_HARDWARE: 840 * - pmu->type: 841 * Handle both type of invocations identical. They address the same hardware. 842 * The result is different when event modifiers exclude_kernel and/or 843 * exclude_user are also set. 844 */ 845 static int cpumf_pmu_event_type(struct perf_event *event) 846 { 847 u64 ev = event->attr.config; 848 849 if (cpumf_generic_events_basic[PERF_COUNT_HW_CPU_CYCLES] == ev || 850 cpumf_generic_events_basic[PERF_COUNT_HW_INSTRUCTIONS] == ev || 851 cpumf_generic_events_user[PERF_COUNT_HW_CPU_CYCLES] == ev || 852 cpumf_generic_events_user[PERF_COUNT_HW_INSTRUCTIONS] == ev) 853 return PERF_TYPE_HARDWARE; 854 return PERF_TYPE_RAW; 855 } 856 857 static int cpumf_pmu_event_init(struct perf_event *event) 858 { 859 unsigned int type = event->attr.type; 860 int err = -ENOENT; 861 862 if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_RAW) 863 err = __hw_perf_event_init(event, type); 864 else if (event->pmu->type == type) 865 /* Registered as unknown PMU */ 866 err = __hw_perf_event_init(event, cpumf_pmu_event_type(event)); 867 868 return err; 869 } 870 871 static int hw_perf_event_reset(struct perf_event *event) 872 { 873 u64 prev, new; 874 int err; 875 876 prev = local64_read(&event->hw.prev_count); 877 do { 878 err = ecctr(event->hw.config, &new); 879 if (err) { 880 if (err != 3) 881 break; 882 /* The counter is not (yet) available. This 883 * might happen if the counter set to which 884 * this counter belongs is in the disabled 885 * state. 886 */ 887 new = 0; 888 } 889 } while (!local64_try_cmpxchg(&event->hw.prev_count, &prev, new)); 890 891 return err; 892 } 893 894 static void hw_perf_event_update(struct perf_event *event) 895 { 896 u64 prev, new, delta; 897 int err; 898 899 prev = local64_read(&event->hw.prev_count); 900 do { 901 err = ecctr(event->hw.config, &new); 902 if (err) 903 return; 904 } while (!local64_try_cmpxchg(&event->hw.prev_count, &prev, new)); 905 906 delta = (prev <= new) ? new - prev 907 : (-1ULL - prev) + new + 1; /* overflow */ 908 local64_add(delta, &event->count); 909 } 910 911 static void cpumf_pmu_read(struct perf_event *event) 912 { 913 if (event->hw.state & PERF_HES_STOPPED) 914 return; 915 916 hw_perf_event_update(event); 917 } 918 919 static void cpumf_pmu_start(struct perf_event *event, int flags) 920 { 921 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 922 struct hw_perf_event *hwc = &event->hw; 923 int i; 924 925 if (!(hwc->state & PERF_HES_STOPPED)) 926 return; 927 928 hwc->state = 0; 929 930 /* (Re-)enable and activate the counter set */ 931 ctr_set_enable(&cpuhw->state, hwc->config_base); 932 ctr_set_start(&cpuhw->state, hwc->config_base); 933 934 /* The counter set to which this counter belongs can be already active. 935 * Because all counters in a set are active, the event->hw.prev_count 936 * needs to be synchronized. At this point, the counter set can be in 937 * the inactive or disabled state. 938 */ 939 if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) { 940 cpuhw->usedss = cfdiag_getctr(cpuhw->start, 941 sizeof(cpuhw->start), 942 hwc->config_base, true); 943 } else { 944 hw_perf_event_reset(event); 945 } 946 947 /* Increment refcount for counter sets */ 948 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) 949 if ((hwc->config_base & cpumf_ctr_ctl[i])) 950 atomic_inc(&cpuhw->ctr_set[i]); 951 } 952 953 /* Create perf event sample with the counter sets as raw data. The sample 954 * is then pushed to the event subsystem and the function checks for 955 * possible event overflows. If an event overflow occurs, the PMU is 956 * stopped. 957 * 958 * Return non-zero if an event overflow occurred. 959 */ 960 static int cfdiag_push_sample(struct perf_event *event, 961 struct cpu_cf_events *cpuhw) 962 { 963 struct perf_sample_data data; 964 struct perf_raw_record raw; 965 struct pt_regs regs; 966 int overflow; 967 968 /* Setup perf sample */ 969 perf_sample_data_init(&data, 0, event->hw.last_period); 970 memset(®s, 0, sizeof(regs)); 971 memset(&raw, 0, sizeof(raw)); 972 973 if (event->attr.sample_type & PERF_SAMPLE_CPU) 974 data.cpu_entry.cpu = event->cpu; 975 if (event->attr.sample_type & PERF_SAMPLE_RAW) { 976 raw.frag.size = cpuhw->usedss; 977 raw.frag.data = cpuhw->stop; 978 perf_sample_save_raw_data(&data, event, &raw); 979 } 980 981 overflow = perf_event_overflow(event, &data, ®s); 982 983 perf_event_update_userpage(event); 984 return overflow; 985 } 986 987 static void cpumf_pmu_stop(struct perf_event *event, int flags) 988 { 989 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 990 struct hw_perf_event *hwc = &event->hw; 991 int i; 992 993 if (!(hwc->state & PERF_HES_STOPPED)) { 994 /* Decrement reference count for this counter set and if this 995 * is the last used counter in the set, clear activation 996 * control and set the counter set state to inactive. 997 */ 998 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 999 if (!(hwc->config_base & cpumf_ctr_ctl[i])) 1000 continue; 1001 if (!atomic_dec_return(&cpuhw->ctr_set[i])) 1002 ctr_set_stop(&cpuhw->state, cpumf_ctr_ctl[i]); 1003 } 1004 hwc->state |= PERF_HES_STOPPED; 1005 } 1006 1007 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) { 1008 if (hwc->config == PERF_EVENT_CPUM_CF_DIAG) { 1009 local64_inc(&event->count); 1010 cpuhw->usedss = cfdiag_getctr(cpuhw->stop, 1011 sizeof(cpuhw->stop), 1012 event->hw.config_base, 1013 false); 1014 if (cfdiag_diffctr(cpuhw, event->hw.config_base)) 1015 cfdiag_push_sample(event, cpuhw); 1016 } else { 1017 hw_perf_event_update(event); 1018 } 1019 hwc->state |= PERF_HES_UPTODATE; 1020 } 1021 } 1022 1023 static int cpumf_pmu_add(struct perf_event *event, int flags) 1024 { 1025 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 1026 1027 ctr_set_enable(&cpuhw->state, event->hw.config_base); 1028 event->hw.state = PERF_HES_UPTODATE | PERF_HES_STOPPED; 1029 1030 if (flags & PERF_EF_START) 1031 cpumf_pmu_start(event, PERF_EF_RELOAD); 1032 1033 return 0; 1034 } 1035 1036 static void cpumf_pmu_del(struct perf_event *event, int flags) 1037 { 1038 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 1039 int i; 1040 1041 cpumf_pmu_stop(event, PERF_EF_UPDATE); 1042 1043 /* Check if any counter in the counter set is still used. If not used, 1044 * change the counter set to the disabled state. This also clears the 1045 * content of all counters in the set. 1046 * 1047 * When a new perf event has been added but not yet started, this can 1048 * clear enable control and resets all counters in a set. Therefore, 1049 * cpumf_pmu_start() always has to re-enable a counter set. 1050 */ 1051 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) 1052 if (!atomic_read(&cpuhw->ctr_set[i])) 1053 ctr_set_disable(&cpuhw->state, cpumf_ctr_ctl[i]); 1054 } 1055 1056 /* Performance monitoring unit for s390x */ 1057 static struct pmu cpumf_pmu = { 1058 .task_ctx_nr = perf_sw_context, 1059 .capabilities = PERF_PMU_CAP_NO_INTERRUPT, 1060 .pmu_enable = cpumf_pmu_enable, 1061 .pmu_disable = cpumf_pmu_disable, 1062 .event_init = cpumf_pmu_event_init, 1063 .add = cpumf_pmu_add, 1064 .del = cpumf_pmu_del, 1065 .start = cpumf_pmu_start, 1066 .stop = cpumf_pmu_stop, 1067 .read = cpumf_pmu_read, 1068 }; 1069 1070 static struct cfset_session { /* CPUs and counter set bit mask */ 1071 struct list_head head; /* Head of list of active processes */ 1072 } cfset_session = { 1073 .head = LIST_HEAD_INIT(cfset_session.head) 1074 }; 1075 1076 static refcount_t cfset_opencnt = REFCOUNT_INIT(0); /* Access count */ 1077 /* 1078 * Synchronize access to device /dev/hwc. This mutex protects against 1079 * concurrent access to functions cfset_open() and cfset_release(). 1080 * Same for CPU hotplug add and remove events triggering 1081 * cpum_cf_online_cpu() and cpum_cf_offline_cpu(). 1082 * It also serializes concurrent device ioctl access from multiple 1083 * processes accessing /dev/hwc. 1084 * 1085 * The mutex protects concurrent access to the /dev/hwctr session management 1086 * struct cfset_session and reference counting variable cfset_opencnt. 1087 */ 1088 static DEFINE_MUTEX(cfset_ctrset_mutex); 1089 1090 /* 1091 * CPU hotplug handles only /dev/hwctr device. 1092 * For perf_event_open() the CPU hotplug handling is done on kernel common 1093 * code: 1094 * - CPU add: Nothing is done since a file descriptor can not be created 1095 * and returned to the user. 1096 * - CPU delete: Handled by common code via pmu_disable(), pmu_stop() and 1097 * pmu_delete(). The event itself is removed when the file descriptor is 1098 * closed. 1099 */ 1100 static int cfset_online_cpu(unsigned int cpu); 1101 1102 static int cpum_cf_online_cpu(unsigned int cpu) 1103 { 1104 int rc = 0; 1105 1106 /* 1107 * Ignore notification for perf_event_open(). 1108 * Handle only /dev/hwctr device sessions. 1109 */ 1110 mutex_lock(&cfset_ctrset_mutex); 1111 if (refcount_read(&cfset_opencnt)) { 1112 rc = cpum_cf_alloc_cpu(cpu); 1113 if (!rc) 1114 cfset_online_cpu(cpu); 1115 } 1116 mutex_unlock(&cfset_ctrset_mutex); 1117 return rc; 1118 } 1119 1120 static int cfset_offline_cpu(unsigned int cpu); 1121 1122 static int cpum_cf_offline_cpu(unsigned int cpu) 1123 { 1124 /* 1125 * During task exit processing of grouped perf events triggered by CPU 1126 * hotplug processing, pmu_disable() is called as part of perf context 1127 * removal process. Therefore do not trigger event removal now for 1128 * perf_event_open() created events. Perf common code triggers event 1129 * destruction when the event file descriptor is closed. 1130 * 1131 * Handle only /dev/hwctr device sessions. 1132 */ 1133 mutex_lock(&cfset_ctrset_mutex); 1134 if (refcount_read(&cfset_opencnt)) { 1135 cfset_offline_cpu(cpu); 1136 cpum_cf_free_cpu(cpu); 1137 } 1138 mutex_unlock(&cfset_ctrset_mutex); 1139 return 0; 1140 } 1141 1142 /* Return true if store counter set multiple instruction is available */ 1143 static inline int stccm_avail(void) 1144 { 1145 return test_facility(142); 1146 } 1147 1148 /* CPU-measurement alerts for the counter facility */ 1149 static void cpumf_measurement_alert(struct ext_code ext_code, 1150 unsigned int alert, unsigned long unused) 1151 { 1152 struct cpu_cf_events *cpuhw; 1153 1154 if (!(alert & CPU_MF_INT_CF_MASK)) 1155 return; 1156 1157 inc_irq_stat(IRQEXT_CMC); 1158 1159 /* 1160 * Measurement alerts are shared and might happen when the PMU 1161 * is not reserved. Ignore these alerts in this case. 1162 */ 1163 cpuhw = this_cpu_cfhw(); 1164 if (!cpuhw) 1165 return; 1166 1167 /* counter authorization change alert */ 1168 if (alert & CPU_MF_INT_CF_CACA) 1169 qctri(&cpumf_ctr_info); 1170 1171 /* loss of counter data alert */ 1172 if (alert & CPU_MF_INT_CF_LCDA) 1173 pr_err("CPU[%i] Counter data was lost\n", smp_processor_id()); 1174 1175 /* loss of MT counter data alert */ 1176 if (alert & CPU_MF_INT_CF_MTDA) 1177 pr_warn("CPU[%i] MT counter data was lost\n", 1178 smp_processor_id()); 1179 } 1180 1181 static int cfset_init(void); 1182 static int __init cpumf_pmu_init(void) 1183 { 1184 int rc; 1185 1186 /* Extract counter measurement facility information */ 1187 if (!cpum_cf_avail() || qctri(&cpumf_ctr_info)) 1188 return -ENODEV; 1189 1190 /* Determine and store counter set sizes for later reference */ 1191 for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc) 1192 cpum_cf_make_setsize(rc); 1193 1194 /* 1195 * Clear bit 15 of cr0 to unauthorize problem-state to 1196 * extract measurement counters 1197 */ 1198 system_ctl_clear_bit(0, CR0_CPUMF_EXTRACTION_AUTH_BIT); 1199 1200 /* register handler for measurement-alert interruptions */ 1201 rc = register_external_irq(EXT_IRQ_MEASURE_ALERT, 1202 cpumf_measurement_alert); 1203 if (rc) { 1204 pr_err("Registering for CPU-measurement alerts failed with rc=%i\n", rc); 1205 return rc; 1206 } 1207 1208 /* Setup s390dbf facility */ 1209 cf_dbg = debug_register(KMSG_COMPONENT, 2, 1, 128); 1210 if (!cf_dbg) { 1211 pr_err("Registration of s390dbf(cpum_cf) failed\n"); 1212 rc = -ENOMEM; 1213 goto out1; 1214 } 1215 debug_register_view(cf_dbg, &debug_sprintf_view); 1216 1217 cpumf_pmu.attr_groups = cpumf_cf_event_group(); 1218 rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", -1); 1219 if (rc) { 1220 pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc); 1221 goto out2; 1222 } else if (stccm_avail()) { /* Setup counter set device */ 1223 cfset_init(); 1224 } 1225 1226 rc = cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE, 1227 "perf/s390/cf:online", 1228 cpum_cf_online_cpu, cpum_cf_offline_cpu); 1229 return rc; 1230 1231 out2: 1232 debug_unregister_view(cf_dbg, &debug_sprintf_view); 1233 debug_unregister(cf_dbg); 1234 out1: 1235 unregister_external_irq(EXT_IRQ_MEASURE_ALERT, cpumf_measurement_alert); 1236 return rc; 1237 } 1238 1239 /* Support for the CPU Measurement Facility counter set extraction using 1240 * device /dev/hwctr. This allows user space programs to extract complete 1241 * counter set via normal file operations. 1242 */ 1243 1244 struct cfset_call_on_cpu_parm { /* Parm struct for smp_call_on_cpu */ 1245 unsigned int sets; /* Counter set bit mask */ 1246 atomic_t cpus_ack; /* # CPUs successfully executed func */ 1247 }; 1248 1249 struct cfset_request { /* CPUs and counter set bit mask */ 1250 unsigned long ctrset; /* Bit mask of counter set to read */ 1251 cpumask_t mask; /* CPU mask to read from */ 1252 struct list_head node; /* Chain to cfset_session.head */ 1253 }; 1254 1255 static void cfset_session_init(void) 1256 { 1257 INIT_LIST_HEAD(&cfset_session.head); 1258 } 1259 1260 /* Remove current request from global bookkeeping. Maintain a counter set bit 1261 * mask on a per CPU basis. 1262 * Done in process context under mutex protection. 1263 */ 1264 static void cfset_session_del(struct cfset_request *p) 1265 { 1266 list_del(&p->node); 1267 } 1268 1269 /* Add current request to global bookkeeping. Maintain a counter set bit mask 1270 * on a per CPU basis. 1271 * Done in process context under mutex protection. 1272 */ 1273 static void cfset_session_add(struct cfset_request *p) 1274 { 1275 list_add(&p->node, &cfset_session.head); 1276 } 1277 1278 /* The /dev/hwctr device access uses PMU_F_IN_USE to mark the device access 1279 * path is currently used. 1280 * The cpu_cf_events::dev_state is used to denote counter sets in use by this 1281 * interface. It is always or'ed in. If this interface is not active, its 1282 * value is zero and no additional counter sets will be included. 1283 * 1284 * The cpu_cf_events::state is used by the perf_event_open SVC and remains 1285 * unchanged. 1286 * 1287 * perf_pmu_enable() and perf_pmu_enable() and its call backs 1288 * cpumf_pmu_enable() and cpumf_pmu_disable() are called by the 1289 * performance measurement subsystem to enable per process 1290 * CPU Measurement counter facility. 1291 * The XXX_enable() and XXX_disable functions are used to turn off 1292 * x86 performance monitoring interrupt (PMI) during scheduling. 1293 * s390 uses these calls to temporarily stop and resume the active CPU 1294 * counters sets during scheduling. 1295 * 1296 * We do allow concurrent access of perf_event_open() SVC and /dev/hwctr 1297 * device access. The perf_event_open() SVC interface makes a lot of effort 1298 * to only run the counters while the calling process is actively scheduled 1299 * to run. 1300 * When /dev/hwctr interface is also used at the same time, the counter sets 1301 * will keep running, even when the process is scheduled off a CPU. 1302 * However this is not a problem and does not lead to wrong counter values 1303 * for the perf_event_open() SVC. The current counter value will be recorded 1304 * during schedule-in. At schedule-out time the current counter value is 1305 * extracted again and the delta is calculated and added to the event. 1306 */ 1307 /* Stop all counter sets via ioctl interface */ 1308 static void cfset_ioctl_off(void *parm) 1309 { 1310 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 1311 struct cfset_call_on_cpu_parm *p = parm; 1312 int rc; 1313 1314 /* Check if any counter set used by /dev/hwctr */ 1315 for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc) 1316 if ((p->sets & cpumf_ctr_ctl[rc])) { 1317 if (!atomic_dec_return(&cpuhw->ctr_set[rc])) { 1318 ctr_set_disable(&cpuhw->dev_state, 1319 cpumf_ctr_ctl[rc]); 1320 ctr_set_stop(&cpuhw->dev_state, 1321 cpumf_ctr_ctl[rc]); 1322 } 1323 } 1324 /* Keep perf_event_open counter sets */ 1325 rc = lcctl(cpuhw->dev_state | cpuhw->state); 1326 if (rc) 1327 pr_err("Counter set stop %#llx of /dev/%s failed rc=%i\n", 1328 cpuhw->state, S390_HWCTR_DEVICE, rc); 1329 if (!cpuhw->dev_state) 1330 cpuhw->flags &= ~PMU_F_IN_USE; 1331 } 1332 1333 /* Start counter sets on particular CPU */ 1334 static void cfset_ioctl_on(void *parm) 1335 { 1336 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 1337 struct cfset_call_on_cpu_parm *p = parm; 1338 int rc; 1339 1340 cpuhw->flags |= PMU_F_IN_USE; 1341 ctr_set_enable(&cpuhw->dev_state, p->sets); 1342 ctr_set_start(&cpuhw->dev_state, p->sets); 1343 for (rc = CPUMF_CTR_SET_BASIC; rc < CPUMF_CTR_SET_MAX; ++rc) 1344 if ((p->sets & cpumf_ctr_ctl[rc])) 1345 atomic_inc(&cpuhw->ctr_set[rc]); 1346 rc = lcctl(cpuhw->dev_state | cpuhw->state); /* Start counter sets */ 1347 if (!rc) 1348 atomic_inc(&p->cpus_ack); 1349 else 1350 pr_err("Counter set start %#llx of /dev/%s failed rc=%i\n", 1351 cpuhw->dev_state | cpuhw->state, S390_HWCTR_DEVICE, rc); 1352 } 1353 1354 static void cfset_release_cpu(void *p) 1355 { 1356 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 1357 int rc; 1358 1359 cpuhw->dev_state = 0; 1360 rc = lcctl(cpuhw->state); /* Keep perf_event_open counter sets */ 1361 if (rc) 1362 pr_err("Counter set release %#llx of /dev/%s failed rc=%i\n", 1363 cpuhw->state, S390_HWCTR_DEVICE, rc); 1364 } 1365 1366 /* This modifies the process CPU mask to adopt it to the currently online 1367 * CPUs. Offline CPUs can not be addresses. This call terminates the access 1368 * and is usually followed by close() or a new iotcl(..., START, ...) which 1369 * creates a new request structure. 1370 */ 1371 static void cfset_all_stop(struct cfset_request *req) 1372 { 1373 struct cfset_call_on_cpu_parm p = { 1374 .sets = req->ctrset, 1375 }; 1376 1377 cpumask_and(&req->mask, &req->mask, cpu_online_mask); 1378 on_each_cpu_mask(&req->mask, cfset_ioctl_off, &p, 1); 1379 } 1380 1381 /* Release function is also called when application gets terminated without 1382 * doing a proper ioctl(..., S390_HWCTR_STOP, ...) command. 1383 */ 1384 static int cfset_release(struct inode *inode, struct file *file) 1385 { 1386 mutex_lock(&cfset_ctrset_mutex); 1387 /* Open followed by close/exit has no private_data */ 1388 if (file->private_data) { 1389 cfset_all_stop(file->private_data); 1390 cfset_session_del(file->private_data); 1391 kfree(file->private_data); 1392 file->private_data = NULL; 1393 } 1394 if (refcount_dec_and_test(&cfset_opencnt)) { /* Last close */ 1395 on_each_cpu(cfset_release_cpu, NULL, 1); 1396 cpum_cf_free(-1); 1397 } 1398 mutex_unlock(&cfset_ctrset_mutex); 1399 return 0; 1400 } 1401 1402 /* 1403 * Open via /dev/hwctr device. Allocate all per CPU resources on the first 1404 * open of the device. The last close releases all per CPU resources. 1405 * Parallel perf_event_open system calls also use per CPU resources. 1406 * These invocations are handled via reference counting on the per CPU data 1407 * structures. 1408 */ 1409 static int cfset_open(struct inode *inode, struct file *file) 1410 { 1411 int rc = 0; 1412 1413 if (!perfmon_capable()) 1414 return -EPERM; 1415 file->private_data = NULL; 1416 1417 mutex_lock(&cfset_ctrset_mutex); 1418 if (!refcount_inc_not_zero(&cfset_opencnt)) { /* First open */ 1419 rc = cpum_cf_alloc(-1); 1420 if (!rc) { 1421 cfset_session_init(); 1422 refcount_set(&cfset_opencnt, 1); 1423 } 1424 } 1425 mutex_unlock(&cfset_ctrset_mutex); 1426 1427 /* nonseekable_open() never fails */ 1428 return rc ?: nonseekable_open(inode, file); 1429 } 1430 1431 static int cfset_all_start(struct cfset_request *req) 1432 { 1433 struct cfset_call_on_cpu_parm p = { 1434 .sets = req->ctrset, 1435 .cpus_ack = ATOMIC_INIT(0), 1436 }; 1437 cpumask_var_t mask; 1438 int rc = 0; 1439 1440 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 1441 return -ENOMEM; 1442 cpumask_and(mask, &req->mask, cpu_online_mask); 1443 on_each_cpu_mask(mask, cfset_ioctl_on, &p, 1); 1444 if (atomic_read(&p.cpus_ack) != cpumask_weight(mask)) { 1445 on_each_cpu_mask(mask, cfset_ioctl_off, &p, 1); 1446 rc = -EIO; 1447 } 1448 free_cpumask_var(mask); 1449 return rc; 1450 } 1451 1452 /* Return the maximum required space for all possible CPUs in case one 1453 * CPU will be onlined during the START, READ, STOP cycles. 1454 * To find out the size of the counter sets, any one CPU will do. They 1455 * all have the same counter sets. 1456 */ 1457 static size_t cfset_needspace(unsigned int sets) 1458 { 1459 size_t bytes = 0; 1460 int i; 1461 1462 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 1463 if (!(sets & cpumf_ctr_ctl[i])) 1464 continue; 1465 bytes += cpum_cf_read_setsize(i) * sizeof(u64) + 1466 sizeof(((struct s390_ctrset_setdata *)0)->set) + 1467 sizeof(((struct s390_ctrset_setdata *)0)->no_cnts); 1468 } 1469 bytes = sizeof(((struct s390_ctrset_read *)0)->no_cpus) + nr_cpu_ids * 1470 (bytes + sizeof(((struct s390_ctrset_cpudata *)0)->cpu_nr) + 1471 sizeof(((struct s390_ctrset_cpudata *)0)->no_sets)); 1472 return bytes; 1473 } 1474 1475 static int cfset_all_copy(unsigned long arg, cpumask_t *mask) 1476 { 1477 struct s390_ctrset_read __user *ctrset_read; 1478 unsigned int cpu, cpus, rc = 0; 1479 void __user *uptr; 1480 1481 ctrset_read = (struct s390_ctrset_read __user *)arg; 1482 uptr = ctrset_read->data; 1483 for_each_cpu(cpu, mask) { 1484 struct cpu_cf_events *cpuhw = get_cpu_cfhw(cpu); 1485 struct s390_ctrset_cpudata __user *ctrset_cpudata; 1486 1487 ctrset_cpudata = uptr; 1488 rc = put_user(cpu, &ctrset_cpudata->cpu_nr); 1489 rc |= put_user(cpuhw->sets, &ctrset_cpudata->no_sets); 1490 rc |= copy_to_user(ctrset_cpudata->data, cpuhw->data, 1491 cpuhw->used); 1492 if (rc) { 1493 rc = -EFAULT; 1494 goto out; 1495 } 1496 uptr += sizeof(struct s390_ctrset_cpudata) + cpuhw->used; 1497 cond_resched(); 1498 } 1499 cpus = cpumask_weight(mask); 1500 if (put_user(cpus, &ctrset_read->no_cpus)) 1501 rc = -EFAULT; 1502 out: 1503 return rc; 1504 } 1505 1506 static size_t cfset_cpuset_read(struct s390_ctrset_setdata *p, int ctrset, 1507 int ctrset_size, size_t room) 1508 { 1509 size_t need = 0; 1510 int rc = -1; 1511 1512 need = sizeof(*p) + sizeof(u64) * ctrset_size; 1513 if (need <= room) { 1514 p->set = cpumf_ctr_ctl[ctrset]; 1515 p->no_cnts = ctrset_size; 1516 rc = ctr_stcctm(ctrset, ctrset_size, (u64 *)p->cv); 1517 if (rc == 3) /* Nothing stored */ 1518 need = 0; 1519 } 1520 return need; 1521 } 1522 1523 /* Read all counter sets. */ 1524 static void cfset_cpu_read(void *parm) 1525 { 1526 struct cpu_cf_events *cpuhw = this_cpu_cfhw(); 1527 struct cfset_call_on_cpu_parm *p = parm; 1528 int set, set_size; 1529 size_t space; 1530 1531 /* No data saved yet */ 1532 cpuhw->used = 0; 1533 cpuhw->sets = 0; 1534 memset(cpuhw->data, 0, sizeof(cpuhw->data)); 1535 1536 /* Scan the counter sets */ 1537 for (set = CPUMF_CTR_SET_BASIC; set < CPUMF_CTR_SET_MAX; ++set) { 1538 struct s390_ctrset_setdata *sp = (void *)cpuhw->data + 1539 cpuhw->used; 1540 1541 if (!(p->sets & cpumf_ctr_ctl[set])) 1542 continue; /* Counter set not in list */ 1543 set_size = cpum_cf_read_setsize(set); 1544 space = sizeof(cpuhw->data) - cpuhw->used; 1545 space = cfset_cpuset_read(sp, set, set_size, space); 1546 if (space) { 1547 cpuhw->used += space; 1548 cpuhw->sets += 1; 1549 } 1550 } 1551 } 1552 1553 static int cfset_all_read(unsigned long arg, struct cfset_request *req) 1554 { 1555 struct cfset_call_on_cpu_parm p; 1556 cpumask_var_t mask; 1557 int rc; 1558 1559 if (!alloc_cpumask_var(&mask, GFP_KERNEL)) 1560 return -ENOMEM; 1561 1562 p.sets = req->ctrset; 1563 cpumask_and(mask, &req->mask, cpu_online_mask); 1564 on_each_cpu_mask(mask, cfset_cpu_read, &p, 1); 1565 rc = cfset_all_copy(arg, mask); 1566 free_cpumask_var(mask); 1567 return rc; 1568 } 1569 1570 static long cfset_ioctl_read(unsigned long arg, struct cfset_request *req) 1571 { 1572 int ret = -ENODATA; 1573 1574 if (req && req->ctrset) 1575 ret = cfset_all_read(arg, req); 1576 return ret; 1577 } 1578 1579 static long cfset_ioctl_stop(struct file *file) 1580 { 1581 struct cfset_request *req = file->private_data; 1582 int ret = -ENXIO; 1583 1584 if (req) { 1585 cfset_all_stop(req); 1586 cfset_session_del(req); 1587 kfree(req); 1588 file->private_data = NULL; 1589 ret = 0; 1590 } 1591 return ret; 1592 } 1593 1594 static long cfset_ioctl_start(unsigned long arg, struct file *file) 1595 { 1596 struct s390_ctrset_start __user *ustart; 1597 struct s390_ctrset_start start; 1598 struct cfset_request *preq; 1599 void __user *umask; 1600 unsigned int len; 1601 int ret = 0; 1602 size_t need; 1603 1604 if (file->private_data) 1605 return -EBUSY; 1606 ustart = (struct s390_ctrset_start __user *)arg; 1607 if (copy_from_user(&start, ustart, sizeof(start))) 1608 return -EFAULT; 1609 if (start.version != S390_HWCTR_START_VERSION) 1610 return -EINVAL; 1611 if (start.counter_sets & ~(cpumf_ctr_ctl[CPUMF_CTR_SET_BASIC] | 1612 cpumf_ctr_ctl[CPUMF_CTR_SET_USER] | 1613 cpumf_ctr_ctl[CPUMF_CTR_SET_CRYPTO] | 1614 cpumf_ctr_ctl[CPUMF_CTR_SET_EXT] | 1615 cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG])) 1616 return -EINVAL; /* Invalid counter set */ 1617 if (!start.counter_sets) 1618 return -EINVAL; /* No counter set at all? */ 1619 1620 preq = kzalloc(sizeof(*preq), GFP_KERNEL); 1621 if (!preq) 1622 return -ENOMEM; 1623 cpumask_clear(&preq->mask); 1624 len = min_t(u64, start.cpumask_len, cpumask_size()); 1625 umask = (void __user *)start.cpumask; 1626 if (copy_from_user(&preq->mask, umask, len)) { 1627 kfree(preq); 1628 return -EFAULT; 1629 } 1630 if (cpumask_empty(&preq->mask)) { 1631 kfree(preq); 1632 return -EINVAL; 1633 } 1634 need = cfset_needspace(start.counter_sets); 1635 if (put_user(need, &ustart->data_bytes)) { 1636 kfree(preq); 1637 return -EFAULT; 1638 } 1639 preq->ctrset = start.counter_sets; 1640 ret = cfset_all_start(preq); 1641 if (!ret) { 1642 cfset_session_add(preq); 1643 file->private_data = preq; 1644 } else { 1645 kfree(preq); 1646 } 1647 return ret; 1648 } 1649 1650 /* Entry point to the /dev/hwctr device interface. 1651 * The ioctl system call supports three subcommands: 1652 * S390_HWCTR_START: Start the specified counter sets on a CPU list. The 1653 * counter set keeps running until explicitly stopped. Returns the number 1654 * of bytes needed to store the counter values. If another S390_HWCTR_START 1655 * ioctl subcommand is called without a previous S390_HWCTR_STOP stop 1656 * command on the same file descriptor, -EBUSY is returned. 1657 * S390_HWCTR_READ: Read the counter set values from specified CPU list given 1658 * with the S390_HWCTR_START command. 1659 * S390_HWCTR_STOP: Stops the counter sets on the CPU list given with the 1660 * previous S390_HWCTR_START subcommand. 1661 */ 1662 static long cfset_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 1663 { 1664 int ret; 1665 1666 cpus_read_lock(); 1667 mutex_lock(&cfset_ctrset_mutex); 1668 switch (cmd) { 1669 case S390_HWCTR_START: 1670 ret = cfset_ioctl_start(arg, file); 1671 break; 1672 case S390_HWCTR_STOP: 1673 ret = cfset_ioctl_stop(file); 1674 break; 1675 case S390_HWCTR_READ: 1676 ret = cfset_ioctl_read(arg, file->private_data); 1677 break; 1678 default: 1679 ret = -ENOTTY; 1680 break; 1681 } 1682 mutex_unlock(&cfset_ctrset_mutex); 1683 cpus_read_unlock(); 1684 return ret; 1685 } 1686 1687 static const struct file_operations cfset_fops = { 1688 .owner = THIS_MODULE, 1689 .open = cfset_open, 1690 .release = cfset_release, 1691 .unlocked_ioctl = cfset_ioctl, 1692 .compat_ioctl = cfset_ioctl, 1693 }; 1694 1695 static struct miscdevice cfset_dev = { 1696 .name = S390_HWCTR_DEVICE, 1697 .minor = MISC_DYNAMIC_MINOR, 1698 .fops = &cfset_fops, 1699 .mode = 0666, 1700 }; 1701 1702 /* Hotplug add of a CPU. Scan through all active processes and add 1703 * that CPU to the list of CPUs supplied with ioctl(..., START, ...). 1704 */ 1705 static int cfset_online_cpu(unsigned int cpu) 1706 { 1707 struct cfset_call_on_cpu_parm p; 1708 struct cfset_request *rp; 1709 1710 if (!list_empty(&cfset_session.head)) { 1711 list_for_each_entry(rp, &cfset_session.head, node) { 1712 p.sets = rp->ctrset; 1713 cfset_ioctl_on(&p); 1714 cpumask_set_cpu(cpu, &rp->mask); 1715 } 1716 } 1717 return 0; 1718 } 1719 1720 /* Hotplug remove of a CPU. Scan through all active processes and clear 1721 * that CPU from the list of CPUs supplied with ioctl(..., START, ...). 1722 * Adjust reference counts. 1723 */ 1724 static int cfset_offline_cpu(unsigned int cpu) 1725 { 1726 struct cfset_call_on_cpu_parm p; 1727 struct cfset_request *rp; 1728 1729 if (!list_empty(&cfset_session.head)) { 1730 list_for_each_entry(rp, &cfset_session.head, node) { 1731 p.sets = rp->ctrset; 1732 cfset_ioctl_off(&p); 1733 cpumask_clear_cpu(cpu, &rp->mask); 1734 } 1735 } 1736 return 0; 1737 } 1738 1739 static void cfdiag_read(struct perf_event *event) 1740 { 1741 } 1742 1743 static int get_authctrsets(void) 1744 { 1745 unsigned long auth = 0; 1746 enum cpumf_ctr_set i; 1747 1748 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 1749 if (cpumf_ctr_info.auth_ctl & cpumf_ctr_ctl[i]) 1750 auth |= cpumf_ctr_ctl[i]; 1751 } 1752 return auth; 1753 } 1754 1755 /* Setup the event. Test for authorized counter sets and only include counter 1756 * sets which are authorized at the time of the setup. Including unauthorized 1757 * counter sets result in specification exception (and panic). 1758 */ 1759 static int cfdiag_event_init2(struct perf_event *event) 1760 { 1761 struct perf_event_attr *attr = &event->attr; 1762 int err = 0; 1763 1764 /* Set sample_period to indicate sampling */ 1765 event->hw.config = attr->config; 1766 event->hw.sample_period = attr->sample_period; 1767 local64_set(&event->hw.period_left, event->hw.sample_period); 1768 local64_set(&event->count, 0); 1769 event->hw.last_period = event->hw.sample_period; 1770 1771 /* Add all authorized counter sets to config_base. The 1772 * the hardware init function is either called per-cpu or just once 1773 * for all CPUS (event->cpu == -1). This depends on the whether 1774 * counting is started for all CPUs or on a per workload base where 1775 * the perf event moves from one CPU to another CPU. 1776 * Checking the authorization on any CPU is fine as the hardware 1777 * applies the same authorization settings to all CPUs. 1778 */ 1779 event->hw.config_base = get_authctrsets(); 1780 1781 /* No authorized counter sets, nothing to count/sample */ 1782 if (!event->hw.config_base) 1783 err = -EINVAL; 1784 1785 return err; 1786 } 1787 1788 static int cfdiag_event_init(struct perf_event *event) 1789 { 1790 struct perf_event_attr *attr = &event->attr; 1791 int err = -ENOENT; 1792 1793 if (event->attr.config != PERF_EVENT_CPUM_CF_DIAG || 1794 event->attr.type != event->pmu->type) 1795 goto out; 1796 1797 /* Raw events are used to access counters directly, 1798 * hence do not permit excludes. 1799 * This event is useless without PERF_SAMPLE_RAW to return counter set 1800 * values as raw data. 1801 */ 1802 if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv || 1803 !(attr->sample_type & (PERF_SAMPLE_CPU | PERF_SAMPLE_RAW))) { 1804 err = -EOPNOTSUPP; 1805 goto out; 1806 } 1807 1808 /* Initialize for using the CPU-measurement counter facility */ 1809 if (cpum_cf_alloc(event->cpu)) 1810 return -ENOMEM; 1811 event->destroy = hw_perf_event_destroy; 1812 1813 err = cfdiag_event_init2(event); 1814 out: 1815 return err; 1816 } 1817 1818 /* Create cf_diag/events/CF_DIAG event sysfs file. This counter is used 1819 * to collect the complete counter sets for a scheduled process. Target 1820 * are complete counter sets attached as raw data to the artificial event. 1821 * This results in complete counter sets available when a process is 1822 * scheduled. Contains the delta of every counter while the process was 1823 * running. 1824 */ 1825 CPUMF_EVENT_ATTR(CF_DIAG, CF_DIAG, PERF_EVENT_CPUM_CF_DIAG); 1826 1827 static struct attribute *cfdiag_events_attr[] = { 1828 CPUMF_EVENT_PTR(CF_DIAG, CF_DIAG), 1829 NULL, 1830 }; 1831 1832 PMU_FORMAT_ATTR(event, "config:0-63"); 1833 1834 static struct attribute *cfdiag_format_attr[] = { 1835 &format_attr_event.attr, 1836 NULL, 1837 }; 1838 1839 static struct attribute_group cfdiag_events_group = { 1840 .name = "events", 1841 .attrs = cfdiag_events_attr, 1842 }; 1843 static struct attribute_group cfdiag_format_group = { 1844 .name = "format", 1845 .attrs = cfdiag_format_attr, 1846 }; 1847 static const struct attribute_group *cfdiag_attr_groups[] = { 1848 &cfdiag_events_group, 1849 &cfdiag_format_group, 1850 NULL, 1851 }; 1852 1853 /* Performance monitoring unit for event CF_DIAG. Since this event 1854 * is also started and stopped via the perf_event_open() system call, use 1855 * the same event enable/disable call back functions. They do not 1856 * have a pointer to the perf_event structure as first parameter. 1857 * 1858 * The functions XXX_add, XXX_del, XXX_start and XXX_stop are also common. 1859 * Reuse them and distinguish the event (always first parameter) via 1860 * 'config' member. 1861 */ 1862 static struct pmu cf_diag = { 1863 .task_ctx_nr = perf_sw_context, 1864 .event_init = cfdiag_event_init, 1865 .pmu_enable = cpumf_pmu_enable, 1866 .pmu_disable = cpumf_pmu_disable, 1867 .add = cpumf_pmu_add, 1868 .del = cpumf_pmu_del, 1869 .start = cpumf_pmu_start, 1870 .stop = cpumf_pmu_stop, 1871 .read = cfdiag_read, 1872 1873 .attr_groups = cfdiag_attr_groups 1874 }; 1875 1876 /* Calculate memory needed to store all counter sets together with header and 1877 * trailer data. This is independent of the counter set authorization which 1878 * can vary depending on the configuration. 1879 */ 1880 static size_t cfdiag_maxsize(struct cpumf_ctr_info *info) 1881 { 1882 size_t max_size = sizeof(struct cf_trailer_entry); 1883 enum cpumf_ctr_set i; 1884 1885 for (i = CPUMF_CTR_SET_BASIC; i < CPUMF_CTR_SET_MAX; ++i) { 1886 size_t size = cpum_cf_read_setsize(i); 1887 1888 if (size) 1889 max_size += size * sizeof(u64) + 1890 sizeof(struct cf_ctrset_entry); 1891 } 1892 return max_size; 1893 } 1894 1895 /* Get the CPU speed, try sampling facility first and CPU attributes second. */ 1896 static void cfdiag_get_cpu_speed(void) 1897 { 1898 unsigned long mhz; 1899 1900 if (cpum_sf_avail()) { /* Sampling facility first */ 1901 struct hws_qsi_info_block si; 1902 1903 memset(&si, 0, sizeof(si)); 1904 if (!qsi(&si)) { 1905 cfdiag_cpu_speed = si.cpu_speed; 1906 return; 1907 } 1908 } 1909 1910 /* Fallback: CPU speed extract static part. Used in case 1911 * CPU Measurement Sampling Facility is turned off. 1912 */ 1913 mhz = __ecag(ECAG_CPU_ATTRIBUTE, 0); 1914 if (mhz != -1UL) 1915 cfdiag_cpu_speed = mhz & 0xffffffff; 1916 } 1917 1918 static int cfset_init(void) 1919 { 1920 size_t need; 1921 int rc; 1922 1923 cfdiag_get_cpu_speed(); 1924 /* Make sure the counter set data fits into predefined buffer. */ 1925 need = cfdiag_maxsize(&cpumf_ctr_info); 1926 if (need > sizeof(((struct cpu_cf_events *)0)->start)) { 1927 pr_err("Insufficient memory for PMU(cpum_cf_diag) need=%zu\n", 1928 need); 1929 return -ENOMEM; 1930 } 1931 1932 rc = misc_register(&cfset_dev); 1933 if (rc) { 1934 pr_err("Registration of /dev/%s failed rc=%i\n", 1935 cfset_dev.name, rc); 1936 goto out; 1937 } 1938 1939 rc = perf_pmu_register(&cf_diag, "cpum_cf_diag", -1); 1940 if (rc) { 1941 misc_deregister(&cfset_dev); 1942 pr_err("Registration of PMU(cpum_cf_diag) failed with rc=%i\n", 1943 rc); 1944 } 1945 out: 1946 return rc; 1947 } 1948 1949 device_initcall(cpumf_pmu_init); 1950