1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * turbostat -- show CPU frequency and C-state residency
4 * on modern Intel and AMD processors.
5 *
6 * Copyright (c) 2010 - 2026 Intel Corporation
7 * Len Brown <len.brown@intel.com>
8 */
9
10 #define _GNU_SOURCE
11 #include MSRHEADER
12
13 // copied from arch/x86/include/asm/cpu_device_id.h
14 #define VFM_MODEL_BIT 0
15 #define VFM_FAMILY_BIT 8
16 #define VFM_VENDOR_BIT 16
17 #define VFM_RSVD_BIT 24
18
19 #define VFM_MODEL_MASK GENMASK(VFM_FAMILY_BIT - 1, VFM_MODEL_BIT)
20 #define VFM_FAMILY_MASK GENMASK(VFM_VENDOR_BIT - 1, VFM_FAMILY_BIT)
21 #define VFM_VENDOR_MASK GENMASK(VFM_RSVD_BIT - 1, VFM_VENDOR_BIT)
22
23 #define VFM_MODEL(vfm) (((vfm) & VFM_MODEL_MASK) >> VFM_MODEL_BIT)
24 #define VFM_FAMILY(vfm) (((vfm) & VFM_FAMILY_MASK) >> VFM_FAMILY_BIT)
25 #define VFM_VENDOR(vfm) (((vfm) & VFM_VENDOR_MASK) >> VFM_VENDOR_BIT)
26
27 #define VFM_MAKE(_vendor, _family, _model) ( \
28 ((_model) << VFM_MODEL_BIT) | \
29 ((_family) << VFM_FAMILY_BIT) | \
30 ((_vendor) << VFM_VENDOR_BIT) \
31 )
32 // end copied section
33
34 #define CPUID_LEAF_MODEL_ID 0x1A
35 #define CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT 24
36
37 #define X86_VENDOR_INTEL 0
38
39 #include INTEL_FAMILY_HEADER
40 #include BUILD_BUG_HEADER
41 #include <stdarg.h>
42 #include <stdio.h>
43 #include <err.h>
44 #include <unistd.h>
45 #include <sys/types.h>
46 #include <sys/wait.h>
47 #include <sys/stat.h>
48 #include <sys/select.h>
49 #include <sys/resource.h>
50 #include <sys/mman.h>
51 #include <fcntl.h>
52 #include <signal.h>
53 #include <sys/time.h>
54 #include <stdlib.h>
55 #include <getopt.h>
56 #include <dirent.h>
57 #include <string.h>
58 #include <ctype.h>
59 #include <sched.h>
60 #include <time.h>
61 #include <cpuid.h>
62 #include <sys/capability.h>
63 #include <errno.h>
64 #include <math.h>
65 #include <linux/perf_event.h>
66 #include <asm/unistd.h>
67 #include <stdbool.h>
68 #include <assert.h>
69 #include <linux/kernel.h>
70 #include <limits.h>
71
72 #define UNUSED(x) (void)(x)
73
74 /*
75 * This list matches the column headers, except
76 * 1. built-in only, the sysfs counters are not here -- we learn of those at run-time
77 * 2. Core and CPU are moved to the end, we can't have strings that contain them
78 * matching on them for --show and --hide.
79 */
80
81 /*
82 * buffer size used by sscanf() for added column names
83 * Usually truncated to 7 characters, but also handles 18 columns for raw 64-bit counters
84 */
85 #define NAME_BYTES 20
86 #define PATH_BYTES 128
87 #define PERF_NAME_BYTES 128
88
89 #define MAX_NOFILE 0x8000
90
91 #define COUNTER_KIND_PERF_PREFIX "perf/"
92 #define COUNTER_KIND_PERF_PREFIX_LEN strlen(COUNTER_KIND_PERF_PREFIX)
93 #define PERF_DEV_NAME_BYTES 32
94 #define PERF_EVT_NAME_BYTES 32
95
96 #define INTEL_ECORE_TYPE 0x20
97 #define INTEL_PCORE_TYPE 0x40
98
99 #define ROUND_UP_TO_PAGE_SIZE(n) (((n) + 0x1000UL-1UL) & ~(0x1000UL-1UL))
100
101 enum counter_scope { SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE };
102 enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC, COUNTER_K2M };
103 enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE };
104 enum counter_source { COUNTER_SOURCE_NONE, COUNTER_SOURCE_PERF, COUNTER_SOURCE_MSR };
105
106 struct perf_counter_info {
107 struct perf_counter_info *next;
108
109 /* How to open the counter / What counter it is. */
110 char device[PERF_DEV_NAME_BYTES];
111 char event[PERF_EVT_NAME_BYTES];
112
113 /* How to show/format the counter. */
114 char name[PERF_NAME_BYTES];
115 unsigned int width;
116 enum counter_scope scope;
117 enum counter_type type;
118 enum counter_format format;
119 double scale;
120
121 /* For reading the counter. */
122 int *fd_perf_per_domain;
123 size_t num_domains;
124 };
125
126 struct sysfs_path {
127 char path[PATH_BYTES];
128 int id;
129 struct sysfs_path *next;
130 };
131
132 struct msr_counter {
133 unsigned int msr_num;
134 char name[NAME_BYTES];
135 struct sysfs_path *sp;
136 unsigned int width;
137 enum counter_type type;
138 enum counter_format format;
139 struct msr_counter *next;
140 unsigned int flags;
141 #define FLAGS_HIDE (1 << 0)
142 #define FLAGS_SHOW (1 << 1)
143 #define SYSFS_PERCPU (1 << 1)
144 };
145 static int use_android_msr_path;
146
147 struct msr_counter bic[] = {
148 { 0x0, "usec", NULL, 0, 0, 0, NULL, 0 },
149 { 0x0, "Time_Of_Day_Seconds", NULL, 0, 0, 0, NULL, 0 },
150 { 0x0, "Package", NULL, 0, 0, 0, NULL, 0 },
151 { 0x0, "Node", NULL, 0, 0, 0, NULL, 0 },
152 { 0x0, "Avg_MHz", NULL, 0, 0, 0, NULL, 0 },
153 { 0x0, "Busy%", NULL, 0, 0, 0, NULL, 0 },
154 { 0x0, "Bzy_MHz", NULL, 0, 0, 0, NULL, 0 },
155 { 0x0, "TSC_MHz", NULL, 0, 0, 0, NULL, 0 },
156 { 0x0, "IRQ", NULL, 0, 0, 0, NULL, 0 },
157 { 0x0, "SMI", NULL, 32, 0, FORMAT_DELTA, NULL, 0 },
158 { 0x0, "cpuidle", NULL, 0, 0, 0, NULL, 0 },
159 { 0x0, "CPU%c1", NULL, 0, 0, 0, NULL, 0 },
160 { 0x0, "CPU%c3", NULL, 0, 0, 0, NULL, 0 },
161 { 0x0, "CPU%c6", NULL, 0, 0, 0, NULL, 0 },
162 { 0x0, "CPU%c7", NULL, 0, 0, 0, NULL, 0 },
163 { 0x0, "ThreadC", NULL, 0, 0, 0, NULL, 0 },
164 { 0x0, "CoreTmp", NULL, 0, 0, 0, NULL, 0 },
165 { 0x0, "CoreCnt", NULL, 0, 0, 0, NULL, 0 },
166 { 0x0, "PkgTmp", NULL, 0, 0, 0, NULL, 0 },
167 { 0x0, "GFX%rc6", NULL, 0, 0, 0, NULL, 0 },
168 { 0x0, "GFXMHz", NULL, 0, 0, 0, NULL, 0 },
169 { 0x0, "Pkg%pc2", NULL, 0, 0, 0, NULL, 0 },
170 { 0x0, "Pkg%pc3", NULL, 0, 0, 0, NULL, 0 },
171 { 0x0, "Pkg%pc6", NULL, 0, 0, 0, NULL, 0 },
172 { 0x0, "Pkg%pc7", NULL, 0, 0, 0, NULL, 0 },
173 { 0x0, "Pkg%pc8", NULL, 0, 0, 0, NULL, 0 },
174 { 0x0, "Pkg%pc9", NULL, 0, 0, 0, NULL, 0 },
175 { 0x0, "Pk%pc10", NULL, 0, 0, 0, NULL, 0 },
176 { 0x0, "CPU%LPI", NULL, 0, 0, 0, NULL, 0 },
177 { 0x0, "SYS%LPI", NULL, 0, 0, 0, NULL, 0 },
178 { 0x0, "PkgWatt", NULL, 0, 0, 0, NULL, 0 },
179 { 0x0, "CorWatt", NULL, 0, 0, 0, NULL, 0 },
180 { 0x0, "GFXWatt", NULL, 0, 0, 0, NULL, 0 },
181 { 0x0, "PkgCnt", NULL, 0, 0, 0, NULL, 0 },
182 { 0x0, "RAMWatt", NULL, 0, 0, 0, NULL, 0 },
183 { 0x0, "PKG_%", NULL, 0, 0, 0, NULL, 0 },
184 { 0x0, "RAM_%", NULL, 0, 0, 0, NULL, 0 },
185 { 0x0, "Pkg_J", NULL, 0, 0, 0, NULL, 0 },
186 { 0x0, "Cor_J", NULL, 0, 0, 0, NULL, 0 },
187 { 0x0, "GFX_J", NULL, 0, 0, 0, NULL, 0 },
188 { 0x0, "RAM_J", NULL, 0, 0, 0, NULL, 0 },
189 { 0x0, "Mod%c6", NULL, 0, 0, 0, NULL, 0 },
190 { 0x0, "Totl%C0", NULL, 0, 0, 0, NULL, 0 },
191 { 0x0, "Any%C0", NULL, 0, 0, 0, NULL, 0 },
192 { 0x0, "GFX%C0", NULL, 0, 0, 0, NULL, 0 },
193 { 0x0, "CPUGFX%", NULL, 0, 0, 0, NULL, 0 },
194 { 0x0, "Module", NULL, 0, 0, 0, NULL, 0 },
195 { 0x0, "Core", NULL, 0, 0, 0, NULL, 0 },
196 { 0x0, "CPU", NULL, 0, 0, 0, NULL, 0 },
197 { 0x0, "APIC", NULL, 0, 0, 0, NULL, 0 },
198 { 0x0, "X2APIC", NULL, 0, 0, 0, NULL, 0 },
199 { 0x0, "Die", NULL, 0, 0, 0, NULL, 0 },
200 { 0x0, "L3", NULL, 0, 0, 0, NULL, 0 },
201 { 0x0, "GFXAMHz", NULL, 0, 0, 0, NULL, 0 },
202 { 0x0, "IPC", NULL, 0, 0, 0, NULL, 0 },
203 { 0x0, "CoreThr", NULL, 0, 0, 0, NULL, 0 },
204 { 0x0, "UncMHz", NULL, 0, 0, 0, NULL, 0 },
205 { 0x0, "SAM%mc6", NULL, 0, 0, 0, NULL, 0 },
206 { 0x0, "SAMMHz", NULL, 0, 0, 0, NULL, 0 },
207 { 0x0, "SAMAMHz", NULL, 0, 0, 0, NULL, 0 },
208 { 0x0, "Die%c6", NULL, 0, 0, 0, NULL, 0 },
209 { 0x0, "SysWatt", NULL, 0, 0, 0, NULL, 0 },
210 { 0x0, "Sys_J", NULL, 0, 0, 0, NULL, 0 },
211 { 0x0, "NMI", NULL, 0, 0, 0, NULL, 0 },
212 { 0x0, "CPU%c1e", NULL, 0, 0, 0, NULL, 0 },
213 { 0x0, "pct_idle", NULL, 0, 0, 0, NULL, 0 },
214 { 0x0, "LLCMRPS", NULL, 0, 0, 0, NULL, 0 },
215 { 0x0, "LLC%hit", NULL, 0, 0, 0, NULL, 0 },
216 { 0x0, "L2MRPS", NULL, 0, 0, 0, NULL, 0 },
217 { 0x0, "L2%hit", NULL, 0, 0, 0, NULL, 0 },
218 };
219
220 /* n.b. bic_names must match the order in bic[], above */
221 enum bic_names {
222 BIC_USEC,
223 BIC_TOD,
224 BIC_Package,
225 BIC_Node,
226 BIC_Avg_MHz,
227 BIC_Busy,
228 BIC_Bzy_MHz,
229 BIC_TSC_MHz,
230 BIC_IRQ,
231 BIC_SMI,
232 BIC_cpuidle,
233 BIC_CPU_c1,
234 BIC_CPU_c3,
235 BIC_CPU_c6,
236 BIC_CPU_c7,
237 BIC_ThreadC,
238 BIC_CoreTmp,
239 BIC_CoreCnt,
240 BIC_PkgTmp,
241 BIC_GFX_rc6,
242 BIC_GFXMHz,
243 BIC_Pkgpc2,
244 BIC_Pkgpc3,
245 BIC_Pkgpc6,
246 BIC_Pkgpc7,
247 BIC_Pkgpc8,
248 BIC_Pkgpc9,
249 BIC_Pkgpc10,
250 BIC_CPU_LPI,
251 BIC_SYS_LPI,
252 BIC_PkgWatt,
253 BIC_CorWatt,
254 BIC_GFXWatt,
255 BIC_PkgCnt,
256 BIC_RAMWatt,
257 BIC_PKG__,
258 BIC_RAM__,
259 BIC_Pkg_J,
260 BIC_Cor_J,
261 BIC_GFX_J,
262 BIC_RAM_J,
263 BIC_Mod_c6,
264 BIC_Totl_c0,
265 BIC_Any_c0,
266 BIC_GFX_c0,
267 BIC_CPUGFX,
268 BIC_Module,
269 BIC_Core,
270 BIC_CPU,
271 BIC_APIC,
272 BIC_X2APIC,
273 BIC_Die,
274 BIC_L3,
275 BIC_GFXACTMHz,
276 BIC_IPC,
277 BIC_CORE_THROT_CNT,
278 BIC_UNCORE_MHZ,
279 BIC_SAM_mc6,
280 BIC_SAMMHz,
281 BIC_SAMACTMHz,
282 BIC_Diec6,
283 BIC_SysWatt,
284 BIC_Sys_J,
285 BIC_NMI,
286 BIC_CPU_c1e,
287 BIC_pct_idle,
288 BIC_LLC_MRPS,
289 BIC_LLC_HIT,
290 BIC_L2_MRPS,
291 BIC_L2_HIT,
292 MAX_BIC
293 };
294
print_bic_set(char * s,cpu_set_t * set)295 void print_bic_set(char *s, cpu_set_t *set)
296 {
297 int i;
298
299 assert(MAX_BIC < CPU_SETSIZE);
300
301 printf("%s:", s);
302
303 for (i = 0; i < MAX_BIC; ++i) {
304
305 if (CPU_ISSET(i, set))
306 printf(" %s", bic[i].name);
307 }
308 putchar('\n');
309 }
310
311 static cpu_set_t bic_group_topology;
312 static cpu_set_t bic_group_thermal_pwr;
313 static cpu_set_t bic_group_frequency;
314 static cpu_set_t bic_group_hw_idle;
315 static cpu_set_t bic_group_sw_idle;
316 static cpu_set_t bic_group_idle;
317 static cpu_set_t bic_group_cache;
318 static cpu_set_t bic_group_other;
319 static cpu_set_t bic_group_disabled_by_default;
320 static cpu_set_t bic_enabled;
321 static cpu_set_t bic_present;
322
323 /* modify */
324 #define BIC_INIT(set) CPU_ZERO(set)
325
326 #define SET_BIC(COUNTER_NUMBER, set) CPU_SET(COUNTER_NUMBER, set)
327 #define CLR_BIC(COUNTER_NUMBER, set) CPU_CLR(COUNTER_NUMBER, set)
328
329 #define BIC_PRESENT(COUNTER_NUMBER) SET_BIC(COUNTER_NUMBER, &bic_present)
330 #define BIC_NOT_PRESENT(COUNTER_NUMBER) CPU_CLR(COUNTER_NUMBER, &bic_present)
331
332 /* test */
333 #define BIC_IS_ENABLED(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_enabled)
334 #define DO_BIC_READ(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_present)
335 #define DO_BIC(COUNTER_NUMBER) (CPU_ISSET(COUNTER_NUMBER, &bic_enabled) && CPU_ISSET(COUNTER_NUMBER, &bic_present))
336
bic_set_all(cpu_set_t * set)337 static void bic_set_all(cpu_set_t *set)
338 {
339 int i;
340
341 assert(MAX_BIC < CPU_SETSIZE);
342
343 for (i = 0; i < MAX_BIC; ++i)
344 SET_BIC(i, set);
345 }
346
347 /*
348 * bic_clear_bits()
349 * clear all the bits from "clr" in "dst"
350 */
bic_clear_bits(cpu_set_t * dst,cpu_set_t * clr)351 static void bic_clear_bits(cpu_set_t *dst, cpu_set_t *clr)
352 {
353 int i;
354
355 assert(MAX_BIC < CPU_SETSIZE);
356
357 for (i = 0; i < MAX_BIC; ++i)
358 if (CPU_ISSET(i, clr))
359 CLR_BIC(i, dst);
360 }
361
bic_groups_init(void)362 static void bic_groups_init(void)
363 {
364 BIC_INIT(&bic_group_topology);
365 SET_BIC(BIC_Package, &bic_group_topology);
366 SET_BIC(BIC_Node, &bic_group_topology);
367 SET_BIC(BIC_CoreCnt, &bic_group_topology);
368 SET_BIC(BIC_PkgCnt, &bic_group_topology);
369 SET_BIC(BIC_Module, &bic_group_topology);
370 SET_BIC(BIC_Core, &bic_group_topology);
371 SET_BIC(BIC_CPU, &bic_group_topology);
372 SET_BIC(BIC_Die, &bic_group_topology);
373 SET_BIC(BIC_L3, &bic_group_topology);
374
375 BIC_INIT(&bic_group_thermal_pwr);
376 SET_BIC(BIC_CoreTmp, &bic_group_thermal_pwr);
377 SET_BIC(BIC_PkgTmp, &bic_group_thermal_pwr);
378 SET_BIC(BIC_PkgWatt, &bic_group_thermal_pwr);
379 SET_BIC(BIC_CorWatt, &bic_group_thermal_pwr);
380 SET_BIC(BIC_GFXWatt, &bic_group_thermal_pwr);
381 SET_BIC(BIC_RAMWatt, &bic_group_thermal_pwr);
382 SET_BIC(BIC_PKG__, &bic_group_thermal_pwr);
383 SET_BIC(BIC_RAM__, &bic_group_thermal_pwr);
384 SET_BIC(BIC_SysWatt, &bic_group_thermal_pwr);
385
386 BIC_INIT(&bic_group_frequency);
387 SET_BIC(BIC_Avg_MHz, &bic_group_frequency);
388 SET_BIC(BIC_Busy, &bic_group_frequency);
389 SET_BIC(BIC_Bzy_MHz, &bic_group_frequency);
390 SET_BIC(BIC_TSC_MHz, &bic_group_frequency);
391 SET_BIC(BIC_GFXMHz, &bic_group_frequency);
392 SET_BIC(BIC_GFXACTMHz, &bic_group_frequency);
393 SET_BIC(BIC_SAMMHz, &bic_group_frequency);
394 SET_BIC(BIC_SAMACTMHz, &bic_group_frequency);
395 SET_BIC(BIC_UNCORE_MHZ, &bic_group_frequency);
396
397 BIC_INIT(&bic_group_hw_idle);
398 SET_BIC(BIC_Busy, &bic_group_hw_idle);
399 SET_BIC(BIC_CPU_c1, &bic_group_hw_idle);
400 SET_BIC(BIC_CPU_c3, &bic_group_hw_idle);
401 SET_BIC(BIC_CPU_c6, &bic_group_hw_idle);
402 SET_BIC(BIC_CPU_c7, &bic_group_hw_idle);
403 SET_BIC(BIC_GFX_rc6, &bic_group_hw_idle);
404 SET_BIC(BIC_Pkgpc2, &bic_group_hw_idle);
405 SET_BIC(BIC_Pkgpc3, &bic_group_hw_idle);
406 SET_BIC(BIC_Pkgpc6, &bic_group_hw_idle);
407 SET_BIC(BIC_Pkgpc7, &bic_group_hw_idle);
408 SET_BIC(BIC_Pkgpc8, &bic_group_hw_idle);
409 SET_BIC(BIC_Pkgpc9, &bic_group_hw_idle);
410 SET_BIC(BIC_Pkgpc10, &bic_group_hw_idle);
411 SET_BIC(BIC_CPU_LPI, &bic_group_hw_idle);
412 SET_BIC(BIC_SYS_LPI, &bic_group_hw_idle);
413 SET_BIC(BIC_Mod_c6, &bic_group_hw_idle);
414 SET_BIC(BIC_Totl_c0, &bic_group_hw_idle);
415 SET_BIC(BIC_Any_c0, &bic_group_hw_idle);
416 SET_BIC(BIC_GFX_c0, &bic_group_hw_idle);
417 SET_BIC(BIC_CPUGFX, &bic_group_hw_idle);
418 SET_BIC(BIC_SAM_mc6, &bic_group_hw_idle);
419 SET_BIC(BIC_Diec6, &bic_group_hw_idle);
420
421 BIC_INIT(&bic_group_sw_idle);
422 SET_BIC(BIC_Busy, &bic_group_sw_idle);
423 SET_BIC(BIC_cpuidle, &bic_group_sw_idle);
424 SET_BIC(BIC_pct_idle, &bic_group_sw_idle);
425
426 BIC_INIT(&bic_group_idle);
427
428 CPU_OR(&bic_group_idle, &bic_group_idle, &bic_group_hw_idle);
429 SET_BIC(BIC_pct_idle, &bic_group_idle);
430
431 BIC_INIT(&bic_group_cache);
432 SET_BIC(BIC_LLC_MRPS, &bic_group_cache);
433 SET_BIC(BIC_LLC_HIT, &bic_group_cache);
434 SET_BIC(BIC_L2_MRPS, &bic_group_cache);
435 SET_BIC(BIC_L2_HIT, &bic_group_cache);
436
437 BIC_INIT(&bic_group_other);
438 SET_BIC(BIC_IRQ, &bic_group_other);
439 SET_BIC(BIC_NMI, &bic_group_other);
440 SET_BIC(BIC_SMI, &bic_group_other);
441 SET_BIC(BIC_ThreadC, &bic_group_other);
442 SET_BIC(BIC_CoreTmp, &bic_group_other);
443 SET_BIC(BIC_IPC, &bic_group_other);
444
445 BIC_INIT(&bic_group_disabled_by_default);
446 SET_BIC(BIC_USEC, &bic_group_disabled_by_default);
447 SET_BIC(BIC_TOD, &bic_group_disabled_by_default);
448 SET_BIC(BIC_cpuidle, &bic_group_disabled_by_default);
449 SET_BIC(BIC_APIC, &bic_group_disabled_by_default);
450 SET_BIC(BIC_X2APIC, &bic_group_disabled_by_default);
451
452 BIC_INIT(&bic_enabled);
453 bic_set_all(&bic_enabled);
454 bic_clear_bits(&bic_enabled, &bic_group_disabled_by_default);
455
456 BIC_INIT(&bic_present);
457 SET_BIC(BIC_USEC, &bic_present);
458 SET_BIC(BIC_TOD, &bic_present);
459 SET_BIC(BIC_cpuidle, &bic_present);
460 SET_BIC(BIC_APIC, &bic_present);
461 SET_BIC(BIC_X2APIC, &bic_present);
462 SET_BIC(BIC_pct_idle, &bic_present);
463 }
464
465 /*
466 * MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit:
467 * If you change the values, note they are used both in comparisons
468 * (>= PCL__7) and to index pkg_cstate_limit_strings[].
469 */
470 #define PCLUKN 0 /* Unknown */
471 #define PCLRSV 1 /* Reserved */
472 #define PCL__0 2 /* PC0 */
473 #define PCL__1 3 /* PC1 */
474 #define PCL__2 4 /* PC2 */
475 #define PCL__3 5 /* PC3 */
476 #define PCL__4 6 /* PC4 */
477 #define PCL__6 7 /* PC6 */
478 #define PCL_6N 8 /* PC6 No Retention */
479 #define PCL_6R 9 /* PC6 Retention */
480 #define PCL__7 10 /* PC7 */
481 #define PCL_7S 11 /* PC7 Shrink */
482 #define PCL__8 12 /* PC8 */
483 #define PCL__9 13 /* PC9 */
484 #define PCL_10 14 /* PC10 */
485 #define PCLUNL 15 /* Unlimited */
486
487 char *proc_stat = "/proc/stat";
488 FILE *outf;
489 int *fd_percpu;
490 int *fd_instr_count_percpu;
491 int *fd_llc_percpu;
492 int *fd_l2_percpu;
493 struct timeval interval_tv = { 5, 0 };
494 struct timespec interval_ts = { 5, 0 };
495
496 unsigned int num_iterations;
497 unsigned int header_iterations;
498 unsigned int debug;
499 unsigned int quiet;
500 unsigned int shown;
501 unsigned int sums_need_wide_columns;
502 unsigned int rapl_joules;
503 unsigned int valid_rapl_msrs;
504 unsigned int summary_only;
505 unsigned int list_header_only;
506 unsigned int dump_only;
507 unsigned int force_load;
508 unsigned int cpuid_has_aperf_mperf;
509 unsigned int cpuid_has_hv;
510 unsigned int has_aperf_access;
511 unsigned int has_epb;
512 unsigned int has_turbo;
513 unsigned int is_hybrid;
514 unsigned int units = 1000000; /* MHz etc */
515 unsigned int genuine_intel;
516 unsigned int authentic_amd;
517 unsigned int hygon_genuine;
518 unsigned int max_level, max_extended_level;
519 unsigned int has_invariant_tsc;
520 unsigned int aperf_mperf_multiplier = 1;
521 double bclk;
522 double base_hz;
523 unsigned int has_base_hz;
524 double tsc_tweak = 1.0;
525 unsigned int show_pkg_only;
526 unsigned int show_core_only;
527 char *output_buffer, *outp;
528 unsigned int do_dts;
529 unsigned int do_ptm;
530 unsigned int do_ipc;
531 unsigned long long cpuidle_cur_cpu_lpi_us;
532 unsigned long long cpuidle_cur_sys_lpi_us;
533 unsigned int tj_max;
534 unsigned int tj_max_override;
535 double rapl_power_units, rapl_time_units;
536 double rapl_dram_energy_units, rapl_energy_units, rapl_psys_energy_units;
537 double rapl_joule_counter_range;
538 unsigned int crystal_hz;
539 unsigned long long tsc_hz;
540 int master_cpu;
541 unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */
542 /* IA32_HWP_REQUEST, IA32_HWP_STATUS */
543 unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */
544 unsigned int has_hwp_activity_window; /* IA32_HWP_REQUEST[bits 41:32] */
545 unsigned int has_hwp_epp; /* IA32_HWP_REQUEST[bits 31:24] */
546 unsigned int has_hwp_pkg; /* IA32_HWP_REQUEST_PKG */
547 unsigned int first_counter_read = 1;
548
549 static struct timeval procsysfs_tv_begin;
550
551 int ignore_stdin;
552 bool no_msr;
553 bool no_perf;
554
555 enum gfx_sysfs_idx {
556 GFX_rc6,
557 GFX_MHz,
558 GFX_ACTMHz,
559 SAM_mc6,
560 SAM_MHz,
561 SAM_ACTMHz,
562 GFX_MAX
563 };
564
565 struct gfx_sysfs_info {
566 FILE *fp;
567 unsigned int val;
568 unsigned long long val_ull;
569 };
570
571 static struct gfx_sysfs_info gfx_info[GFX_MAX];
572
573 int get_msr(int cpu, off_t offset, unsigned long long *msr);
574 int add_counter(unsigned int msr_num, char *path, char *name,
575 unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int package_num);
576
577 /* Model specific support Start */
578
579 /* List of features that may diverge among different platforms */
580 struct platform_features {
581 bool has_msr_misc_feature_control; /* MSR_MISC_FEATURE_CONTROL */
582 bool has_msr_misc_pwr_mgmt; /* MSR_MISC_PWR_MGMT */
583 bool has_nhm_msrs; /* MSR_PLATFORM_INFO, MSR_IA32_TEMPERATURE_TARGET, MSR_SMI_COUNT, MSR_PKG_CST_CONFIG_CONTROL, MSR_IA32_POWER_CTL, TRL MSRs */
584 bool has_config_tdp; /* MSR_CONFIG_TDP_NOMINAL/LEVEL_1/LEVEL_2/CONTROL, MSR_TURBO_ACTIVATION_RATIO */
585 int bclk_freq; /* CPU base clock */
586 int crystal_freq; /* Crystal clock to use when not available from CPUID.15 */
587 int supported_cstates; /* Core cstates and Package cstates supported */
588 int cst_limit; /* MSR_PKG_CST_CONFIG_CONTROL */
589 bool has_cst_auto_convension; /* AUTOMATIC_CSTATE_CONVERSION bit in MSR_PKG_CST_CONFIG_CONTROL */
590 bool has_irtl_msrs; /* MSR_PKGC3/PKGC6/PKGC7/PKGC8/PKGC9/PKGC10_IRTL */
591 bool has_msr_core_c1_res; /* MSR_CORE_C1_RES */
592 bool has_msr_module_c6_res_ms; /* MSR_MODULE_C6_RES_MS */
593 bool has_msr_c6_demotion_policy_config; /* MSR_CC6_DEMOTION_POLICY_CONFIG/MSR_MC6_DEMOTION_POLICY_CONFIG */
594 bool has_msr_atom_pkg_c6_residency; /* MSR_ATOM_PKG_C6_RESIDENCY */
595 bool has_msr_knl_core_c6_residency; /* MSR_KNL_CORE_C6_RESIDENCY */
596 bool has_ext_cst_msrs; /* MSR_PKG_WEIGHTED_CORE_C0_RES/MSR_PKG_ANY_CORE_C0_RES/MSR_PKG_ANY_GFXE_C0_RES/MSR_PKG_BOTH_CORE_GFXE_C0_RES */
597 bool has_cst_prewake_bit; /* Cstate prewake bit in MSR_IA32_POWER_CTL */
598 int trl_msrs; /* MSR_TURBO_RATIO_LIMIT/LIMIT1/LIMIT2/SECONDARY, Atom TRL MSRs */
599 int plr_msrs; /* MSR_CORE/GFX/RING_PERF_LIMIT_REASONS */
600 int plat_rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */
601 bool has_per_core_rapl; /* Indicates cores energy collection is per-core, not per-package. AMD specific for now */
602 bool has_rapl_divisor; /* Divisor for Energy unit raw value from MSR_RAPL_POWER_UNIT */
603 bool has_fixed_rapl_unit; /* Fixed Energy Unit used for DRAM RAPL Domain */
604 bool has_fixed_rapl_psys_unit; /* Fixed Energy Unit used for PSYS RAPL Domain */
605 int rapl_quirk_tdp; /* Hardcoded TDP value when cannot be retrieved from hardware */
606 int tcc_offset_bits; /* TCC Offset bits in MSR_IA32_TEMPERATURE_TARGET */
607 bool enable_tsc_tweak; /* Use CPU Base freq instead of TSC freq for aperf/mperf counter */
608 bool need_perf_multiplier; /* mperf/aperf multiplier */
609 };
610
611 struct platform_data {
612 unsigned int vfm;
613 const struct platform_features *features;
614 };
615
616 /* For BCLK */
617 enum bclk_freq {
618 BCLK_100MHZ = 1,
619 BCLK_133MHZ,
620 BCLK_SLV,
621 };
622
623 #define SLM_BCLK_FREQS 5
624 double slm_freq_table[SLM_BCLK_FREQS] = { 83.3, 100.0, 133.3, 116.7, 80.0 };
625
slm_bclk(void)626 double slm_bclk(void)
627 {
628 unsigned long long msr = 3;
629 unsigned int i;
630 double freq;
631
632 if (get_msr(master_cpu, MSR_FSB_FREQ, &msr))
633 fprintf(outf, "SLM BCLK: unknown\n");
634
635 i = msr & 0xf;
636 if (i >= SLM_BCLK_FREQS) {
637 fprintf(outf, "SLM BCLK[%d] invalid\n", i);
638 i = 3;
639 }
640 freq = slm_freq_table[i];
641
642 if (!quiet)
643 fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq);
644
645 return freq;
646 }
647
648 /* For Package cstate limit */
649 enum package_cstate_limit {
650 CST_LIMIT_NHM = 1,
651 CST_LIMIT_SNB,
652 CST_LIMIT_HSW,
653 CST_LIMIT_SKX,
654 CST_LIMIT_ICX,
655 CST_LIMIT_SLV,
656 CST_LIMIT_AMT,
657 CST_LIMIT_KNL,
658 CST_LIMIT_GMT,
659 };
660
661 /* For Turbo Ratio Limit MSRs */
662 enum turbo_ratio_limit_msrs {
663 TRL_BASE = BIT(0),
664 TRL_LIMIT1 = BIT(1),
665 TRL_LIMIT2 = BIT(2),
666 TRL_ATOM = BIT(3),
667 TRL_KNL = BIT(4),
668 TRL_CORECOUNT = BIT(5),
669 };
670
671 /* For Perf Limit Reason MSRs */
672 enum perf_limit_reason_msrs {
673 PLR_CORE = BIT(0),
674 PLR_GFX = BIT(1),
675 PLR_RING = BIT(2),
676 };
677
678 /* For RAPL MSRs */
679 enum rapl_msrs {
680 RAPL_PKG_POWER_LIMIT = BIT(0), /* 0x610 MSR_PKG_POWER_LIMIT */
681 RAPL_PKG_ENERGY_STATUS = BIT(1), /* 0x611 MSR_PKG_ENERGY_STATUS */
682 RAPL_PKG_PERF_STATUS = BIT(2), /* 0x613 MSR_PKG_PERF_STATUS */
683 RAPL_PKG_POWER_INFO = BIT(3), /* 0x614 MSR_PKG_POWER_INFO */
684 RAPL_DRAM_POWER_LIMIT = BIT(4), /* 0x618 MSR_DRAM_POWER_LIMIT */
685 RAPL_DRAM_ENERGY_STATUS = BIT(5), /* 0x619 MSR_DRAM_ENERGY_STATUS */
686 RAPL_DRAM_PERF_STATUS = BIT(6), /* 0x61b MSR_DRAM_PERF_STATUS */
687 RAPL_DRAM_POWER_INFO = BIT(7), /* 0x61c MSR_DRAM_POWER_INFO */
688 RAPL_CORE_POWER_LIMIT = BIT(8), /* 0x638 MSR_PP0_POWER_LIMIT */
689 RAPL_CORE_ENERGY_STATUS = BIT(9), /* 0x639 MSR_PP0_ENERGY_STATUS */
690 RAPL_CORE_POLICY = BIT(10), /* 0x63a MSR_PP0_POLICY */
691 RAPL_GFX_POWER_LIMIT = BIT(11), /* 0x640 MSR_PP1_POWER_LIMIT */
692 RAPL_GFX_ENERGY_STATUS = BIT(12), /* 0x641 MSR_PP1_ENERGY_STATUS */
693 RAPL_GFX_POLICY = BIT(13), /* 0x642 MSR_PP1_POLICY */
694 RAPL_AMD_PWR_UNIT = BIT(14), /* 0xc0010299 MSR_AMD_RAPL_POWER_UNIT */
695 RAPL_AMD_CORE_ENERGY_STAT = BIT(15), /* 0xc001029a MSR_AMD_CORE_ENERGY_STATUS */
696 RAPL_AMD_PKG_ENERGY_STAT = BIT(16), /* 0xc001029b MSR_AMD_PKG_ENERGY_STATUS */
697 RAPL_PLATFORM_ENERGY_LIMIT = BIT(17), /* 0x64c MSR_PLATFORM_ENERGY_LIMIT */
698 RAPL_PLATFORM_ENERGY_STATUS = BIT(18), /* 0x64d MSR_PLATFORM_ENERGY_STATUS */
699 };
700
701 #define RAPL_PKG (RAPL_PKG_ENERGY_STATUS | RAPL_PKG_POWER_LIMIT)
702 #define RAPL_DRAM (RAPL_DRAM_ENERGY_STATUS | RAPL_DRAM_POWER_LIMIT)
703 #define RAPL_CORE (RAPL_CORE_ENERGY_STATUS | RAPL_CORE_POWER_LIMIT)
704 #define RAPL_GFX (RAPL_GFX_POWER_LIMIT | RAPL_GFX_ENERGY_STATUS)
705 #define RAPL_PSYS (RAPL_PLATFORM_ENERGY_STATUS | RAPL_PLATFORM_ENERGY_LIMIT)
706
707 #define RAPL_PKG_ALL (RAPL_PKG | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO)
708 #define RAPL_DRAM_ALL (RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_DRAM_POWER_INFO)
709 #define RAPL_CORE_ALL (RAPL_CORE | RAPL_CORE_POLICY)
710 #define RAPL_GFX_ALL (RAPL_GFX | RAPL_GFX_POLICY)
711
712 #define RAPL_AMD_F17H (RAPL_AMD_PWR_UNIT | RAPL_AMD_CORE_ENERGY_STAT | RAPL_AMD_PKG_ENERGY_STAT)
713
714 /* For Cstates */
715 enum cstates {
716 CC1 = BIT(0),
717 CC3 = BIT(1),
718 CC6 = BIT(2),
719 CC7 = BIT(3),
720 PC2 = BIT(4),
721 PC3 = BIT(5),
722 PC6 = BIT(6),
723 PC7 = BIT(7),
724 PC8 = BIT(8),
725 PC9 = BIT(9),
726 PC10 = BIT(10),
727 };
728
729 static const struct platform_features nhm_features = {
730 .has_msr_misc_pwr_mgmt = 1,
731 .has_nhm_msrs = 1,
732 .bclk_freq = BCLK_133MHZ,
733 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
734 .cst_limit = CST_LIMIT_NHM,
735 .trl_msrs = TRL_BASE,
736 };
737
738 static const struct platform_features nhx_features = {
739 .has_msr_misc_pwr_mgmt = 1,
740 .has_nhm_msrs = 1,
741 .bclk_freq = BCLK_133MHZ,
742 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
743 .cst_limit = CST_LIMIT_NHM,
744 };
745
746 static const struct platform_features snb_features = {
747 .has_msr_misc_feature_control = 1,
748 .has_msr_misc_pwr_mgmt = 1,
749 .has_nhm_msrs = 1,
750 .bclk_freq = BCLK_100MHZ,
751 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
752 .cst_limit = CST_LIMIT_SNB,
753 .has_irtl_msrs = 1,
754 .trl_msrs = TRL_BASE,
755 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
756 };
757
758 static const struct platform_features snx_features = {
759 .has_msr_misc_feature_control = 1,
760 .has_msr_misc_pwr_mgmt = 1,
761 .has_nhm_msrs = 1,
762 .bclk_freq = BCLK_100MHZ,
763 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
764 .cst_limit = CST_LIMIT_SNB,
765 .has_irtl_msrs = 1,
766 .trl_msrs = TRL_BASE,
767 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
768 };
769
770 static const struct platform_features ivb_features = {
771 .has_msr_misc_feature_control = 1,
772 .has_msr_misc_pwr_mgmt = 1,
773 .has_nhm_msrs = 1,
774 .has_config_tdp = 1,
775 .bclk_freq = BCLK_100MHZ,
776 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
777 .cst_limit = CST_LIMIT_SNB,
778 .has_irtl_msrs = 1,
779 .trl_msrs = TRL_BASE,
780 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
781 };
782
783 static const struct platform_features ivx_features = {
784 .has_msr_misc_feature_control = 1,
785 .has_msr_misc_pwr_mgmt = 1,
786 .has_nhm_msrs = 1,
787 .bclk_freq = BCLK_100MHZ,
788 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
789 .cst_limit = CST_LIMIT_SNB,
790 .has_irtl_msrs = 1,
791 .trl_msrs = TRL_BASE | TRL_LIMIT1,
792 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
793 };
794
795 static const struct platform_features hsw_features = {
796 .has_msr_misc_feature_control = 1,
797 .has_msr_misc_pwr_mgmt = 1,
798 .has_nhm_msrs = 1,
799 .has_config_tdp = 1,
800 .bclk_freq = BCLK_100MHZ,
801 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
802 .cst_limit = CST_LIMIT_HSW,
803 .has_irtl_msrs = 1,
804 .trl_msrs = TRL_BASE,
805 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
806 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
807 };
808
809 static const struct platform_features hsx_features = {
810 .has_msr_misc_feature_control = 1,
811 .has_msr_misc_pwr_mgmt = 1,
812 .has_nhm_msrs = 1,
813 .has_config_tdp = 1,
814 .bclk_freq = BCLK_100MHZ,
815 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
816 .cst_limit = CST_LIMIT_HSW,
817 .has_irtl_msrs = 1,
818 .trl_msrs = TRL_BASE | TRL_LIMIT1 | TRL_LIMIT2,
819 .plr_msrs = PLR_CORE | PLR_RING,
820 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
821 .has_fixed_rapl_unit = 1,
822 };
823
824 static const struct platform_features hswl_features = {
825 .has_msr_misc_feature_control = 1,
826 .has_msr_misc_pwr_mgmt = 1,
827 .has_nhm_msrs = 1,
828 .has_config_tdp = 1,
829 .bclk_freq = BCLK_100MHZ,
830 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
831 .cst_limit = CST_LIMIT_HSW,
832 .has_irtl_msrs = 1,
833 .trl_msrs = TRL_BASE,
834 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
835 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
836 };
837
838 static const struct platform_features hswg_features = {
839 .has_msr_misc_feature_control = 1,
840 .has_msr_misc_pwr_mgmt = 1,
841 .has_nhm_msrs = 1,
842 .has_config_tdp = 1,
843 .bclk_freq = BCLK_100MHZ,
844 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
845 .cst_limit = CST_LIMIT_HSW,
846 .has_irtl_msrs = 1,
847 .trl_msrs = TRL_BASE,
848 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
849 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
850 };
851
852 static const struct platform_features bdw_features = {
853 .has_msr_misc_feature_control = 1,
854 .has_msr_misc_pwr_mgmt = 1,
855 .has_nhm_msrs = 1,
856 .has_config_tdp = 1,
857 .bclk_freq = BCLK_100MHZ,
858 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
859 .cst_limit = CST_LIMIT_HSW,
860 .has_irtl_msrs = 1,
861 .trl_msrs = TRL_BASE,
862 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
863 };
864
865 static const struct platform_features bdwg_features = {
866 .has_msr_misc_feature_control = 1,
867 .has_msr_misc_pwr_mgmt = 1,
868 .has_nhm_msrs = 1,
869 .has_config_tdp = 1,
870 .bclk_freq = BCLK_100MHZ,
871 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
872 .cst_limit = CST_LIMIT_HSW,
873 .has_irtl_msrs = 1,
874 .trl_msrs = TRL_BASE,
875 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
876 };
877
878 static const struct platform_features bdx_features = {
879 .has_msr_misc_feature_control = 1,
880 .has_msr_misc_pwr_mgmt = 1,
881 .has_nhm_msrs = 1,
882 .has_config_tdp = 1,
883 .bclk_freq = BCLK_100MHZ,
884 .supported_cstates = CC1 | CC3 | CC6 | PC2 | PC3 | PC6,
885 .cst_limit = CST_LIMIT_HSW,
886 .has_irtl_msrs = 1,
887 .has_cst_auto_convension = 1,
888 .trl_msrs = TRL_BASE,
889 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
890 .has_fixed_rapl_unit = 1,
891 };
892
893 static const struct platform_features skl_features = {
894 .has_msr_misc_feature_control = 1,
895 .has_msr_misc_pwr_mgmt = 1,
896 .has_nhm_msrs = 1,
897 .has_config_tdp = 1,
898 .bclk_freq = BCLK_100MHZ,
899 .crystal_freq = 24000000,
900 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
901 .cst_limit = CST_LIMIT_HSW,
902 .has_irtl_msrs = 1,
903 .has_ext_cst_msrs = 1,
904 .trl_msrs = TRL_BASE,
905 .tcc_offset_bits = 6,
906 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS,
907 .enable_tsc_tweak = 1,
908 };
909
910 static const struct platform_features cnl_features = {
911 .has_msr_misc_feature_control = 1,
912 .has_msr_misc_pwr_mgmt = 1,
913 .has_nhm_msrs = 1,
914 .has_config_tdp = 1,
915 .bclk_freq = BCLK_100MHZ,
916 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
917 .cst_limit = CST_LIMIT_HSW,
918 .has_irtl_msrs = 1,
919 .has_msr_core_c1_res = 1,
920 .has_ext_cst_msrs = 1,
921 .trl_msrs = TRL_BASE,
922 .tcc_offset_bits = 6,
923 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS,
924 .enable_tsc_tweak = 1,
925 };
926
927 /* Copied from cnl_features, with PC7/PC9 removed */
928 static const struct platform_features adl_features = {
929 .has_msr_misc_feature_control = cnl_features.has_msr_misc_feature_control,
930 .has_msr_misc_pwr_mgmt = cnl_features.has_msr_misc_pwr_mgmt,
931 .has_nhm_msrs = cnl_features.has_nhm_msrs,
932 .has_config_tdp = cnl_features.has_config_tdp,
933 .bclk_freq = cnl_features.bclk_freq,
934 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC8 | PC10,
935 .cst_limit = cnl_features.cst_limit,
936 .has_irtl_msrs = cnl_features.has_irtl_msrs,
937 .has_msr_core_c1_res = cnl_features.has_msr_core_c1_res,
938 .has_ext_cst_msrs = cnl_features.has_ext_cst_msrs,
939 .trl_msrs = cnl_features.trl_msrs,
940 .tcc_offset_bits = cnl_features.tcc_offset_bits,
941 .plat_rapl_msrs = cnl_features.plat_rapl_msrs,
942 .enable_tsc_tweak = cnl_features.enable_tsc_tweak,
943 };
944
945 /* Copied from adl_features, with PC3/PC8 removed */
946 static const struct platform_features lnl_features = {
947 .has_msr_misc_feature_control = adl_features.has_msr_misc_feature_control,
948 .has_msr_misc_pwr_mgmt = adl_features.has_msr_misc_pwr_mgmt,
949 .has_nhm_msrs = adl_features.has_nhm_msrs,
950 .has_config_tdp = adl_features.has_config_tdp,
951 .bclk_freq = adl_features.bclk_freq,
952 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC6 | PC10,
953 .cst_limit = adl_features.cst_limit,
954 .has_irtl_msrs = adl_features.has_irtl_msrs,
955 .has_msr_core_c1_res = adl_features.has_msr_core_c1_res,
956 .has_ext_cst_msrs = adl_features.has_ext_cst_msrs,
957 .trl_msrs = adl_features.trl_msrs,
958 .tcc_offset_bits = adl_features.tcc_offset_bits,
959 .plat_rapl_msrs = adl_features.plat_rapl_msrs,
960 .enable_tsc_tweak = adl_features.enable_tsc_tweak,
961 };
962
963 static const struct platform_features skx_features = {
964 .has_msr_misc_feature_control = 1,
965 .has_msr_misc_pwr_mgmt = 1,
966 .has_nhm_msrs = 1,
967 .has_config_tdp = 1,
968 .bclk_freq = BCLK_100MHZ,
969 .supported_cstates = CC1 | CC6 | PC2 | PC6,
970 .cst_limit = CST_LIMIT_SKX,
971 .has_irtl_msrs = 1,
972 .has_cst_auto_convension = 1,
973 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
974 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
975 .has_fixed_rapl_unit = 1,
976 };
977
978 static const struct platform_features icx_features = {
979 .has_msr_misc_feature_control = 1,
980 .has_msr_misc_pwr_mgmt = 1,
981 .has_nhm_msrs = 1,
982 .has_config_tdp = 1,
983 .bclk_freq = BCLK_100MHZ,
984 .supported_cstates = CC1 | CC6 | PC2 | PC6,
985 .cst_limit = CST_LIMIT_ICX,
986 .has_msr_core_c1_res = 1,
987 .has_irtl_msrs = 1,
988 .has_cst_prewake_bit = 1,
989 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
990 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
991 .has_fixed_rapl_unit = 1,
992 };
993
994 static const struct platform_features spr_features = {
995 .has_msr_misc_feature_control = 1,
996 .has_msr_misc_pwr_mgmt = 1,
997 .has_nhm_msrs = 1,
998 .has_config_tdp = 1,
999 .bclk_freq = BCLK_100MHZ,
1000 .supported_cstates = CC1 | CC6 | PC2 | PC6,
1001 .cst_limit = CST_LIMIT_SKX,
1002 .has_msr_core_c1_res = 1,
1003 .has_irtl_msrs = 1,
1004 .has_cst_prewake_bit = 1,
1005 .has_fixed_rapl_psys_unit = 1,
1006 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1007 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1008 };
1009
1010 static const struct platform_features dmr_features = {
1011 .has_msr_misc_feature_control = spr_features.has_msr_misc_feature_control,
1012 .has_msr_misc_pwr_mgmt = spr_features.has_msr_misc_pwr_mgmt,
1013 .has_nhm_msrs = spr_features.has_nhm_msrs,
1014 .bclk_freq = spr_features.bclk_freq,
1015 .supported_cstates = spr_features.supported_cstates,
1016 .cst_limit = spr_features.cst_limit,
1017 .has_msr_core_c1_res = spr_features.has_msr_core_c1_res,
1018 .has_cst_prewake_bit = spr_features.has_cst_prewake_bit,
1019 .has_fixed_rapl_psys_unit = spr_features.has_fixed_rapl_psys_unit,
1020 .trl_msrs = spr_features.trl_msrs,
1021 .has_msr_module_c6_res_ms = 1, /* DMR has Dual-Core-Module and MC6 MSR */
1022 .plat_rapl_msrs = 0, /* DMR does not have RAPL MSRs */
1023 .plr_msrs = 0, /* DMR does not have PLR MSRs */
1024 .has_irtl_msrs = 0, /* DMR does not have IRTL MSRs */
1025 .has_config_tdp = 0, /* DMR does not have CTDP MSRs */
1026 };
1027
1028 static const struct platform_features srf_features = {
1029 .has_msr_misc_feature_control = 1,
1030 .has_msr_misc_pwr_mgmt = 1,
1031 .has_nhm_msrs = 1,
1032 .has_config_tdp = 1,
1033 .bclk_freq = BCLK_100MHZ,
1034 .supported_cstates = CC1 | CC6 | PC2 | PC6,
1035 .cst_limit = CST_LIMIT_SKX,
1036 .has_msr_core_c1_res = 1,
1037 .has_msr_module_c6_res_ms = 1,
1038 .has_irtl_msrs = 1,
1039 .has_cst_prewake_bit = 1,
1040 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1041 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1042 };
1043
1044 static const struct platform_features grr_features = {
1045 .has_msr_misc_feature_control = 1,
1046 .has_msr_misc_pwr_mgmt = 1,
1047 .has_nhm_msrs = 1,
1048 .has_config_tdp = 1,
1049 .bclk_freq = BCLK_100MHZ,
1050 .supported_cstates = CC1 | CC6,
1051 .cst_limit = CST_LIMIT_SKX,
1052 .has_msr_core_c1_res = 1,
1053 .has_msr_module_c6_res_ms = 1,
1054 .has_irtl_msrs = 1,
1055 .has_cst_prewake_bit = 1,
1056 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1057 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1058 };
1059
1060 static const struct platform_features slv_features = {
1061 .has_nhm_msrs = 1,
1062 .bclk_freq = BCLK_SLV,
1063 .supported_cstates = CC1 | CC6 | PC6,
1064 .cst_limit = CST_LIMIT_SLV,
1065 .has_msr_core_c1_res = 1,
1066 .has_msr_module_c6_res_ms = 1,
1067 .has_msr_c6_demotion_policy_config = 1,
1068 .has_msr_atom_pkg_c6_residency = 1,
1069 .trl_msrs = TRL_ATOM,
1070 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE,
1071 .has_rapl_divisor = 1,
1072 .rapl_quirk_tdp = 30,
1073 };
1074
1075 static const struct platform_features slvd_features = {
1076 .has_msr_misc_pwr_mgmt = 1,
1077 .has_nhm_msrs = 1,
1078 .bclk_freq = BCLK_SLV,
1079 .supported_cstates = CC1 | CC6 | PC3 | PC6,
1080 .cst_limit = CST_LIMIT_SLV,
1081 .has_msr_atom_pkg_c6_residency = 1,
1082 .trl_msrs = TRL_BASE,
1083 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE,
1084 .rapl_quirk_tdp = 30,
1085 };
1086
1087 static const struct platform_features amt_features = {
1088 .has_nhm_msrs = 1,
1089 .bclk_freq = BCLK_133MHZ,
1090 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
1091 .cst_limit = CST_LIMIT_AMT,
1092 .trl_msrs = TRL_BASE,
1093 };
1094
1095 static const struct platform_features gmt_features = {
1096 .has_msr_misc_pwr_mgmt = 1,
1097 .has_nhm_msrs = 1,
1098 .bclk_freq = BCLK_100MHZ,
1099 .crystal_freq = 19200000,
1100 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1101 .cst_limit = CST_LIMIT_GMT,
1102 .has_irtl_msrs = 1,
1103 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1104 .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
1105 };
1106
1107 static const struct platform_features gmtd_features = {
1108 .has_msr_misc_pwr_mgmt = 1,
1109 .has_nhm_msrs = 1,
1110 .bclk_freq = BCLK_100MHZ,
1111 .crystal_freq = 25000000,
1112 .supported_cstates = CC1 | CC6 | PC2 | PC6,
1113 .cst_limit = CST_LIMIT_GMT,
1114 .has_irtl_msrs = 1,
1115 .has_msr_core_c1_res = 1,
1116 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1117 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS,
1118 };
1119
1120 static const struct platform_features gmtp_features = {
1121 .has_msr_misc_pwr_mgmt = 1,
1122 .has_nhm_msrs = 1,
1123 .bclk_freq = BCLK_100MHZ,
1124 .crystal_freq = 19200000,
1125 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1126 .cst_limit = CST_LIMIT_GMT,
1127 .has_irtl_msrs = 1,
1128 .trl_msrs = TRL_BASE,
1129 .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
1130 };
1131
1132 static const struct platform_features tmt_features = {
1133 .has_msr_misc_pwr_mgmt = 1,
1134 .has_nhm_msrs = 1,
1135 .bclk_freq = BCLK_100MHZ,
1136 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1137 .cst_limit = CST_LIMIT_GMT,
1138 .has_irtl_msrs = 1,
1139 .trl_msrs = TRL_BASE,
1140 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX,
1141 .enable_tsc_tweak = 1,
1142 };
1143
1144 static const struct platform_features tmtd_features = {
1145 .has_msr_misc_pwr_mgmt = 1,
1146 .has_nhm_msrs = 1,
1147 .bclk_freq = BCLK_100MHZ,
1148 .supported_cstates = CC1 | CC6,
1149 .cst_limit = CST_LIMIT_GMT,
1150 .has_irtl_msrs = 1,
1151 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1152 .plat_rapl_msrs = RAPL_PKG_ALL,
1153 };
1154
1155 static const struct platform_features knl_features = {
1156 .has_msr_misc_pwr_mgmt = 1,
1157 .has_nhm_msrs = 1,
1158 .has_config_tdp = 1,
1159 .bclk_freq = BCLK_100MHZ,
1160 .supported_cstates = CC1 | CC6 | PC3 | PC6,
1161 .cst_limit = CST_LIMIT_KNL,
1162 .has_msr_knl_core_c6_residency = 1,
1163 .trl_msrs = TRL_KNL,
1164 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
1165 .has_fixed_rapl_unit = 1,
1166 .need_perf_multiplier = 1,
1167 };
1168
1169 static const struct platform_features default_features = {
1170 };
1171
1172 static const struct platform_features amd_features_with_rapl = {
1173 .plat_rapl_msrs = RAPL_AMD_F17H,
1174 .has_per_core_rapl = 1,
1175 .rapl_quirk_tdp = 280, /* This is the max stock TDP of HEDT/Server Fam17h+ chips */
1176 };
1177
1178 static const struct platform_data turbostat_pdata[] = {
1179 { INTEL_NEHALEM, &nhm_features },
1180 { INTEL_NEHALEM_G, &nhm_features },
1181 { INTEL_NEHALEM_EP, &nhm_features },
1182 { INTEL_NEHALEM_EX, &nhx_features },
1183 { INTEL_WESTMERE, &nhm_features },
1184 { INTEL_WESTMERE_EP, &nhm_features },
1185 { INTEL_WESTMERE_EX, &nhx_features },
1186 { INTEL_SANDYBRIDGE, &snb_features },
1187 { INTEL_SANDYBRIDGE_X, &snx_features },
1188 { INTEL_IVYBRIDGE, &ivb_features },
1189 { INTEL_IVYBRIDGE_X, &ivx_features },
1190 { INTEL_HASWELL, &hsw_features },
1191 { INTEL_HASWELL_X, &hsx_features },
1192 { INTEL_HASWELL_L, &hswl_features },
1193 { INTEL_HASWELL_G, &hswg_features },
1194 { INTEL_BROADWELL, &bdw_features },
1195 { INTEL_BROADWELL_G, &bdwg_features },
1196 { INTEL_BROADWELL_X, &bdx_features },
1197 { INTEL_BROADWELL_D, &bdx_features },
1198 { INTEL_SKYLAKE_L, &skl_features },
1199 { INTEL_SKYLAKE, &skl_features },
1200 { INTEL_SKYLAKE_X, &skx_features },
1201 { INTEL_KABYLAKE_L, &skl_features },
1202 { INTEL_KABYLAKE, &skl_features },
1203 { INTEL_COMETLAKE, &skl_features },
1204 { INTEL_COMETLAKE_L, &skl_features },
1205 { INTEL_CANNONLAKE_L, &cnl_features },
1206 { INTEL_ICELAKE_X, &icx_features },
1207 { INTEL_ICELAKE_D, &icx_features },
1208 { INTEL_ICELAKE_L, &cnl_features },
1209 { INTEL_ICELAKE_NNPI, &cnl_features },
1210 { INTEL_ROCKETLAKE, &cnl_features },
1211 { INTEL_TIGERLAKE_L, &cnl_features },
1212 { INTEL_TIGERLAKE, &cnl_features },
1213 { INTEL_SAPPHIRERAPIDS_X, &spr_features },
1214 { INTEL_EMERALDRAPIDS_X, &spr_features },
1215 { INTEL_GRANITERAPIDS_X, &spr_features },
1216 { INTEL_GRANITERAPIDS_D, &spr_features },
1217 { INTEL_DIAMONDRAPIDS_X, &dmr_features },
1218 { INTEL_LAKEFIELD, &cnl_features },
1219 { INTEL_ALDERLAKE, &adl_features },
1220 { INTEL_ALDERLAKE_L, &adl_features },
1221 { INTEL_RAPTORLAKE, &adl_features },
1222 { INTEL_RAPTORLAKE_P, &adl_features },
1223 { INTEL_RAPTORLAKE_S, &adl_features },
1224 { INTEL_BARTLETTLAKE, &adl_features },
1225 { INTEL_METEORLAKE, &adl_features },
1226 { INTEL_METEORLAKE_L, &adl_features },
1227 { INTEL_ARROWLAKE_H, &adl_features },
1228 { INTEL_ARROWLAKE_U, &adl_features },
1229 { INTEL_ARROWLAKE, &adl_features },
1230 { INTEL_LUNARLAKE_M, &lnl_features },
1231 { INTEL_PANTHERLAKE_L, &lnl_features },
1232 { INTEL_NOVALAKE, &lnl_features },
1233 { INTEL_NOVALAKE_L, &lnl_features },
1234 { INTEL_WILDCATLAKE_L, &lnl_features },
1235 { INTEL_ATOM_SILVERMONT, &slv_features },
1236 { INTEL_ATOM_SILVERMONT_D, &slvd_features },
1237 { INTEL_ATOM_AIRMONT, &amt_features },
1238 { INTEL_ATOM_GOLDMONT, &gmt_features },
1239 { INTEL_ATOM_GOLDMONT_D, &gmtd_features },
1240 { INTEL_ATOM_GOLDMONT_PLUS, &gmtp_features },
1241 { INTEL_ATOM_TREMONT_D, &tmtd_features },
1242 { INTEL_ATOM_TREMONT, &tmt_features },
1243 { INTEL_ATOM_TREMONT_L, &tmt_features },
1244 { INTEL_ATOM_GRACEMONT, &adl_features },
1245 { INTEL_ATOM_CRESTMONT_X, &srf_features },
1246 { INTEL_ATOM_CRESTMONT, &grr_features },
1247 { INTEL_ATOM_DARKMONT_X, &srf_features },
1248 { INTEL_XEON_PHI_KNL, &knl_features },
1249 { INTEL_XEON_PHI_KNM, &knl_features },
1250 /*
1251 * Missing support for
1252 * INTEL_ICELAKE
1253 * INTEL_ATOM_SILVERMONT_MID
1254 * INTEL_ATOM_SILVERMONT_MID2
1255 * INTEL_ATOM_AIRMONT_NP
1256 */
1257 { 0, NULL },
1258 };
1259
1260 struct {
1261 unsigned int uniform;
1262 unsigned int pcore;
1263 unsigned int ecore;
1264 unsigned int lcore;
1265 } perf_pmu_types;
1266
1267 /*
1268 * Events are enumerated in https://github.com/intel/perfmon
1269 * and tools/perf/pmu-events/arch/x86/.../cache.json
1270 */
1271 struct perf_l2_events {
1272 unsigned long long refs; /* L2_REQUEST.ALL */
1273 unsigned long long hits; /* L2_REQUEST.HIT */
1274 };
1275
1276 struct perf_model_support {
1277 unsigned int vfm;
1278 struct perf_l2_events first;
1279 struct perf_l2_events second;
1280 struct perf_l2_events third;
1281 } *perf_model_support;
1282
1283 /* Perf Cache Events */
1284 #define PCE(ext_umask, umask) (((unsigned long long) ext_umask) << 40 | umask << 8 | 0x24)
1285
1286 /*
1287 * Enumerate up to three perf CPU PMU's in a system.
1288 * The first, second, and third columns are populated without skipping, describing
1289 * pcore, ecore, lcore PMUs, in order, if present. (The associated PMU "type" field is
1290 * read from sysfs in all cases.) Eg.
1291 *
1292 * non-hybrid:
1293 * GNR: pcore, {}, {}
1294 * ADL-N: ecore, {}, {}
1295 * hybrid:
1296 * MTL: pcore, ecore, {}%
1297 * ARL-H: pcore, ecore, lcore
1298 * LNL: ecore, ecore%%, {}
1299 *
1300 * % MTL physical lcore share architecture and PMU with ecore, and are thus not enumerated separately.
1301 * %% LNL physical lcore is enumerated by perf as ecore
1302 */
1303 static struct perf_model_support turbostat_perf_model_support[] = {
1304 { INTEL_SAPPHIRERAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} },
1305 { INTEL_EMERALDRAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} },
1306 { INTEL_GRANITERAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} },
1307 { INTEL_GRANITERAPIDS_D, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} },
1308 { INTEL_DIAMONDRAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, {}, {} },
1309
1310 { INTEL_ATOM_GRACEMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* ADL-N */
1311 { INTEL_ATOM_CRESTMONT_X, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* SRF */
1312 { INTEL_ATOM_CRESTMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* GRR */
1313 { INTEL_ATOM_DARKMONT_X, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {}, {} }, /* CWF */
1314
1315 { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1316 { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1317 { INTEL_ALDERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1318 { INTEL_RAPTORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1319 { INTEL_RAPTORLAKE_P, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1320 { INTEL_RAPTORLAKE_S, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1321 { INTEL_METEORLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1322 { INTEL_METEORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1323 { INTEL_ARROWLAKE_U, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1324
1325 { INTEL_LUNARLAKE_M, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, {} },
1326 { INTEL_ARROWLAKE_H, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)} },
1327 { INTEL_ARROWLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, {} },
1328
1329 { INTEL_PANTHERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} },
1330 { INTEL_WILDCATLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} },
1331
1332 { INTEL_NOVALAKE, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} },
1333 { INTEL_NOVALAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} },
1334
1335 { 0, {}, {}, {} }
1336 };
1337
1338 static const struct platform_features *platform;
1339
probe_platform_features(unsigned int family,unsigned int model)1340 void probe_platform_features(unsigned int family, unsigned int model)
1341 {
1342 int i;
1343
1344 if (authentic_amd || hygon_genuine) {
1345 /* fallback to default features on unsupported models */
1346 force_load++;
1347 if (max_extended_level >= 0x80000007) {
1348 unsigned int eax, ebx, ecx, edx;
1349
1350 __cpuid(0x80000007, eax, ebx, ecx, edx);
1351 /* RAPL (Fam 17h+) */
1352 if ((edx & (1 << 14)) && family >= 0x17)
1353 platform = &amd_features_with_rapl;
1354 }
1355 goto end;
1356 }
1357
1358 if (!genuine_intel)
1359 goto end;
1360
1361 for (i = 0; turbostat_pdata[i].features; i++) {
1362 if (VFM_FAMILY(turbostat_pdata[i].vfm) == family && VFM_MODEL(turbostat_pdata[i].vfm) == model) {
1363 platform = turbostat_pdata[i].features;
1364 return;
1365 }
1366 }
1367
1368 end:
1369 if (force_load && !platform) {
1370 fprintf(outf, "Forced to run on unsupported platform!\n");
1371 platform = &default_features;
1372 }
1373
1374 if (platform)
1375 return;
1376
1377 fprintf(stderr, "Unsupported platform detected.\n\tSee RUN THE LATEST VERSION on turbostat(8)\n");
1378 exit(1);
1379 }
1380
init_perf_model_support(unsigned int family,unsigned int model)1381 void init_perf_model_support(unsigned int family, unsigned int model)
1382 {
1383 int i;
1384
1385 if (!genuine_intel)
1386 return;
1387
1388 for (i = 0; turbostat_perf_model_support[i].vfm; i++) {
1389 if (VFM_FAMILY(turbostat_perf_model_support[i].vfm) == family && VFM_MODEL(turbostat_perf_model_support[i].vfm) == model) {
1390 perf_model_support = &turbostat_perf_model_support[i];
1391 return;
1392 }
1393 }
1394 }
1395
1396 /* Model specific support End */
1397
1398 #define TJMAX_DEFAULT 100
1399
1400 /* MSRs that are not yet in the kernel-provided header. */
1401 #define MSR_RAPL_PWR_UNIT 0xc0010299
1402 #define MSR_CORE_ENERGY_STAT 0xc001029a
1403 #define MSR_PKG_ENERGY_STAT 0xc001029b
1404
1405 #define MAX(a, b) ((a) > (b) ? (a) : (b))
1406
1407 int backwards_count;
1408 char *progname;
1409
1410 #define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */
1411 cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset;
1412 cpu_set_t *perf_pcore_set, *perf_ecore_set, *perf_lcore_set;
1413 size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size;
1414 #define MAX_ADDED_THREAD_COUNTERS 24
1415 #define MAX_ADDED_CORE_COUNTERS 8
1416 #define MAX_ADDED_PACKAGE_COUNTERS 16
1417 #define PMT_MAX_ADDED_THREAD_COUNTERS 24
1418 #define PMT_MAX_ADDED_CORE_COUNTERS 8
1419 #define PMT_MAX_ADDED_PACKAGE_COUNTERS 16
1420 #define BITMASK_SIZE 32
1421
1422 #define ZERO_ARRAY(arr) (memset(arr, 0, sizeof(arr)) + __must_be_array(arr))
1423
1424 /* Indexes used to map data read from perf and MSRs into global variables */
1425 enum rapl_rci_index {
1426 RAPL_RCI_INDEX_ENERGY_PKG = 0,
1427 RAPL_RCI_INDEX_ENERGY_CORES = 1,
1428 RAPL_RCI_INDEX_DRAM = 2,
1429 RAPL_RCI_INDEX_GFX = 3,
1430 RAPL_RCI_INDEX_PKG_PERF_STATUS = 4,
1431 RAPL_RCI_INDEX_DRAM_PERF_STATUS = 5,
1432 RAPL_RCI_INDEX_CORE_ENERGY = 6,
1433 RAPL_RCI_INDEX_ENERGY_PLATFORM = 7,
1434 NUM_RAPL_COUNTERS,
1435 };
1436
1437 enum rapl_unit {
1438 RAPL_UNIT_INVALID,
1439 RAPL_UNIT_JOULES,
1440 RAPL_UNIT_WATTS,
1441 };
1442
1443 struct rapl_counter_info_t {
1444 unsigned long long data[NUM_RAPL_COUNTERS];
1445 enum counter_source source[NUM_RAPL_COUNTERS];
1446 unsigned long long flags[NUM_RAPL_COUNTERS];
1447 double scale[NUM_RAPL_COUNTERS];
1448 enum rapl_unit unit[NUM_RAPL_COUNTERS];
1449 unsigned long long msr[NUM_RAPL_COUNTERS];
1450 unsigned long long msr_mask[NUM_RAPL_COUNTERS];
1451 int msr_shift[NUM_RAPL_COUNTERS];
1452
1453 int fd_perf;
1454 };
1455
1456 /* struct rapl_counter_info_t for each RAPL domain */
1457 struct rapl_counter_info_t *rapl_counter_info_perdomain;
1458 unsigned int rapl_counter_info_perdomain_size;
1459
1460 #define RAPL_COUNTER_FLAG_PLATFORM_COUNTER (1u << 0)
1461 #define RAPL_COUNTER_FLAG_USE_MSR_SUM (1u << 1)
1462
1463 struct rapl_counter_arch_info {
1464 int feature_mask; /* Mask for testing if the counter is supported on host */
1465 const char *perf_subsys;
1466 const char *perf_name;
1467 unsigned long long msr;
1468 unsigned long long msr_mask;
1469 int msr_shift; /* Positive mean shift right, negative mean shift left */
1470 double *platform_rapl_msr_scale; /* Scale applied to values read by MSR (platform dependent, filled at runtime) */
1471 unsigned int rci_index; /* Maps data from perf counters to global variables */
1472 unsigned int bic_number;
1473 double compat_scale; /* Some counters require constant scaling to be in the same range as other, similar ones */
1474 unsigned long long flags;
1475 };
1476
1477 static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = {
1478 {
1479 .feature_mask = RAPL_PKG,
1480 .perf_subsys = "power",
1481 .perf_name = "energy-pkg",
1482 .msr = MSR_PKG_ENERGY_STATUS,
1483 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1484 .msr_shift = 0,
1485 .platform_rapl_msr_scale = &rapl_energy_units,
1486 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1487 .bic_number = BIC_PkgWatt,
1488 .compat_scale = 1.0,
1489 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1490 },
1491 {
1492 .feature_mask = RAPL_PKG,
1493 .perf_subsys = "power",
1494 .perf_name = "energy-pkg",
1495 .msr = MSR_PKG_ENERGY_STATUS,
1496 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1497 .msr_shift = 0,
1498 .platform_rapl_msr_scale = &rapl_energy_units,
1499 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1500 .bic_number = BIC_Pkg_J,
1501 .compat_scale = 1.0,
1502 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1503 },
1504 {
1505 .feature_mask = RAPL_AMD_F17H,
1506 .perf_subsys = "power",
1507 .perf_name = "energy-pkg",
1508 .msr = MSR_PKG_ENERGY_STAT,
1509 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1510 .msr_shift = 0,
1511 .platform_rapl_msr_scale = &rapl_energy_units,
1512 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1513 .bic_number = BIC_PkgWatt,
1514 .compat_scale = 1.0,
1515 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1516 },
1517 {
1518 .feature_mask = RAPL_AMD_F17H,
1519 .perf_subsys = "power",
1520 .perf_name = "energy-pkg",
1521 .msr = MSR_PKG_ENERGY_STAT,
1522 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1523 .msr_shift = 0,
1524 .platform_rapl_msr_scale = &rapl_energy_units,
1525 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1526 .bic_number = BIC_Pkg_J,
1527 .compat_scale = 1.0,
1528 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1529 },
1530 {
1531 .feature_mask = RAPL_CORE_ENERGY_STATUS,
1532 .perf_subsys = "power",
1533 .perf_name = "energy-cores",
1534 .msr = MSR_PP0_ENERGY_STATUS,
1535 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1536 .msr_shift = 0,
1537 .platform_rapl_msr_scale = &rapl_energy_units,
1538 .rci_index = RAPL_RCI_INDEX_ENERGY_CORES,
1539 .bic_number = BIC_CorWatt,
1540 .compat_scale = 1.0,
1541 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1542 },
1543 {
1544 .feature_mask = RAPL_CORE_ENERGY_STATUS,
1545 .perf_subsys = "power",
1546 .perf_name = "energy-cores",
1547 .msr = MSR_PP0_ENERGY_STATUS,
1548 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1549 .msr_shift = 0,
1550 .platform_rapl_msr_scale = &rapl_energy_units,
1551 .rci_index = RAPL_RCI_INDEX_ENERGY_CORES,
1552 .bic_number = BIC_Cor_J,
1553 .compat_scale = 1.0,
1554 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1555 },
1556 {
1557 .feature_mask = RAPL_DRAM,
1558 .perf_subsys = "power",
1559 .perf_name = "energy-ram",
1560 .msr = MSR_DRAM_ENERGY_STATUS,
1561 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1562 .msr_shift = 0,
1563 .platform_rapl_msr_scale = &rapl_dram_energy_units,
1564 .rci_index = RAPL_RCI_INDEX_DRAM,
1565 .bic_number = BIC_RAMWatt,
1566 .compat_scale = 1.0,
1567 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1568 },
1569 {
1570 .feature_mask = RAPL_DRAM,
1571 .perf_subsys = "power",
1572 .perf_name = "energy-ram",
1573 .msr = MSR_DRAM_ENERGY_STATUS,
1574 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1575 .msr_shift = 0,
1576 .platform_rapl_msr_scale = &rapl_dram_energy_units,
1577 .rci_index = RAPL_RCI_INDEX_DRAM,
1578 .bic_number = BIC_RAM_J,
1579 .compat_scale = 1.0,
1580 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1581 },
1582 {
1583 .feature_mask = RAPL_GFX,
1584 .perf_subsys = "power",
1585 .perf_name = "energy-gpu",
1586 .msr = MSR_PP1_ENERGY_STATUS,
1587 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1588 .msr_shift = 0,
1589 .platform_rapl_msr_scale = &rapl_energy_units,
1590 .rci_index = RAPL_RCI_INDEX_GFX,
1591 .bic_number = BIC_GFXWatt,
1592 .compat_scale = 1.0,
1593 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1594 },
1595 {
1596 .feature_mask = RAPL_GFX,
1597 .perf_subsys = "power",
1598 .perf_name = "energy-gpu",
1599 .msr = MSR_PP1_ENERGY_STATUS,
1600 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1601 .msr_shift = 0,
1602 .platform_rapl_msr_scale = &rapl_energy_units,
1603 .rci_index = RAPL_RCI_INDEX_GFX,
1604 .bic_number = BIC_GFX_J,
1605 .compat_scale = 1.0,
1606 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1607 },
1608 {
1609 .feature_mask = RAPL_PKG_PERF_STATUS,
1610 .perf_subsys = NULL,
1611 .perf_name = NULL,
1612 .msr = MSR_PKG_PERF_STATUS,
1613 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1614 .msr_shift = 0,
1615 .platform_rapl_msr_scale = &rapl_time_units,
1616 .rci_index = RAPL_RCI_INDEX_PKG_PERF_STATUS,
1617 .bic_number = BIC_PKG__,
1618 .compat_scale = 100.0,
1619 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1620 },
1621 {
1622 .feature_mask = RAPL_DRAM_PERF_STATUS,
1623 .perf_subsys = NULL,
1624 .perf_name = NULL,
1625 .msr = MSR_DRAM_PERF_STATUS,
1626 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1627 .msr_shift = 0,
1628 .platform_rapl_msr_scale = &rapl_time_units,
1629 .rci_index = RAPL_RCI_INDEX_DRAM_PERF_STATUS,
1630 .bic_number = BIC_RAM__,
1631 .compat_scale = 100.0,
1632 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1633 },
1634 {
1635 .feature_mask = RAPL_AMD_F17H,
1636 .perf_subsys = NULL,
1637 .perf_name = NULL,
1638 .msr = MSR_CORE_ENERGY_STAT,
1639 .msr_mask = 0xFFFFFFFF,
1640 .msr_shift = 0,
1641 .platform_rapl_msr_scale = &rapl_energy_units,
1642 .rci_index = RAPL_RCI_INDEX_CORE_ENERGY,
1643 .bic_number = BIC_CorWatt,
1644 .compat_scale = 1.0,
1645 .flags = 0,
1646 },
1647 {
1648 .feature_mask = RAPL_AMD_F17H,
1649 .perf_subsys = NULL,
1650 .perf_name = NULL,
1651 .msr = MSR_CORE_ENERGY_STAT,
1652 .msr_mask = 0xFFFFFFFF,
1653 .msr_shift = 0,
1654 .platform_rapl_msr_scale = &rapl_energy_units,
1655 .rci_index = RAPL_RCI_INDEX_CORE_ENERGY,
1656 .bic_number = BIC_Cor_J,
1657 .compat_scale = 1.0,
1658 .flags = 0,
1659 },
1660 {
1661 .feature_mask = RAPL_PSYS,
1662 .perf_subsys = "power",
1663 .perf_name = "energy-psys",
1664 .msr = MSR_PLATFORM_ENERGY_STATUS,
1665 .msr_mask = 0x00000000FFFFFFFF,
1666 .msr_shift = 0,
1667 .platform_rapl_msr_scale = &rapl_psys_energy_units,
1668 .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM,
1669 .bic_number = BIC_SysWatt,
1670 .compat_scale = 1.0,
1671 .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM,
1672 },
1673 {
1674 .feature_mask = RAPL_PSYS,
1675 .perf_subsys = "power",
1676 .perf_name = "energy-psys",
1677 .msr = MSR_PLATFORM_ENERGY_STATUS,
1678 .msr_mask = 0x00000000FFFFFFFF,
1679 .msr_shift = 0,
1680 .platform_rapl_msr_scale = &rapl_psys_energy_units,
1681 .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM,
1682 .bic_number = BIC_Sys_J,
1683 .compat_scale = 1.0,
1684 .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM,
1685 },
1686 };
1687
1688 struct rapl_counter {
1689 unsigned long long raw_value;
1690 enum rapl_unit unit;
1691 double scale;
1692 };
1693
1694 /* Indexes used to map data read from perf and MSRs into global variables */
1695 enum ccstate_rci_index {
1696 CCSTATE_RCI_INDEX_C1_RESIDENCY = 0,
1697 CCSTATE_RCI_INDEX_C3_RESIDENCY = 1,
1698 CCSTATE_RCI_INDEX_C6_RESIDENCY = 2,
1699 CCSTATE_RCI_INDEX_C7_RESIDENCY = 3,
1700 PCSTATE_RCI_INDEX_C2_RESIDENCY = 4,
1701 PCSTATE_RCI_INDEX_C3_RESIDENCY = 5,
1702 PCSTATE_RCI_INDEX_C6_RESIDENCY = 6,
1703 PCSTATE_RCI_INDEX_C7_RESIDENCY = 7,
1704 PCSTATE_RCI_INDEX_C8_RESIDENCY = 8,
1705 PCSTATE_RCI_INDEX_C9_RESIDENCY = 9,
1706 PCSTATE_RCI_INDEX_C10_RESIDENCY = 10,
1707 NUM_CSTATE_COUNTERS,
1708 };
1709
1710 struct cstate_counter_info_t {
1711 unsigned long long data[NUM_CSTATE_COUNTERS];
1712 enum counter_source source[NUM_CSTATE_COUNTERS];
1713 unsigned long long msr[NUM_CSTATE_COUNTERS];
1714 int fd_perf_core;
1715 int fd_perf_pkg;
1716 };
1717
1718 struct cstate_counter_info_t *ccstate_counter_info;
1719 unsigned int ccstate_counter_info_size;
1720
1721 #define CSTATE_COUNTER_FLAG_COLLECT_PER_CORE (1u << 0)
1722 #define CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD ((1u << 1) | CSTATE_COUNTER_FLAG_COLLECT_PER_CORE)
1723 #define CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY (1u << 2)
1724
1725 struct cstate_counter_arch_info {
1726 int feature_mask; /* Mask for testing if the counter is supported on host */
1727 const char *perf_subsys;
1728 const char *perf_name;
1729 unsigned long long msr;
1730 unsigned int rci_index; /* Maps data from perf counters to global variables */
1731 unsigned int bic_number;
1732 unsigned long long flags;
1733 int pkg_cstate_limit;
1734 };
1735
1736 static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = {
1737 {
1738 .feature_mask = CC1,
1739 .perf_subsys = "cstate_core",
1740 .perf_name = "c1-residency",
1741 .msr = MSR_CORE_C1_RES,
1742 .rci_index = CCSTATE_RCI_INDEX_C1_RESIDENCY,
1743 .bic_number = BIC_CPU_c1,
1744 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD,
1745 .pkg_cstate_limit = 0,
1746 },
1747 {
1748 .feature_mask = CC3,
1749 .perf_subsys = "cstate_core",
1750 .perf_name = "c3-residency",
1751 .msr = MSR_CORE_C3_RESIDENCY,
1752 .rci_index = CCSTATE_RCI_INDEX_C3_RESIDENCY,
1753 .bic_number = BIC_CPU_c3,
1754 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1755 .pkg_cstate_limit = 0,
1756 },
1757 {
1758 .feature_mask = CC6,
1759 .perf_subsys = "cstate_core",
1760 .perf_name = "c6-residency",
1761 .msr = MSR_CORE_C6_RESIDENCY,
1762 .rci_index = CCSTATE_RCI_INDEX_C6_RESIDENCY,
1763 .bic_number = BIC_CPU_c6,
1764 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1765 .pkg_cstate_limit = 0,
1766 },
1767 {
1768 .feature_mask = CC7,
1769 .perf_subsys = "cstate_core",
1770 .perf_name = "c7-residency",
1771 .msr = MSR_CORE_C7_RESIDENCY,
1772 .rci_index = CCSTATE_RCI_INDEX_C7_RESIDENCY,
1773 .bic_number = BIC_CPU_c7,
1774 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1775 .pkg_cstate_limit = 0,
1776 },
1777 {
1778 .feature_mask = PC2,
1779 .perf_subsys = "cstate_pkg",
1780 .perf_name = "c2-residency",
1781 .msr = MSR_PKG_C2_RESIDENCY,
1782 .rci_index = PCSTATE_RCI_INDEX_C2_RESIDENCY,
1783 .bic_number = BIC_Pkgpc2,
1784 .flags = 0,
1785 .pkg_cstate_limit = PCL__2,
1786 },
1787 {
1788 .feature_mask = PC3,
1789 .perf_subsys = "cstate_pkg",
1790 .perf_name = "c3-residency",
1791 .msr = MSR_PKG_C3_RESIDENCY,
1792 .rci_index = PCSTATE_RCI_INDEX_C3_RESIDENCY,
1793 .bic_number = BIC_Pkgpc3,
1794 .flags = 0,
1795 .pkg_cstate_limit = PCL__3,
1796 },
1797 {
1798 .feature_mask = PC6,
1799 .perf_subsys = "cstate_pkg",
1800 .perf_name = "c6-residency",
1801 .msr = MSR_PKG_C6_RESIDENCY,
1802 .rci_index = PCSTATE_RCI_INDEX_C6_RESIDENCY,
1803 .bic_number = BIC_Pkgpc6,
1804 .flags = 0,
1805 .pkg_cstate_limit = PCL__6,
1806 },
1807 {
1808 .feature_mask = PC7,
1809 .perf_subsys = "cstate_pkg",
1810 .perf_name = "c7-residency",
1811 .msr = MSR_PKG_C7_RESIDENCY,
1812 .rci_index = PCSTATE_RCI_INDEX_C7_RESIDENCY,
1813 .bic_number = BIC_Pkgpc7,
1814 .flags = 0,
1815 .pkg_cstate_limit = PCL__7,
1816 },
1817 {
1818 .feature_mask = PC8,
1819 .perf_subsys = "cstate_pkg",
1820 .perf_name = "c8-residency",
1821 .msr = MSR_PKG_C8_RESIDENCY,
1822 .rci_index = PCSTATE_RCI_INDEX_C8_RESIDENCY,
1823 .bic_number = BIC_Pkgpc8,
1824 .flags = 0,
1825 .pkg_cstate_limit = PCL__8,
1826 },
1827 {
1828 .feature_mask = PC9,
1829 .perf_subsys = "cstate_pkg",
1830 .perf_name = "c9-residency",
1831 .msr = MSR_PKG_C9_RESIDENCY,
1832 .rci_index = PCSTATE_RCI_INDEX_C9_RESIDENCY,
1833 .bic_number = BIC_Pkgpc9,
1834 .flags = 0,
1835 .pkg_cstate_limit = PCL__9,
1836 },
1837 {
1838 .feature_mask = PC10,
1839 .perf_subsys = "cstate_pkg",
1840 .perf_name = "c10-residency",
1841 .msr = MSR_PKG_C10_RESIDENCY,
1842 .rci_index = PCSTATE_RCI_INDEX_C10_RESIDENCY,
1843 .bic_number = BIC_Pkgpc10,
1844 .flags = 0,
1845 .pkg_cstate_limit = PCL_10,
1846 },
1847 };
1848
1849 /* Indexes used to map data read from perf and MSRs into global variables */
1850 enum msr_rci_index {
1851 MSR_RCI_INDEX_APERF = 0,
1852 MSR_RCI_INDEX_MPERF = 1,
1853 MSR_RCI_INDEX_SMI = 2,
1854 NUM_MSR_COUNTERS,
1855 };
1856
1857 struct msr_counter_info_t {
1858 unsigned long long data[NUM_MSR_COUNTERS];
1859 enum counter_source source[NUM_MSR_COUNTERS];
1860 unsigned long long msr[NUM_MSR_COUNTERS];
1861 unsigned long long msr_mask[NUM_MSR_COUNTERS];
1862 int fd_perf;
1863 };
1864
1865 struct msr_counter_info_t *msr_counter_info;
1866 unsigned int msr_counter_info_size;
1867
1868 struct msr_counter_arch_info {
1869 const char *perf_subsys;
1870 const char *perf_name;
1871 unsigned long long msr;
1872 unsigned long long msr_mask;
1873 unsigned int rci_index; /* Maps data from perf counters to global variables */
1874 bool needed;
1875 bool present;
1876 };
1877
1878 enum msr_arch_info_index {
1879 MSR_ARCH_INFO_APERF_INDEX = 0,
1880 MSR_ARCH_INFO_MPERF_INDEX = 1,
1881 MSR_ARCH_INFO_SMI_INDEX = 2,
1882 };
1883
1884 static struct msr_counter_arch_info msr_counter_arch_infos[] = {
1885 [MSR_ARCH_INFO_APERF_INDEX] = {
1886 .perf_subsys = "msr",
1887 .perf_name = "aperf",
1888 .msr = MSR_IA32_APERF,
1889 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1890 .rci_index = MSR_RCI_INDEX_APERF,
1891 },
1892
1893 [MSR_ARCH_INFO_MPERF_INDEX] = {
1894 .perf_subsys = "msr",
1895 .perf_name = "mperf",
1896 .msr = MSR_IA32_MPERF,
1897 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1898 .rci_index = MSR_RCI_INDEX_MPERF,
1899 },
1900
1901 [MSR_ARCH_INFO_SMI_INDEX] = {
1902 .perf_subsys = "msr",
1903 .perf_name = "smi",
1904 .msr = MSR_SMI_COUNT,
1905 .msr_mask = 0xFFFFFFFF,
1906 .rci_index = MSR_RCI_INDEX_SMI,
1907 },
1908 };
1909
1910 /* Can be redefined when compiling, useful for testing. */
1911 #ifndef SYSFS_TELEM_PATH
1912 #define SYSFS_TELEM_PATH "/sys/class/intel_pmt"
1913 #endif
1914
1915 #define PMT_COUNTER_MTL_DC6_OFFSET 120
1916 #define PMT_COUNTER_MTL_DC6_LSB 0
1917 #define PMT_COUNTER_MTL_DC6_MSB 63
1918 #define PMT_MTL_DC6_GUID 0x1a067102
1919 #define PMT_MTL_DC6_SEQ 0
1920
1921 #define PMT_COUNTER_CWF_MC1E_OFFSET_BASE 20936
1922 #define PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT 24
1923 #define PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE 12
1924 #define PMT_COUNTER_CWF_CPUS_PER_MODULE 4
1925 #define PMT_COUNTER_CWF_MC1E_LSB 0
1926 #define PMT_COUNTER_CWF_MC1E_MSB 63
1927 #define PMT_CWF_MC1E_GUID 0x14421519
1928
1929 unsigned long long tcore_clock_freq_hz = 800000000;
1930
1931 #define PMT_COUNTER_NAME_SIZE_BYTES 16
1932 #define PMT_COUNTER_TYPE_NAME_SIZE_BYTES 32
1933
1934 struct pmt_mmio {
1935 struct pmt_mmio *next;
1936
1937 unsigned int guid;
1938 unsigned int size;
1939
1940 /* Base pointer to the mmaped memory. */
1941 void *mmio_base;
1942
1943 /*
1944 * Offset to be applied to the mmio_base
1945 * to get the beginning of the PMT counters for given GUID.
1946 */
1947 unsigned long pmt_offset;
1948 } *pmt_mmios;
1949
1950 enum pmt_datatype {
1951 PMT_TYPE_RAW,
1952 PMT_TYPE_XTAL_TIME,
1953 PMT_TYPE_TCORE_CLOCK,
1954 };
1955
1956 struct pmt_domain_info {
1957 /*
1958 * Pointer to the MMIO obtained by applying a counter offset
1959 * to the mmio_base of the mmaped region for the given GUID.
1960 *
1961 * This is where to read the raw value of the counter from.
1962 */
1963 unsigned long *pcounter;
1964 };
1965
1966 struct pmt_counter {
1967 struct pmt_counter *next;
1968
1969 /* PMT metadata */
1970 char name[PMT_COUNTER_NAME_SIZE_BYTES];
1971 enum pmt_datatype type;
1972 enum counter_scope scope;
1973 unsigned int lsb;
1974 unsigned int msb;
1975
1976 /* BIC-like metadata */
1977 enum counter_format format;
1978
1979 unsigned int num_domains;
1980 struct pmt_domain_info *domains;
1981 };
1982
1983 /*
1984 * PMT telemetry directory iterator.
1985 * Used to iterate telemetry files in sysfs in correct order.
1986 */
1987 struct pmt_diriter_t {
1988 DIR *dir;
1989 struct dirent **namelist;
1990 unsigned int num_names;
1991 unsigned int current_name_idx;
1992 };
1993
pmt_telemdir_filter(const struct dirent * e)1994 int pmt_telemdir_filter(const struct dirent *e)
1995 {
1996 unsigned int dummy;
1997
1998 return sscanf(e->d_name, "telem%u", &dummy);
1999 }
2000
pmt_telemdir_sort(const struct dirent ** a,const struct dirent ** b)2001 int pmt_telemdir_sort(const struct dirent **a, const struct dirent **b)
2002 {
2003 unsigned int aidx = 0, bidx = 0;
2004
2005 sscanf((*a)->d_name, "telem%u", &aidx);
2006 sscanf((*b)->d_name, "telem%u", &bidx);
2007
2008 return (aidx > bidx) ? 1 : (aidx < bidx) ? -1 : 0;
2009 }
2010
pmt_diriter_next(struct pmt_diriter_t * iter)2011 const struct dirent *pmt_diriter_next(struct pmt_diriter_t *iter)
2012 {
2013 const struct dirent *ret = NULL;
2014
2015 if (!iter->dir)
2016 return NULL;
2017
2018 if (iter->current_name_idx >= iter->num_names)
2019 return NULL;
2020
2021 ret = iter->namelist[iter->current_name_idx];
2022 ++iter->current_name_idx;
2023
2024 return ret;
2025 }
2026
pmt_diriter_begin(struct pmt_diriter_t * iter,const char * pmt_root_path)2027 const struct dirent *pmt_diriter_begin(struct pmt_diriter_t *iter, const char *pmt_root_path)
2028 {
2029 int num_names = iter->num_names;
2030
2031 if (!iter->dir) {
2032 iter->dir = opendir(pmt_root_path);
2033 if (iter->dir == NULL)
2034 return NULL;
2035
2036 num_names = scandir(pmt_root_path, &iter->namelist, pmt_telemdir_filter, pmt_telemdir_sort);
2037 if (num_names == -1)
2038 return NULL;
2039 }
2040
2041 iter->current_name_idx = 0;
2042 iter->num_names = num_names;
2043
2044 return pmt_diriter_next(iter);
2045 }
2046
pmt_diriter_init(struct pmt_diriter_t * iter)2047 void pmt_diriter_init(struct pmt_diriter_t *iter)
2048 {
2049 memset(iter, 0, sizeof(*iter));
2050 }
2051
pmt_diriter_remove(struct pmt_diriter_t * iter)2052 void pmt_diriter_remove(struct pmt_diriter_t *iter)
2053 {
2054 if (iter->namelist) {
2055 for (unsigned int i = 0; i < iter->num_names; i++) {
2056 free(iter->namelist[i]);
2057 iter->namelist[i] = NULL;
2058 }
2059 }
2060
2061 free(iter->namelist);
2062 iter->namelist = NULL;
2063 iter->num_names = 0;
2064 iter->current_name_idx = 0;
2065
2066 closedir(iter->dir);
2067 iter->dir = NULL;
2068 }
2069
pmt_counter_get_width(const struct pmt_counter * p)2070 unsigned int pmt_counter_get_width(const struct pmt_counter *p)
2071 {
2072 return (p->msb - p->lsb) + 1;
2073 }
2074
pmt_counter_resize_(struct pmt_counter * pcounter,unsigned int new_size)2075 void pmt_counter_resize_(struct pmt_counter *pcounter, unsigned int new_size)
2076 {
2077 struct pmt_domain_info *new_mem;
2078
2079 new_mem = (struct pmt_domain_info *)reallocarray(pcounter->domains, new_size, sizeof(*pcounter->domains));
2080 if (!new_mem) {
2081 fprintf(stderr, "%s: failed to allocate memory for PMT counters\n", __func__);
2082 exit(1);
2083 }
2084
2085 /* Zero initialize just allocated memory. */
2086 const size_t num_new_domains = new_size - pcounter->num_domains;
2087
2088 memset(&new_mem[pcounter->num_domains], 0, num_new_domains * sizeof(*pcounter->domains));
2089
2090 pcounter->num_domains = new_size;
2091 pcounter->domains = new_mem;
2092 }
2093
pmt_counter_resize(struct pmt_counter * pcounter,unsigned int new_size)2094 void pmt_counter_resize(struct pmt_counter *pcounter, unsigned int new_size)
2095 {
2096 /*
2097 * Allocate more memory ahead of time.
2098 *
2099 * Always allocate space for at least 8 elements
2100 * and double the size when growing.
2101 */
2102 if (new_size < 8)
2103 new_size = 8;
2104 new_size = MAX(new_size, pcounter->num_domains * 2);
2105
2106 pmt_counter_resize_(pcounter, new_size);
2107 }
2108
2109 struct llc_stats {
2110 unsigned long long references;
2111 unsigned long long misses;
2112 };
2113 struct l2_stats {
2114 unsigned long long references;
2115 unsigned long long hits;
2116 };
2117 struct thread_data {
2118 struct timeval tv_begin;
2119 struct timeval tv_end;
2120 struct timeval tv_delta;
2121 unsigned long long tsc;
2122 unsigned long long aperf;
2123 unsigned long long mperf;
2124 unsigned long long c1;
2125 unsigned long long instr_count;
2126 unsigned long long irq_count;
2127 unsigned long long nmi_count;
2128 unsigned int smi_count;
2129 struct llc_stats llc;
2130 struct l2_stats l2;
2131 unsigned int cpu_id;
2132 unsigned int apic_id;
2133 unsigned int x2apic_id;
2134 unsigned int flags;
2135 bool is_atom;
2136 unsigned long long counter[MAX_ADDED_THREAD_COUNTERS];
2137 unsigned long long perf_counter[MAX_ADDED_THREAD_COUNTERS];
2138 unsigned long long pmt_counter[PMT_MAX_ADDED_THREAD_COUNTERS];
2139 };
2140
2141 struct core_data {
2142 int first_cpu;
2143 unsigned long long c3;
2144 unsigned long long c6;
2145 unsigned long long c7;
2146 unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */
2147 unsigned int core_temp_c;
2148 struct rapl_counter core_energy; /* MSR_CORE_ENERGY_STAT */
2149 unsigned long long core_throt_cnt;
2150 unsigned long long counter[MAX_ADDED_CORE_COUNTERS];
2151 unsigned long long perf_counter[MAX_ADDED_CORE_COUNTERS];
2152 unsigned long long pmt_counter[PMT_MAX_ADDED_CORE_COUNTERS];
2153 };
2154
2155 struct pkg_data {
2156 int first_cpu;
2157 unsigned long long pc2;
2158 unsigned long long pc3;
2159 unsigned long long pc6;
2160 unsigned long long pc7;
2161 unsigned long long pc8;
2162 unsigned long long pc9;
2163 unsigned long long pc10;
2164 long long cpu_lpi;
2165 long long sys_lpi;
2166 unsigned long long pkg_wtd_core_c0;
2167 unsigned long long pkg_any_core_c0;
2168 unsigned long long pkg_any_gfxe_c0;
2169 unsigned long long pkg_both_core_gfxe_c0;
2170 long long gfx_rc6_ms;
2171 unsigned int gfx_mhz;
2172 unsigned int gfx_act_mhz;
2173 long long sam_mc6_ms;
2174 unsigned int sam_mhz;
2175 unsigned int sam_act_mhz;
2176 struct rapl_counter energy_pkg; /* MSR_PKG_ENERGY_STATUS */
2177 struct rapl_counter energy_dram; /* MSR_DRAM_ENERGY_STATUS */
2178 struct rapl_counter energy_cores; /* MSR_PP0_ENERGY_STATUS */
2179 struct rapl_counter energy_gfx; /* MSR_PP1_ENERGY_STATUS */
2180 struct rapl_counter rapl_pkg_perf_status; /* MSR_PKG_PERF_STATUS */
2181 struct rapl_counter rapl_dram_perf_status; /* MSR_DRAM_PERF_STATUS */
2182 unsigned int pkg_temp_c;
2183 unsigned int uncore_mhz;
2184 unsigned long long die_c6;
2185 unsigned long long counter[MAX_ADDED_PACKAGE_COUNTERS];
2186 unsigned long long perf_counter[MAX_ADDED_PACKAGE_COUNTERS];
2187 unsigned long long pmt_counter[PMT_MAX_ADDED_PACKAGE_COUNTERS];
2188 };
2189
2190 #define ODD_COUNTERS odd.threads, odd.cores, odd.packages
2191 #define EVEN_COUNTERS even.threads, even.cores, even.packages
2192
2193 /*
2194 * The accumulated sum of MSR is defined as a monotonic
2195 * increasing MSR, it will be accumulated periodically,
2196 * despite its register's bit width.
2197 */
2198 enum {
2199 IDX_PKG_ENERGY,
2200 IDX_DRAM_ENERGY,
2201 IDX_PP0_ENERGY,
2202 IDX_PP1_ENERGY,
2203 IDX_PKG_PERF,
2204 IDX_DRAM_PERF,
2205 IDX_PSYS_ENERGY,
2206 IDX_COUNT,
2207 };
2208
2209 int get_msr_sum(int cpu, off_t offset, unsigned long long *msr);
2210
2211 struct msr_sum_array {
2212 /* get_msr_sum() = sum + (get_msr() - last) */
2213 struct {
2214 /*The accumulated MSR value is updated by the timer */
2215 unsigned long long sum;
2216 /*The MSR footprint recorded in last timer */
2217 unsigned long long last;
2218 } entries[IDX_COUNT];
2219 };
2220
2221 /* The percpu MSR sum array.*/
2222 struct msr_sum_array *per_cpu_msr_sum;
2223
idx_to_offset(int idx)2224 off_t idx_to_offset(int idx)
2225 {
2226 off_t offset;
2227
2228 switch (idx) {
2229 case IDX_PKG_ENERGY:
2230 if (platform->plat_rapl_msrs & RAPL_AMD_F17H)
2231 offset = MSR_PKG_ENERGY_STAT;
2232 else
2233 offset = MSR_PKG_ENERGY_STATUS;
2234 break;
2235 case IDX_DRAM_ENERGY:
2236 offset = MSR_DRAM_ENERGY_STATUS;
2237 break;
2238 case IDX_PP0_ENERGY:
2239 offset = MSR_PP0_ENERGY_STATUS;
2240 break;
2241 case IDX_PP1_ENERGY:
2242 offset = MSR_PP1_ENERGY_STATUS;
2243 break;
2244 case IDX_PKG_PERF:
2245 offset = MSR_PKG_PERF_STATUS;
2246 break;
2247 case IDX_DRAM_PERF:
2248 offset = MSR_DRAM_PERF_STATUS;
2249 break;
2250 case IDX_PSYS_ENERGY:
2251 offset = MSR_PLATFORM_ENERGY_STATUS;
2252 break;
2253 default:
2254 offset = -1;
2255 }
2256 return offset;
2257 }
2258
offset_to_idx(off_t offset)2259 int offset_to_idx(off_t offset)
2260 {
2261 int idx;
2262
2263 switch (offset) {
2264 case MSR_PKG_ENERGY_STATUS:
2265 case MSR_PKG_ENERGY_STAT:
2266 idx = IDX_PKG_ENERGY;
2267 break;
2268 case MSR_DRAM_ENERGY_STATUS:
2269 idx = IDX_DRAM_ENERGY;
2270 break;
2271 case MSR_PP0_ENERGY_STATUS:
2272 idx = IDX_PP0_ENERGY;
2273 break;
2274 case MSR_PP1_ENERGY_STATUS:
2275 idx = IDX_PP1_ENERGY;
2276 break;
2277 case MSR_PKG_PERF_STATUS:
2278 idx = IDX_PKG_PERF;
2279 break;
2280 case MSR_DRAM_PERF_STATUS:
2281 idx = IDX_DRAM_PERF;
2282 break;
2283 case MSR_PLATFORM_ENERGY_STATUS:
2284 idx = IDX_PSYS_ENERGY;
2285 break;
2286 default:
2287 idx = -1;
2288 }
2289 return idx;
2290 }
2291
idx_valid(int idx)2292 int idx_valid(int idx)
2293 {
2294 switch (idx) {
2295 case IDX_PKG_ENERGY:
2296 return valid_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H);
2297 case IDX_DRAM_ENERGY:
2298 return valid_rapl_msrs & RAPL_DRAM;
2299 case IDX_PP0_ENERGY:
2300 return valid_rapl_msrs & RAPL_CORE_ENERGY_STATUS;
2301 case IDX_PP1_ENERGY:
2302 return valid_rapl_msrs & RAPL_GFX;
2303 case IDX_PKG_PERF:
2304 return valid_rapl_msrs & RAPL_PKG_PERF_STATUS;
2305 case IDX_DRAM_PERF:
2306 return valid_rapl_msrs & RAPL_DRAM_PERF_STATUS;
2307 case IDX_PSYS_ENERGY:
2308 return valid_rapl_msrs & RAPL_PSYS;
2309 default:
2310 return 0;
2311 }
2312 }
2313
2314 struct sys_counters {
2315 /* MSR added counters */
2316 unsigned int added_thread_counters;
2317 unsigned int added_core_counters;
2318 unsigned int added_package_counters;
2319 struct msr_counter *tp;
2320 struct msr_counter *cp;
2321 struct msr_counter *pp;
2322
2323 /* perf added counters */
2324 unsigned int added_thread_perf_counters;
2325 unsigned int added_core_perf_counters;
2326 unsigned int added_package_perf_counters;
2327 struct perf_counter_info *perf_tp;
2328 struct perf_counter_info *perf_cp;
2329 struct perf_counter_info *perf_pp;
2330
2331 struct pmt_counter *pmt_tp;
2332 struct pmt_counter *pmt_cp;
2333 struct pmt_counter *pmt_pp;
2334 } sys;
2335
free_msr_counters_(struct msr_counter ** pp)2336 static size_t free_msr_counters_(struct msr_counter **pp)
2337 {
2338 struct msr_counter *p = NULL;
2339 size_t num_freed = 0;
2340
2341 while (*pp) {
2342 p = *pp;
2343
2344 if (p->msr_num != 0) {
2345 *pp = p->next;
2346
2347 free(p);
2348 ++num_freed;
2349
2350 continue;
2351 }
2352
2353 pp = &p->next;
2354 }
2355
2356 return num_freed;
2357 }
2358
2359 /*
2360 * Free all added counters accessed via msr.
2361 */
free_sys_msr_counters(void)2362 static void free_sys_msr_counters(void)
2363 {
2364 /* Thread counters */
2365 sys.added_thread_counters -= free_msr_counters_(&sys.tp);
2366
2367 /* Core counters */
2368 sys.added_core_counters -= free_msr_counters_(&sys.cp);
2369
2370 /* Package counters */
2371 sys.added_package_counters -= free_msr_counters_(&sys.pp);
2372 }
2373
2374 struct counters {
2375 struct thread_data *threads;
2376 struct core_data *cores;
2377 struct pkg_data *packages;
2378 } average, even, odd;
2379
2380 struct platform_counters {
2381 struct rapl_counter energy_psys; /* MSR_PLATFORM_ENERGY_STATUS */
2382 } platform_counters_odd, platform_counters_even;
2383
2384 #define MAX_HT_ID 3 /* support SMT-4 */
2385
2386 struct cpu_topology {
2387 int cpu_id;
2388 int core_id; /* unique within a package */
2389 int module_id;
2390 int package_id;
2391 int die_id;
2392 int l3_id;
2393 int physical_node_id;
2394 int logical_node_id; /* 0-based count within the package */
2395 int ht_id; /* unique within a core */
2396 int ht_sibling_cpu_id[MAX_HT_ID + 1];
2397 int type;
2398 cpu_set_t *put_ids; /* Processing Unit/Thread IDs */
2399 } *cpus;
2400
2401 struct topo_params {
2402 int num_packages;
2403 int num_die;
2404 int num_cpus;
2405 int num_cores; /* system wide */
2406 int allowed_packages;
2407 int allowed_cpus;
2408 int allowed_cores;
2409 int max_cpu_num;
2410 int max_core_id; /* within a package */
2411 int min_module_id; /* system wide */
2412 int max_module_id; /* system wide */
2413 int max_package_id;
2414 int max_die_id;
2415 int max_l3_id;
2416 int max_node_num;
2417 int nodes_per_pkg;
2418 int cores_per_pkg;
2419 int threads_per_core;
2420 } topo;
2421
2422 struct timeval tv_even, tv_odd, tv_delta;
2423
2424 int *irq_column_2_cpu; /* /proc/interrupts column numbers */
2425 int *irqs_per_cpu; /* indexed by cpu_num */
2426 int *nmi_per_cpu; /* indexed by cpu_num */
2427
2428 void setup_all_buffers(bool startup);
2429
2430 char *sys_lpi_file;
2431 char *sys_lpi_file_sysfs = "/sys/devices/system/cpu/cpuidle/low_power_idle_system_residency_us";
2432 char *sys_lpi_file_debugfs = "/sys/kernel/debug/pmc_core/slp_s0_residency_usec";
2433
cpu_is_not_present(int cpu)2434 int cpu_is_not_present(int cpu)
2435 {
2436 if (cpu < 0)
2437 return 1;
2438
2439 return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set);
2440 }
2441
cpu_is_not_allowed(int cpu)2442 int cpu_is_not_allowed(int cpu)
2443 {
2444 if (cpu < 0)
2445 return 1;
2446
2447 return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set);
2448 }
2449
2450 #define GLOBAL_CORE_ID(core_id, pkg_id) (core_id + pkg_id * (topo.max_core_id + 1))
2451 /*
2452 * run func(thread, core, package) in topology order
2453 * skip non-present cpus
2454 */
2455
2456 #define PER_THREAD_PARAMS struct thread_data *t, struct core_data *c, struct pkg_data *p
2457
has_allowed_lower_ht_sibling(int cpu)2458 int has_allowed_lower_ht_sibling(int cpu)
2459 {
2460 int i;
2461
2462 for (i = 0; i <= cpus[cpu].ht_id; ++i) {
2463 int sibling_cpu_id = cpus[cpu].ht_sibling_cpu_id[i];
2464
2465 if (sibling_cpu_id == cpu)
2466 return 0;
2467
2468 if (!cpu_is_not_allowed(sibling_cpu_id))
2469 return 1;
2470 }
2471 return 0;
2472 }
2473
for_all_cpus(int (func)(struct thread_data *,struct core_data *,struct pkg_data *),struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base)2474 int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *),
2475 struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base)
2476 {
2477 int cpu, retval;
2478
2479 retval = 0;
2480
2481 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
2482 struct thread_data *t;
2483 struct core_data *c;
2484 struct pkg_data *p;
2485
2486 int pkg_id = cpus[cpu].package_id;
2487
2488 if (cpu_is_not_allowed(cpu))
2489 continue;
2490
2491 if (has_allowed_lower_ht_sibling(cpu)) /* skip HT sibling */
2492 continue;
2493
2494 t = &thread_base[cpu];
2495 c = &core_base[GLOBAL_CORE_ID(cpus[cpu].core_id, pkg_id)];
2496 p = &pkg_base[pkg_id];
2497
2498 retval |= func(t, c, p);
2499
2500 /* Handle other HT siblings now */
2501 int i;
2502
2503 for (i = 0; i <= MAX_HT_ID; ++i) {
2504 int sibling_cpu_id = cpus[cpu].ht_sibling_cpu_id[i];
2505
2506 if (sibling_cpu_id < 0)
2507 break;
2508
2509 if (sibling_cpu_id == cpu)
2510 continue;
2511
2512 if (cpu_is_not_allowed(sibling_cpu_id))
2513 continue;
2514
2515 t = &thread_base[sibling_cpu_id];
2516
2517 retval |= func(t, c, p);
2518 }
2519 }
2520 return retval;
2521 }
2522
is_cpu_first_thread_in_core(struct thread_data * t,struct core_data * c)2523 int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c)
2524 {
2525 return ((int)t->cpu_id == c->first_cpu || c->first_cpu < 0);
2526 }
2527
is_cpu_first_core_in_package(struct thread_data * t,struct pkg_data * p)2528 int is_cpu_first_core_in_package(struct thread_data *t, struct pkg_data *p)
2529 {
2530 return ((int)t->cpu_id == p->first_cpu || p->first_cpu < 0);
2531 }
2532
is_cpu_first_thread_in_package(struct thread_data * t,struct core_data * c,struct pkg_data * p)2533 int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p)
2534 {
2535 return is_cpu_first_thread_in_core(t, c) && is_cpu_first_core_in_package(t, p);
2536 }
2537
cpu_migrate(int cpu)2538 int cpu_migrate(int cpu)
2539 {
2540 CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
2541 CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set);
2542 if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1)
2543 return -1;
2544 else
2545 return 0;
2546 }
2547
get_msr_fd(int cpu)2548 int get_msr_fd(int cpu)
2549 {
2550 char pathname[32];
2551 int fd;
2552
2553 fd = fd_percpu[cpu];
2554
2555 if (fd)
2556 return fd;
2557 sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu);
2558 fd = open(pathname, O_RDONLY);
2559 if (fd < 0)
2560 err(-1, "%s open failed, try chown or chmod +r %s, "
2561 "or run with --no-msr, or run as root", pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr");
2562 fd_percpu[cpu] = fd;
2563
2564 return fd;
2565 }
2566
bic_disable_msr_access(void)2567 static void bic_disable_msr_access(void)
2568 {
2569 CLR_BIC(BIC_Mod_c6, &bic_enabled);
2570 CLR_BIC(BIC_CoreTmp, &bic_enabled);
2571 CLR_BIC(BIC_Totl_c0, &bic_enabled);
2572 CLR_BIC(BIC_Any_c0, &bic_enabled);
2573 CLR_BIC(BIC_GFX_c0, &bic_enabled);
2574 CLR_BIC(BIC_CPUGFX, &bic_enabled);
2575 CLR_BIC(BIC_PkgTmp, &bic_enabled);
2576
2577 free_sys_msr_counters();
2578 }
2579
bic_disable_perf_access(void)2580 static void bic_disable_perf_access(void)
2581 {
2582 CLR_BIC(BIC_IPC, &bic_enabled);
2583 CLR_BIC(BIC_LLC_MRPS, &bic_enabled);
2584 CLR_BIC(BIC_LLC_HIT, &bic_enabled);
2585 CLR_BIC(BIC_L2_MRPS, &bic_enabled);
2586 CLR_BIC(BIC_L2_HIT, &bic_enabled);
2587 }
2588
perf_event_open(struct perf_event_attr * hw_event,pid_t pid,int cpu,int group_fd,unsigned long flags)2589 static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags)
2590 {
2591 assert(!no_perf);
2592
2593 return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags);
2594 }
2595
open_perf_counter(int cpu,unsigned int type,unsigned int config,int group_fd,__u64 read_format)2596 static long open_perf_counter(int cpu, unsigned int type, unsigned int config, int group_fd, __u64 read_format)
2597 {
2598 struct perf_event_attr attr;
2599 const pid_t pid = -1;
2600 const unsigned long flags = 0;
2601
2602 assert(!no_perf);
2603
2604 memset(&attr, 0, sizeof(struct perf_event_attr));
2605
2606 attr.type = type;
2607 attr.size = sizeof(struct perf_event_attr);
2608 attr.config = config;
2609 attr.disabled = 0;
2610 attr.sample_type = PERF_SAMPLE_IDENTIFIER;
2611 attr.read_format = read_format;
2612
2613 const int fd = perf_event_open(&attr, pid, cpu, group_fd, flags);
2614
2615 return fd;
2616 }
2617
get_instr_count_fd(int cpu)2618 int get_instr_count_fd(int cpu)
2619 {
2620 if (fd_instr_count_percpu[cpu])
2621 return fd_instr_count_percpu[cpu];
2622
2623 fd_instr_count_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0);
2624
2625 return fd_instr_count_percpu[cpu];
2626 }
2627
get_msr(int cpu,off_t offset,unsigned long long * msr)2628 int get_msr(int cpu, off_t offset, unsigned long long *msr)
2629 {
2630 ssize_t retval;
2631
2632 assert(!no_msr);
2633
2634 retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset);
2635
2636 if (retval != sizeof *msr)
2637 err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset);
2638
2639 return 0;
2640 }
2641
add_msr_counter(int cpu,off_t offset)2642 int add_msr_counter(int cpu, off_t offset)
2643 {
2644 ssize_t retval;
2645 unsigned long long value;
2646
2647 if (no_msr)
2648 return -1;
2649
2650 if (!offset)
2651 return -1;
2652
2653 retval = pread(get_msr_fd(cpu), &value, sizeof(value), offset);
2654
2655 /* if the read failed, the probe fails */
2656 if (retval != sizeof(value))
2657 return -1;
2658
2659 if (value == 0)
2660 return 0;
2661
2662 return 1;
2663 }
2664
add_rapl_msr_counter(int cpu,const struct rapl_counter_arch_info * cai)2665 int add_rapl_msr_counter(int cpu, const struct rapl_counter_arch_info *cai)
2666 {
2667 int ret;
2668
2669 if (!(valid_rapl_msrs & cai->feature_mask))
2670 return -1;
2671
2672 ret = add_msr_counter(cpu, cai->msr);
2673 if (ret < 0)
2674 return -1;
2675
2676 switch (cai->rci_index) {
2677 case RAPL_RCI_INDEX_ENERGY_PKG:
2678 case RAPL_RCI_INDEX_ENERGY_CORES:
2679 case RAPL_RCI_INDEX_DRAM:
2680 case RAPL_RCI_INDEX_GFX:
2681 case RAPL_RCI_INDEX_ENERGY_PLATFORM:
2682 if (ret == 0)
2683 return 1;
2684 }
2685
2686 /* PKG,DRAM_PERF_STATUS MSRs, can return any value */
2687 return 1;
2688 }
2689
2690 /* Convert CPU ID to domain ID for given added perf counter. */
cpu_to_domain(const struct perf_counter_info * pc,int cpu)2691 unsigned int cpu_to_domain(const struct perf_counter_info *pc, int cpu)
2692 {
2693 switch (pc->scope) {
2694 case SCOPE_CPU:
2695 return cpu;
2696
2697 case SCOPE_CORE:
2698 return cpus[cpu].core_id;
2699
2700 case SCOPE_PACKAGE:
2701 return cpus[cpu].package_id;
2702 }
2703
2704 __builtin_unreachable();
2705 }
2706
2707 #define MAX_DEFERRED 16
2708 char *deferred_add_names[MAX_DEFERRED];
2709 char *deferred_skip_names[MAX_DEFERRED];
2710 int deferred_add_index;
2711 int deferred_skip_index;
2712 unsigned int deferred_add_consumed;
2713 unsigned int deferred_skip_consumed;
2714
2715 /*
2716 * HIDE_LIST - hide this list of counters, show the rest [default]
2717 * SHOW_LIST - show this list of counters, hide the rest
2718 */
2719 enum show_hide_mode { SHOW_LIST, HIDE_LIST } global_show_hide_mode = HIDE_LIST;
2720
help(void)2721 void help(void)
2722 {
2723 fprintf(outf,
2724 "Usage: turbostat [OPTIONS][(--interval seconds) | COMMAND ...]\n"
2725 "\n"
2726 "Turbostat forks the specified COMMAND and prints statistics\n"
2727 "when COMMAND completes.\n"
2728 "If no COMMAND is specified, turbostat wakes every 5-seconds\n"
2729 "to print statistics, until interrupted.\n"
2730 " -a, --add counter\n"
2731 " add a counter\n"
2732 " eg. --add msr0x10,u64,cpu,delta,MY_TSC\n"
2733 " eg. --add perf/cstate_pkg/c2-residency,package,delta,percent,perfPC2\n"
2734 " eg. --add pmt,name=XTAL,type=raw,domain=package0,offset=0,lsb=0,msb=63,guid=0x1a067102\n"
2735 " -c, --cpu cpu-set\n"
2736 " limit output to summary plus cpu-set:\n"
2737 " {core | package | j,k,l..m,n-p }\n"
2738 " -d, --debug\n"
2739 " displays usec, Time_Of_Day_Seconds and more debugging\n"
2740 " debug messages are printed to stderr\n"
2741 " -D, --Dump\n"
2742 " displays the raw counter values\n"
2743 " -e, --enable [all | column]\n"
2744 " shows all or the specified disabled column\n"
2745 " -f, --force\n"
2746 " force load turbostat with minimum default features on unsupported platforms.\n"
2747 " -H, --hide [column | column,column,...]\n"
2748 " hide the specified column(s)\n"
2749 " -i, --interval sec.subsec\n"
2750 " override default 5-second measurement interval\n"
2751 " -J, --Joules\n"
2752 " displays energy in Joules instead of Watts\n"
2753 " -l, --list\n"
2754 " list column headers only\n"
2755 " -M, --no-msr\n"
2756 " disable all uses of the MSR driver\n"
2757 " -P, --no-perf\n"
2758 " disable all uses of the perf API\n"
2759 " -n, --num_iterations num\n"
2760 " number of the measurement iterations\n"
2761 " -N, --header_iterations num\n"
2762 " print header every num iterations\n"
2763 " -o, --out file\n"
2764 " create or truncate \"file\" for all output\n"
2765 " -q, --quiet\n"
2766 " skip decoding system configuration header\n"
2767 " -s, --show [column | column,column,...]\n"
2768 " show only the specified column(s)\n"
2769 " -S, --Summary\n"
2770 " limits output to 1-line system summary per interval\n"
2771 " -T, --TCC temperature\n"
2772 " sets the Thermal Control Circuit temperature in\n"
2773 " degrees Celsius\n"
2774 " -h, --help\n"
2775 " print this help message\n -v, --version\n\t\tprint version information\n\nFor more help, run \"man turbostat\"\n");
2776 }
2777
2778 /*
2779 * bic_lookup
2780 * for all the strings in comma separate name_list,
2781 * set the approprate bit in return value.
2782 */
bic_lookup(cpu_set_t * ret_set,char * name_list,enum show_hide_mode mode)2783 void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode)
2784 {
2785 unsigned int i;
2786
2787 while (name_list) {
2788 char *comma;
2789
2790 comma = strchr(name_list, ',');
2791
2792 if (comma)
2793 *comma = '\0';
2794
2795 for (i = 0; i < MAX_BIC; ++i) {
2796 if (!strcmp(name_list, bic[i].name)) {
2797 SET_BIC(i, ret_set);
2798 break;
2799 }
2800 if (!strcmp(name_list, "all")) {
2801 bic_set_all(ret_set);
2802 break;
2803 } else if (!strcmp(name_list, "topology")) {
2804 CPU_OR(ret_set, ret_set, &bic_group_topology);
2805 break;
2806 } else if (!strcmp(name_list, "power")) {
2807 CPU_OR(ret_set, ret_set, &bic_group_thermal_pwr);
2808 break;
2809 } else if (!strcmp(name_list, "idle")) {
2810 CPU_OR(ret_set, ret_set, &bic_group_idle);
2811 break;
2812 } else if (!strcmp(name_list, "cache")) {
2813 CPU_OR(ret_set, ret_set, &bic_group_cache);
2814 break;
2815 } else if (!strcmp(name_list, "llc")) {
2816 CPU_OR(ret_set, ret_set, &bic_group_cache);
2817 break;
2818 } else if (!strcmp(name_list, "swidle")) {
2819 CPU_OR(ret_set, ret_set, &bic_group_sw_idle);
2820 break;
2821 } else if (!strcmp(name_list, "sysfs")) { /* legacy compatibility */
2822 CPU_OR(ret_set, ret_set, &bic_group_sw_idle);
2823 break;
2824 } else if (!strcmp(name_list, "hwidle")) {
2825 CPU_OR(ret_set, ret_set, &bic_group_hw_idle);
2826 break;
2827 } else if (!strcmp(name_list, "frequency")) {
2828 CPU_OR(ret_set, ret_set, &bic_group_frequency);
2829 break;
2830 } else if (!strcmp(name_list, "other")) {
2831 CPU_OR(ret_set, ret_set, &bic_group_other);
2832 break;
2833 }
2834 }
2835 if (i == MAX_BIC) {
2836 if (mode == SHOW_LIST) {
2837 deferred_add_names[deferred_add_index++] = name_list;
2838 if (deferred_add_index >= MAX_DEFERRED) {
2839 fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", MAX_DEFERRED, name_list);
2840 help();
2841 exit(1);
2842 }
2843 } else {
2844 deferred_skip_names[deferred_skip_index++] = name_list;
2845 if (debug)
2846 fprintf(stderr, "deferred \"%s\"\n", name_list);
2847 if (deferred_skip_index >= MAX_DEFERRED) {
2848 fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", MAX_DEFERRED, name_list);
2849 help();
2850 exit(1);
2851 }
2852 }
2853 }
2854
2855 name_list = comma;
2856 if (name_list)
2857 name_list++;
2858
2859 }
2860 }
2861
2862 /*
2863 * print_name()
2864 * Print column header name for raw 64-bit counter in 16 columns (at least 8-char plus a tab)
2865 * Otherwise, allow the name + tab to fit within 8-coumn tab-stop.
2866 * In both cases, left justififed, just like other turbostat columns,
2867 * to allow the column values to consume the tab.
2868 *
2869 * Yes, 32-bit counters can overflow 8-columns, and
2870 * 64-bit counters can overflow 16-columns, but that is uncommon.
2871 */
print_name(int width,int * printed,char * delim,char * name,enum counter_type type,enum counter_format format)2872 static inline int print_name(int width, int *printed, char *delim, char *name, enum counter_type type, enum counter_format format)
2873 {
2874 UNUSED(type);
2875 char *sep = (*printed)++ ? delim : "";
2876
2877 if (format == FORMAT_RAW && width >= 64)
2878 return sprintf(outp, "%s%-8s", sep, name);
2879 else
2880 return sprintf(outp, "%s%s", sep, name);
2881 }
2882
print_hex_value(int width,int * printed,char * delim,unsigned long long value)2883 static inline int print_hex_value(int width, int *printed, char *delim, unsigned long long value)
2884 {
2885 char *sep = (*printed)++ ? delim : "";
2886
2887 if (width <= 32)
2888 return sprintf(outp, "%s%08llx", sep, value);
2889 else
2890 return sprintf(outp, "%s%016llx", sep, value);
2891 }
2892
print_decimal_value(int width,int * printed,char * delim,unsigned long long value)2893 static inline int print_decimal_value(int width, int *printed, char *delim, unsigned long long value)
2894 {
2895 char *sep = (*printed)++ ? delim : "";
2896
2897 UNUSED(width);
2898
2899 return sprintf(outp, "%s%lld", sep, value);
2900 }
2901
print_float_value(int * printed,char * delim,double value)2902 static inline int print_float_value(int *printed, char *delim, double value)
2903 {
2904 char *sep = (*printed)++ ? delim : "";
2905
2906 return sprintf(outp, "%s%0.2f", sep, value);
2907 }
2908
print_header(char * delim)2909 void print_header(char *delim)
2910 {
2911 struct msr_counter *mp;
2912 struct perf_counter_info *pp;
2913 struct pmt_counter *ppmt;
2914 int printed = 0;
2915
2916 if (DO_BIC(BIC_USEC))
2917 outp += sprintf(outp, "%susec", (printed++ ? delim : ""));
2918 if (DO_BIC(BIC_TOD))
2919 outp += sprintf(outp, "%sTime_Of_Day_Seconds", (printed++ ? delim : ""));
2920 if (DO_BIC(BIC_Package))
2921 outp += sprintf(outp, "%sPackage", (printed++ ? delim : ""));
2922 if (DO_BIC(BIC_Die))
2923 outp += sprintf(outp, "%sDie", (printed++ ? delim : ""));
2924 if (DO_BIC(BIC_L3))
2925 outp += sprintf(outp, "%sL3", (printed++ ? delim : ""));
2926 if (DO_BIC(BIC_Node))
2927 outp += sprintf(outp, "%sNode", (printed++ ? delim : ""));
2928 if (DO_BIC(BIC_Module))
2929 outp += sprintf(outp, "%sModule", (printed++ ? delim : ""));
2930 if (DO_BIC(BIC_Core))
2931 outp += sprintf(outp, "%sCore", (printed++ ? delim : ""));
2932 if (DO_BIC(BIC_CPU))
2933 outp += sprintf(outp, "%sCPU", (printed++ ? delim : ""));
2934 if (DO_BIC(BIC_APIC))
2935 outp += sprintf(outp, "%sAPIC", (printed++ ? delim : ""));
2936 if (DO_BIC(BIC_X2APIC))
2937 outp += sprintf(outp, "%sX2APIC", (printed++ ? delim : ""));
2938 if (DO_BIC(BIC_Avg_MHz))
2939 outp += sprintf(outp, "%sAvg_MHz", (printed++ ? delim : ""));
2940 if (DO_BIC(BIC_Busy))
2941 outp += sprintf(outp, "%sBusy%%", (printed++ ? delim : ""));
2942 if (DO_BIC(BIC_Bzy_MHz))
2943 outp += sprintf(outp, "%sBzy_MHz", (printed++ ? delim : ""));
2944 if (DO_BIC(BIC_TSC_MHz))
2945 outp += sprintf(outp, "%sTSC_MHz", (printed++ ? delim : ""));
2946
2947 if (DO_BIC(BIC_IPC))
2948 outp += sprintf(outp, "%sIPC", (printed++ ? delim : ""));
2949
2950 if (DO_BIC(BIC_IRQ)) {
2951 if (sums_need_wide_columns)
2952 outp += sprintf(outp, "%s IRQ", (printed++ ? delim : ""));
2953 else
2954 outp += sprintf(outp, "%sIRQ", (printed++ ? delim : ""));
2955 }
2956 if (DO_BIC(BIC_NMI)) {
2957 if (sums_need_wide_columns)
2958 outp += sprintf(outp, "%s NMI", (printed++ ? delim : ""));
2959 else
2960 outp += sprintf(outp, "%sNMI", (printed++ ? delim : ""));
2961 }
2962
2963 if (DO_BIC(BIC_SMI))
2964 outp += sprintf(outp, "%sSMI", (printed++ ? delim : ""));
2965
2966 if (DO_BIC(BIC_LLC_MRPS))
2967 outp += sprintf(outp, "%sLLCMRPS", (printed++ ? delim : ""));
2968
2969 if (DO_BIC(BIC_LLC_HIT))
2970 outp += sprintf(outp, "%sLLC%%hit", (printed++ ? delim : ""));
2971
2972 if (DO_BIC(BIC_L2_MRPS))
2973 outp += sprintf(outp, "%sL2MRPS", (printed++ ? delim : ""));
2974
2975 if (DO_BIC(BIC_L2_HIT))
2976 outp += sprintf(outp, "%sL2%%hit", (printed++ ? delim : ""));
2977
2978 for (mp = sys.tp; mp; mp = mp->next)
2979 outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format);
2980
2981 for (pp = sys.perf_tp; pp; pp = pp->next)
2982 outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format);
2983
2984 ppmt = sys.pmt_tp;
2985 while (ppmt) {
2986 switch (ppmt->type) {
2987 case PMT_TYPE_RAW:
2988 outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2989 break;
2990
2991 case PMT_TYPE_XTAL_TIME:
2992 case PMT_TYPE_TCORE_CLOCK:
2993 outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2994 break;
2995 }
2996
2997 ppmt = ppmt->next;
2998 }
2999
3000 if (DO_BIC(BIC_CPU_c1))
3001 outp += sprintf(outp, "%sCPU%%c1", (printed++ ? delim : ""));
3002 if (DO_BIC(BIC_CPU_c3))
3003 outp += sprintf(outp, "%sCPU%%c3", (printed++ ? delim : ""));
3004 if (DO_BIC(BIC_CPU_c6))
3005 outp += sprintf(outp, "%sCPU%%c6", (printed++ ? delim : ""));
3006 if (DO_BIC(BIC_CPU_c7))
3007 outp += sprintf(outp, "%sCPU%%c7", (printed++ ? delim : ""));
3008
3009 if (DO_BIC(BIC_Mod_c6))
3010 outp += sprintf(outp, "%sMod%%c6", (printed++ ? delim : ""));
3011
3012 if (DO_BIC(BIC_CoreTmp))
3013 outp += sprintf(outp, "%sCoreTmp", (printed++ ? delim : ""));
3014
3015 if (DO_BIC(BIC_CORE_THROT_CNT))
3016 outp += sprintf(outp, "%sCoreThr", (printed++ ? delim : ""));
3017
3018 if (valid_rapl_msrs && !rapl_joules) {
3019 if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
3020 outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : ""));
3021 } else if (valid_rapl_msrs && rapl_joules) {
3022 if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
3023 outp += sprintf(outp, "%sCor_J", (printed++ ? delim : ""));
3024 }
3025
3026 for (mp = sys.cp; mp; mp = mp->next)
3027 outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format);
3028
3029 for (pp = sys.perf_cp; pp; pp = pp->next)
3030 outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format);
3031
3032 ppmt = sys.pmt_cp;
3033 while (ppmt) {
3034 switch (ppmt->type) {
3035 case PMT_TYPE_RAW:
3036 outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
3037
3038 break;
3039
3040 case PMT_TYPE_XTAL_TIME:
3041 case PMT_TYPE_TCORE_CLOCK:
3042 outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
3043 break;
3044 }
3045
3046 ppmt = ppmt->next;
3047 }
3048 if (DO_BIC(BIC_PkgTmp))
3049 outp += sprintf(outp, "%sPkgTmp", (printed++ ? delim : ""));
3050
3051 if (DO_BIC(BIC_GFX_rc6))
3052 outp += sprintf(outp, "%sGFX%%rc6", (printed++ ? delim : ""));
3053
3054 if (DO_BIC(BIC_GFXMHz))
3055 outp += sprintf(outp, "%sGFXMHz", (printed++ ? delim : ""));
3056
3057 if (DO_BIC(BIC_GFXACTMHz))
3058 outp += sprintf(outp, "%sGFXAMHz", (printed++ ? delim : ""));
3059
3060 if (DO_BIC(BIC_SAM_mc6))
3061 outp += sprintf(outp, "%sSAM%%mc6", (printed++ ? delim : ""));
3062
3063 if (DO_BIC(BIC_SAMMHz))
3064 outp += sprintf(outp, "%sSAMMHz", (printed++ ? delim : ""));
3065
3066 if (DO_BIC(BIC_SAMACTMHz))
3067 outp += sprintf(outp, "%sSAMAMHz", (printed++ ? delim : ""));
3068
3069 if (DO_BIC(BIC_Totl_c0))
3070 outp += sprintf(outp, "%sTotl%%C0", (printed++ ? delim : ""));
3071 if (DO_BIC(BIC_Any_c0))
3072 outp += sprintf(outp, "%sAny%%C0", (printed++ ? delim : ""));
3073 if (DO_BIC(BIC_GFX_c0))
3074 outp += sprintf(outp, "%sGFX%%C0", (printed++ ? delim : ""));
3075 if (DO_BIC(BIC_CPUGFX))
3076 outp += sprintf(outp, "%sCPUGFX%%", (printed++ ? delim : ""));
3077
3078 if (DO_BIC(BIC_Pkgpc2))
3079 outp += sprintf(outp, "%sPkg%%pc2", (printed++ ? delim : ""));
3080 if (DO_BIC(BIC_Pkgpc3))
3081 outp += sprintf(outp, "%sPkg%%pc3", (printed++ ? delim : ""));
3082 if (DO_BIC(BIC_Pkgpc6))
3083 outp += sprintf(outp, "%sPkg%%pc6", (printed++ ? delim : ""));
3084 if (DO_BIC(BIC_Pkgpc7))
3085 outp += sprintf(outp, "%sPkg%%pc7", (printed++ ? delim : ""));
3086 if (DO_BIC(BIC_Pkgpc8))
3087 outp += sprintf(outp, "%sPkg%%pc8", (printed++ ? delim : ""));
3088 if (DO_BIC(BIC_Pkgpc9))
3089 outp += sprintf(outp, "%sPkg%%pc9", (printed++ ? delim : ""));
3090 if (DO_BIC(BIC_Pkgpc10))
3091 outp += sprintf(outp, "%sPk%%pc10", (printed++ ? delim : ""));
3092 if (DO_BIC(BIC_Diec6))
3093 outp += sprintf(outp, "%sDie%%c6", (printed++ ? delim : ""));
3094 if (DO_BIC(BIC_CPU_LPI))
3095 outp += sprintf(outp, "%sCPU%%LPI", (printed++ ? delim : ""));
3096 if (DO_BIC(BIC_SYS_LPI))
3097 outp += sprintf(outp, "%sSYS%%LPI", (printed++ ? delim : ""));
3098
3099 if (!rapl_joules) {
3100 if (DO_BIC(BIC_PkgWatt))
3101 outp += sprintf(outp, "%sPkgWatt", (printed++ ? delim : ""));
3102 if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
3103 outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : ""));
3104 if (DO_BIC(BIC_GFXWatt))
3105 outp += sprintf(outp, "%sGFXWatt", (printed++ ? delim : ""));
3106 if (DO_BIC(BIC_RAMWatt))
3107 outp += sprintf(outp, "%sRAMWatt", (printed++ ? delim : ""));
3108 if (DO_BIC(BIC_PKG__))
3109 outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
3110 if (DO_BIC(BIC_RAM__))
3111 outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
3112 } else {
3113 if (DO_BIC(BIC_Pkg_J))
3114 outp += sprintf(outp, "%sPkg_J", (printed++ ? delim : ""));
3115 if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
3116 outp += sprintf(outp, "%sCor_J", (printed++ ? delim : ""));
3117 if (DO_BIC(BIC_GFX_J))
3118 outp += sprintf(outp, "%sGFX_J", (printed++ ? delim : ""));
3119 if (DO_BIC(BIC_RAM_J))
3120 outp += sprintf(outp, "%sRAM_J", (printed++ ? delim : ""));
3121 if (DO_BIC(BIC_PKG__))
3122 outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
3123 if (DO_BIC(BIC_RAM__))
3124 outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
3125 }
3126 if (DO_BIC(BIC_UNCORE_MHZ))
3127 outp += sprintf(outp, "%sUncMHz", (printed++ ? delim : ""));
3128
3129 for (mp = sys.pp; mp; mp = mp->next)
3130 outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format);
3131
3132 for (pp = sys.perf_pp; pp; pp = pp->next)
3133 outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format);
3134
3135 ppmt = sys.pmt_pp;
3136 while (ppmt) {
3137 switch (ppmt->type) {
3138 case PMT_TYPE_RAW:
3139 outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
3140 break;
3141
3142 case PMT_TYPE_XTAL_TIME:
3143 case PMT_TYPE_TCORE_CLOCK:
3144 outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
3145 break;
3146 }
3147
3148 ppmt = ppmt->next;
3149 }
3150
3151 if (DO_BIC(BIC_SysWatt))
3152 outp += sprintf(outp, "%sSysWatt", (printed++ ? delim : ""));
3153 if (DO_BIC(BIC_Sys_J))
3154 outp += sprintf(outp, "%sSys_J", (printed++ ? delim : ""));
3155
3156 outp += sprintf(outp, "\n");
3157 }
3158
3159 /*
3160 * pct(numerator, denominator)
3161 *
3162 * Return sanity checked percentage (100.0 * numerator/denominotor)
3163 *
3164 * n < 0: nan
3165 * d <= 0: nan
3166 * n/d > 1.1: nan
3167 */
pct(double numerator,double denominator)3168 double pct(double numerator, double denominator)
3169 {
3170 double retval;
3171
3172 if (numerator < 0)
3173 return nan("");
3174
3175 if (denominator <= 0)
3176 return nan("");
3177
3178 retval = 100.0 * numerator / denominator;
3179
3180 if (retval > 110.0)
3181 return nan("");
3182
3183 return retval;
3184 }
3185
dump_counters(PER_THREAD_PARAMS)3186 int dump_counters(PER_THREAD_PARAMS)
3187 {
3188 int i;
3189 struct msr_counter *mp;
3190 struct platform_counters *pplat_cnt = p == odd.packages ? &platform_counters_odd : &platform_counters_even;
3191
3192 outp += sprintf(outp, "t %p, c %p, p %p\n", t, c, p);
3193
3194 if (t) {
3195 outp += sprintf(outp, "CPU: %d flags 0x%x\n", t->cpu_id, t->flags);
3196 outp += sprintf(outp, "TSC: %016llX\n", t->tsc);
3197 outp += sprintf(outp, "aperf: %016llX\n", t->aperf);
3198 outp += sprintf(outp, "mperf: %016llX\n", t->mperf);
3199 outp += sprintf(outp, "c1: %016llX\n", t->c1);
3200
3201 if (DO_BIC(BIC_IPC))
3202 outp += sprintf(outp, "IPC: %lld\n", t->instr_count);
3203
3204 if (DO_BIC(BIC_IRQ))
3205 outp += sprintf(outp, "IRQ: %lld\n", t->irq_count);
3206 if (DO_BIC(BIC_NMI))
3207 outp += sprintf(outp, "IRQ: %lld\n", t->nmi_count);
3208 if (DO_BIC(BIC_SMI))
3209 outp += sprintf(outp, "SMI: %d\n", t->smi_count);
3210
3211 outp += sprintf(outp, "LLC refs: %lld", t->llc.references);
3212 outp += sprintf(outp, "LLC miss: %lld", t->llc.misses);
3213 outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses), t->llc.references));
3214
3215 outp += sprintf(outp, "L2 refs: %lld", t->l2.references);
3216 outp += sprintf(outp, "L2 hits: %lld", t->l2.hits);
3217 outp += sprintf(outp, "L2 Hit%%: %.2f", pct(t->l2.hits, t->l2.references));
3218
3219 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3220 outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path);
3221 }
3222 }
3223
3224 if (c && is_cpu_first_thread_in_core(t, c)) {
3225 outp += sprintf(outp, "core: 0x%x\n", cpus[t->cpu_id].core_id);
3226 outp += sprintf(outp, "c3: %016llX\n", c->c3);
3227 outp += sprintf(outp, "c6: %016llX\n", c->c6);
3228 outp += sprintf(outp, "c7: %016llX\n", c->c7);
3229 outp += sprintf(outp, "DTS: %dC\n", c->core_temp_c);
3230 outp += sprintf(outp, "cpu_throt_count: %016llX\n", c->core_throt_cnt);
3231
3232 const unsigned long long energy_value = c->core_energy.raw_value * c->core_energy.scale;
3233 const double energy_scale = c->core_energy.scale;
3234
3235 if (c->core_energy.unit == RAPL_UNIT_JOULES)
3236 outp += sprintf(outp, "Joules: %0llX (scale: %lf)\n", energy_value, energy_scale);
3237
3238 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3239 outp += sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, c->counter[i], mp->sp->path);
3240 }
3241 outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us);
3242 }
3243
3244 if (p && is_cpu_first_core_in_package(t, p)) {
3245 outp += sprintf(outp, "Weighted cores: %016llX\n", p->pkg_wtd_core_c0);
3246 outp += sprintf(outp, "Any cores: %016llX\n", p->pkg_any_core_c0);
3247 outp += sprintf(outp, "Any GFX: %016llX\n", p->pkg_any_gfxe_c0);
3248 outp += sprintf(outp, "CPU + GFX: %016llX\n", p->pkg_both_core_gfxe_c0);
3249
3250 outp += sprintf(outp, "pc2: %016llX\n", p->pc2);
3251 if (DO_BIC(BIC_Pkgpc3))
3252 outp += sprintf(outp, "pc3: %016llX\n", p->pc3);
3253 if (DO_BIC(BIC_Pkgpc6))
3254 outp += sprintf(outp, "pc6: %016llX\n", p->pc6);
3255 if (DO_BIC(BIC_Pkgpc7))
3256 outp += sprintf(outp, "pc7: %016llX\n", p->pc7);
3257 outp += sprintf(outp, "pc8: %016llX\n", p->pc8);
3258 outp += sprintf(outp, "pc9: %016llX\n", p->pc9);
3259 outp += sprintf(outp, "pc10: %016llX\n", p->pc10);
3260 outp += sprintf(outp, "cpu_lpi: %016llX\n", p->cpu_lpi);
3261 outp += sprintf(outp, "sys_lpi: %016llX\n", p->sys_lpi);
3262 outp += sprintf(outp, "Joules PKG: %0llX\n", p->energy_pkg.raw_value);
3263 outp += sprintf(outp, "Joules COR: %0llX\n", p->energy_cores.raw_value);
3264 outp += sprintf(outp, "Joules GFX: %0llX\n", p->energy_gfx.raw_value);
3265 outp += sprintf(outp, "Joules RAM: %0llX\n", p->energy_dram.raw_value);
3266 outp += sprintf(outp, "Joules PSYS: %0llX\n", pplat_cnt->energy_psys.raw_value);
3267 outp += sprintf(outp, "Throttle PKG: %0llX\n", p->rapl_pkg_perf_status.raw_value);
3268 outp += sprintf(outp, "Throttle RAM: %0llX\n", p->rapl_dram_perf_status.raw_value);
3269 outp += sprintf(outp, "PTM: %dC\n", p->pkg_temp_c);
3270
3271 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3272 outp += sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, p->counter[i], mp->sp->path);
3273 }
3274 }
3275
3276 outp += sprintf(outp, "\n");
3277
3278 return 0;
3279 }
3280
rapl_counter_get_value(const struct rapl_counter * c,enum rapl_unit desired_unit,double interval)3281 double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desired_unit, double interval)
3282 {
3283 assert(desired_unit != RAPL_UNIT_INVALID);
3284
3285 /*
3286 * For now we don't expect anything other than joules,
3287 * so just simplify the logic.
3288 */
3289 assert(c->unit == RAPL_UNIT_JOULES);
3290
3291 const double scaled = c->raw_value * c->scale;
3292
3293 if (desired_unit == RAPL_UNIT_WATTS)
3294 return scaled / interval;
3295 return scaled;
3296 }
3297
get_perf_llc_stats(int cpu,struct llc_stats * llc)3298 void get_perf_llc_stats(int cpu, struct llc_stats *llc)
3299 {
3300 struct read_format {
3301 unsigned long long num_read;
3302 struct llc_stats llc;
3303 } r;
3304 const ssize_t expected_read_size = sizeof(r);
3305 ssize_t actual_read_size;
3306
3307 actual_read_size = read(fd_llc_percpu[cpu], &r, expected_read_size);
3308
3309 if (actual_read_size == -1)
3310 err(-1, "%s(cpu%d,) %d,,%ld", __func__, cpu, fd_llc_percpu[cpu], expected_read_size);
3311
3312 llc->references = r.llc.references;
3313 llc->misses = r.llc.misses;
3314 if (actual_read_size != expected_read_size)
3315 warn("%s: failed to read perf_data (req %zu act %zu)", __func__, expected_read_size, actual_read_size);
3316 }
3317
get_perf_l2_stats(int cpu,struct l2_stats * l2)3318 void get_perf_l2_stats(int cpu, struct l2_stats *l2)
3319 {
3320 struct read_format {
3321 unsigned long long num_read;
3322 struct l2_stats l2;
3323 } r;
3324 const ssize_t expected_read_size = sizeof(r);
3325 ssize_t actual_read_size;
3326
3327 actual_read_size = read(fd_l2_percpu[cpu], &r, expected_read_size);
3328
3329 if (actual_read_size == -1)
3330 err(-1, "%s(cpu%d,) %d,,%ld", __func__, cpu, fd_l2_percpu[cpu], expected_read_size);
3331
3332 l2->references = r.l2.references;
3333 l2->hits = r.l2.hits;
3334 if (actual_read_size != expected_read_size)
3335 warn("%s: cpu%d: failed to read(%d) perf_data (req %zu act %zu)", __func__, cpu, fd_l2_percpu[cpu], expected_read_size, actual_read_size);
3336 }
3337
3338 /*
3339 * column formatting convention & formats
3340 */
format_counters(PER_THREAD_PARAMS)3341 int format_counters(PER_THREAD_PARAMS)
3342 {
3343 static int count;
3344
3345 struct platform_counters *pplat_cnt = NULL;
3346 double interval_float, tsc;
3347 char *fmt8 = "%s%.2f";
3348
3349 int i;
3350 struct msr_counter *mp;
3351 struct perf_counter_info *pp;
3352 struct pmt_counter *ppmt;
3353 char *delim = "\t";
3354 int printed = 0;
3355
3356 if (t == average.threads) {
3357 pplat_cnt = count & 1 ? &platform_counters_odd : &platform_counters_even;
3358 ++count;
3359 }
3360
3361 /* if showing only 1st thread in core and this isn't one, bail out */
3362 if (show_core_only && !is_cpu_first_thread_in_core(t, c))
3363 return 0;
3364
3365 /* if showing only 1st thread in pkg and this isn't one, bail out */
3366 if (show_pkg_only && !is_cpu_first_core_in_package(t, p))
3367 return 0;
3368
3369 /*if not summary line and --cpu is used */
3370 if ((t != average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset)))
3371 return 0;
3372
3373 if (DO_BIC(BIC_USEC)) {
3374 /* on each row, print how many usec each timestamp took to gather */
3375 struct timeval tv;
3376
3377 timersub(&t->tv_end, &t->tv_begin, &tv);
3378 outp += sprintf(outp, "%5ld\t", tv.tv_sec * 1000000 + tv.tv_usec);
3379 }
3380
3381 /* Time_Of_Day_Seconds: on each row, print sec.usec last timestamp taken */
3382 if (DO_BIC(BIC_TOD))
3383 outp += sprintf(outp, "%10ld.%06ld\t", t->tv_end.tv_sec, t->tv_end.tv_usec);
3384
3385 interval_float = t->tv_delta.tv_sec + t->tv_delta.tv_usec / 1000000.0;
3386
3387 tsc = t->tsc * tsc_tweak;
3388
3389 /* topo columns, print blanks on 1st (average) line */
3390 if (t == average.threads) {
3391 if (DO_BIC(BIC_Package))
3392 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3393 if (DO_BIC(BIC_Die))
3394 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3395 if (DO_BIC(BIC_L3))
3396 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3397 if (DO_BIC(BIC_Node))
3398 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3399 if (DO_BIC(BIC_Module))
3400 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3401 if (DO_BIC(BIC_Core))
3402 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3403 if (DO_BIC(BIC_CPU))
3404 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3405 if (DO_BIC(BIC_APIC))
3406 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3407 if (DO_BIC(BIC_X2APIC))
3408 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3409 } else {
3410 if (DO_BIC(BIC_Package)) {
3411 if (p)
3412 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].package_id);
3413 else
3414 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3415 }
3416 if (DO_BIC(BIC_Die)) {
3417 if (c)
3418 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].die_id);
3419 else
3420 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3421 }
3422 if (DO_BIC(BIC_L3)) {
3423 if (c)
3424 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].l3_id);
3425 else
3426 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3427 }
3428 if (DO_BIC(BIC_Node)) {
3429 if (t)
3430 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id);
3431 else
3432 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3433 }
3434 if (DO_BIC(BIC_Module)) {
3435 if (c)
3436 outp += sprintf(outp, "%s0x%x", (printed++ ? delim : ""), cpus[t->cpu_id].module_id);
3437 else
3438 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3439 }
3440 if (DO_BIC(BIC_Core)) {
3441 if (c)
3442 outp += sprintf(outp, "%s0x%x", (printed++ ? delim : ""), cpus[t->cpu_id].core_id);
3443 else
3444 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3445 }
3446 if (DO_BIC(BIC_CPU))
3447 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->cpu_id);
3448 if (DO_BIC(BIC_APIC))
3449 outp += sprintf(outp, "%s0x%x", (printed++ ? delim : ""), t->apic_id);
3450 if (DO_BIC(BIC_X2APIC))
3451 outp += sprintf(outp, "%s0x%x", (printed++ ? delim : ""), t->x2apic_id);
3452 }
3453
3454 if (DO_BIC(BIC_Avg_MHz))
3455 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float);
3456
3457 if (DO_BIC(BIC_Busy))
3458 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf, tsc));
3459
3460 if (DO_BIC(BIC_Bzy_MHz)) {
3461 if (has_base_hz)
3462 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf);
3463 else
3464 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), tsc / units * t->aperf / t->mperf / interval_float);
3465 }
3466
3467 if (DO_BIC(BIC_TSC_MHz))
3468 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 * t->tsc / units / interval_float);
3469
3470 if (DO_BIC(BIC_IPC))
3471 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 1.0 * t->instr_count / t->aperf);
3472
3473 /* IRQ */
3474 if (DO_BIC(BIC_IRQ)) {
3475 if (sums_need_wide_columns)
3476 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->irq_count);
3477 else
3478 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->irq_count);
3479 }
3480
3481 /* NMI */
3482 if (DO_BIC(BIC_NMI)) {
3483 if (sums_need_wide_columns)
3484 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->nmi_count);
3485 else
3486 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->nmi_count);
3487 }
3488
3489 /* SMI */
3490 if (DO_BIC(BIC_SMI))
3491 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count);
3492
3493 /* LLC Stats */
3494 if (DO_BIC(BIC_LLC_MRPS))
3495 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000000);
3496
3497 if (DO_BIC(BIC_LLC_HIT))
3498 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses), t->llc.references));
3499
3500 /* L2 Stats */
3501 if (DO_BIC(BIC_L2_MRPS))
3502 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->l2.references / interval_float / 1000000);
3503
3504 if (DO_BIC(BIC_L2_HIT))
3505 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct(t->l2.hits, t->l2.references));
3506
3507 /* Added Thread Counters */
3508 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3509 if (mp->format == FORMAT_RAW)
3510 outp += print_hex_value(mp->width, &printed, delim, t->counter[i]);
3511 else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3512 outp += print_decimal_value(mp->width, &printed, delim, t->counter[i]);
3513 else if (mp->format == FORMAT_PERCENT) {
3514 if (mp->type == COUNTER_USEC)
3515 outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000);
3516 else
3517 outp += print_float_value(&printed, delim, pct(t->counter[i], tsc));
3518 }
3519 }
3520
3521 /* Added perf Thread Counters */
3522 for (i = 0, pp = sys.perf_tp; pp; ++i, pp = pp->next) {
3523 if (pp->format == FORMAT_RAW)
3524 outp += print_hex_value(pp->width, &printed, delim, t->perf_counter[i]);
3525 else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE)
3526 outp += print_decimal_value(pp->width, &printed, delim, t->perf_counter[i]);
3527 else if (pp->format == FORMAT_PERCENT) {
3528 if (pp->type == COUNTER_USEC)
3529 outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000);
3530 else
3531 outp += print_float_value(&printed, delim, pct(t->perf_counter[i], tsc));
3532 }
3533 }
3534
3535 /* Added PMT Thread Counters */
3536 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
3537 const unsigned long value_raw = t->pmt_counter[i];
3538 double value_converted;
3539 switch (ppmt->type) {
3540 case PMT_TYPE_RAW:
3541 outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, t->pmt_counter[i]);
3542 break;
3543
3544 case PMT_TYPE_XTAL_TIME:
3545 value_converted = pct(value_raw / crystal_hz, interval_float);
3546 outp += print_float_value(&printed, delim, value_converted);
3547 break;
3548
3549 case PMT_TYPE_TCORE_CLOCK:
3550 value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float);
3551 outp += print_float_value(&printed, delim, value_converted);
3552 }
3553 }
3554
3555 /* C1 */
3556 if (DO_BIC(BIC_CPU_c1))
3557 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1, tsc));
3558
3559 /* print per-core data only for 1st thread in core */
3560 if (!is_cpu_first_thread_in_core(t, c))
3561 goto done;
3562
3563 if (DO_BIC(BIC_CPU_c3))
3564 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3, tsc));
3565 if (DO_BIC(BIC_CPU_c6))
3566 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6, tsc));
3567 if (DO_BIC(BIC_CPU_c7))
3568 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7, tsc));
3569
3570 /* Mod%c6 */
3571 if (DO_BIC(BIC_Mod_c6))
3572 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us, tsc));
3573
3574 if (DO_BIC(BIC_CoreTmp))
3575 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c);
3576
3577 /* Core throttle count */
3578 if (DO_BIC(BIC_CORE_THROT_CNT))
3579 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt);
3580
3581 /* Added Core Counters */
3582 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3583 if (mp->format == FORMAT_RAW)
3584 outp += print_hex_value(mp->width, &printed, delim, c->counter[i]);
3585 else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3586 outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]);
3587 else if (mp->format == FORMAT_PERCENT)
3588 outp += print_float_value(&printed, delim, pct(c->counter[i], tsc));
3589 }
3590
3591 /* Added perf Core counters */
3592 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3593 if (pp->format == FORMAT_RAW)
3594 outp += print_hex_value(pp->width, &printed, delim, c->perf_counter[i]);
3595 else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE)
3596 outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]);
3597 else if (pp->format == FORMAT_PERCENT)
3598 outp += print_float_value(&printed, delim, pct(c->perf_counter[i], tsc));
3599 }
3600
3601 /* Added PMT Core counters */
3602 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3603 const unsigned long value_raw = c->pmt_counter[i];
3604 double value_converted;
3605 switch (ppmt->type) {
3606 case PMT_TYPE_RAW:
3607 outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, c->pmt_counter[i]);
3608 break;
3609
3610 case PMT_TYPE_XTAL_TIME:
3611 value_converted = pct(value_raw / crystal_hz, interval_float);
3612 outp += print_float_value(&printed, delim, value_converted);
3613 break;
3614
3615 case PMT_TYPE_TCORE_CLOCK:
3616 value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float);
3617 outp += print_float_value(&printed, delim, value_converted);
3618 }
3619 }
3620
3621 if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
3622 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_WATTS, interval_float));
3623 if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
3624 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_JOULES, interval_float));
3625
3626 /* print per-package data only for 1st core in package */
3627 if (!is_cpu_first_core_in_package(t, p))
3628 goto done;
3629
3630 /* PkgTmp */
3631 if (DO_BIC(BIC_PkgTmp))
3632 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->pkg_temp_c);
3633
3634 /* GFXrc6 */
3635 if (DO_BIC(BIC_GFX_rc6)) {
3636 if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */
3637 outp += sprintf(outp, "%s**.**", (printed++ ? delim : ""));
3638 } else {
3639 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->gfx_rc6_ms / 10.0 / interval_float);
3640 }
3641 }
3642
3643 /* GFXMHz */
3644 if (DO_BIC(BIC_GFXMHz))
3645 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_mhz);
3646
3647 /* GFXACTMHz */
3648 if (DO_BIC(BIC_GFXACTMHz))
3649 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_act_mhz);
3650
3651 /* SAMmc6 */
3652 if (DO_BIC(BIC_SAM_mc6)) {
3653 if (p->sam_mc6_ms == -1) { /* detect GFX counter reset */
3654 outp += sprintf(outp, "%s**.**", (printed++ ? delim : ""));
3655 } else {
3656 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->sam_mc6_ms / 10.0 / interval_float);
3657 }
3658 }
3659
3660 /* SAMMHz */
3661 if (DO_BIC(BIC_SAMMHz))
3662 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_mhz);
3663
3664 /* SAMACTMHz */
3665 if (DO_BIC(BIC_SAMACTMHz))
3666 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_act_mhz);
3667
3668 /* Totl%C0, Any%C0 GFX%C0 CPUGFX% */
3669 if (DO_BIC(BIC_Totl_c0))
3670 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */
3671 if (DO_BIC(BIC_Any_c0))
3672 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0, tsc));
3673 if (DO_BIC(BIC_GFX_c0))
3674 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0, tsc));
3675 if (DO_BIC(BIC_CPUGFX))
3676 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0, tsc));
3677
3678 if (DO_BIC(BIC_Pkgpc2))
3679 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2, tsc));
3680 if (DO_BIC(BIC_Pkgpc3))
3681 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3, tsc));
3682 if (DO_BIC(BIC_Pkgpc6))
3683 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6, tsc));
3684 if (DO_BIC(BIC_Pkgpc7))
3685 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7, tsc));
3686 if (DO_BIC(BIC_Pkgpc8))
3687 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8, tsc));
3688 if (DO_BIC(BIC_Pkgpc9))
3689 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9, tsc));
3690 if (DO_BIC(BIC_Pkgpc10))
3691 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10, tsc));
3692
3693 if (DO_BIC(BIC_Diec6))
3694 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz, interval_float));
3695
3696 if (DO_BIC(BIC_CPU_LPI)) {
3697 if (p->cpu_lpi >= 0)
3698 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0, interval_float));
3699 else
3700 outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
3701 }
3702 if (DO_BIC(BIC_SYS_LPI)) {
3703 if (p->sys_lpi >= 0)
3704 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0, interval_float));
3705 else
3706 outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
3707 }
3708
3709 if (DO_BIC(BIC_PkgWatt))
3710 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_WATTS, interval_float));
3711 if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
3712 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_WATTS, interval_float));
3713 if (DO_BIC(BIC_GFXWatt))
3714 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_WATTS, interval_float));
3715 if (DO_BIC(BIC_RAMWatt))
3716 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_WATTS, interval_float));
3717 if (DO_BIC(BIC_Pkg_J))
3718 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_JOULES, interval_float));
3719 if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
3720 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_JOULES, interval_float));
3721 if (DO_BIC(BIC_GFX_J))
3722 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_JOULES, interval_float));
3723 if (DO_BIC(BIC_RAM_J))
3724 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float));
3725 if (DO_BIC(BIC_PKG__))
3726 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float));
3727 if (DO_BIC(BIC_RAM__))
3728 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float));
3729 /* UncMHz */
3730 if (DO_BIC(BIC_UNCORE_MHZ))
3731 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz);
3732
3733 /* Added Package Counters */
3734 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3735 if (mp->format == FORMAT_RAW)
3736 outp += print_hex_value(mp->width, &printed, delim, p->counter[i]);
3737 else if (mp->type == COUNTER_K2M)
3738 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->counter[i] / 1000);
3739 else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3740 outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]);
3741 else if (mp->format == FORMAT_PERCENT)
3742 outp += print_float_value(&printed, delim, pct(p->counter[i], tsc));
3743 }
3744
3745 /* Added perf Package Counters */
3746 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3747 if (pp->format == FORMAT_RAW)
3748 outp += print_hex_value(pp->width, &printed, delim, p->perf_counter[i]);
3749 else if (pp->type == COUNTER_K2M)
3750 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000);
3751 else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE)
3752 outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]);
3753 else if (pp->format == FORMAT_PERCENT)
3754 outp += print_float_value(&printed, delim, pct(p->perf_counter[i], tsc));
3755 }
3756
3757 /* Added PMT Package Counters */
3758 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3759 const unsigned long value_raw = p->pmt_counter[i];
3760 double value_converted;
3761 switch (ppmt->type) {
3762 case PMT_TYPE_RAW:
3763 outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, p->pmt_counter[i]);
3764 break;
3765
3766 case PMT_TYPE_XTAL_TIME:
3767 value_converted = pct(value_raw / crystal_hz, interval_float);
3768 outp += print_float_value(&printed, delim, value_converted);
3769 break;
3770
3771 case PMT_TYPE_TCORE_CLOCK:
3772 value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float);
3773 outp += print_float_value(&printed, delim, value_converted);
3774 }
3775 }
3776
3777 if (DO_BIC(BIC_SysWatt) && (t == average.threads))
3778 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_WATTS, interval_float));
3779 if (DO_BIC(BIC_Sys_J) && (t == average.threads))
3780 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_JOULES, interval_float));
3781
3782 done:
3783 if (*(outp - 1) != '\n')
3784 outp += sprintf(outp, "\n");
3785
3786 return 0;
3787 }
3788
flush_output_stdout(void)3789 void flush_output_stdout(void)
3790 {
3791 FILE *filep;
3792
3793 if (outf == stderr)
3794 filep = stdout;
3795 else
3796 filep = outf;
3797
3798 fputs(output_buffer, filep);
3799 fflush(filep);
3800
3801 outp = output_buffer;
3802 }
3803
flush_output_stderr(void)3804 void flush_output_stderr(void)
3805 {
3806 fputs(output_buffer, outf);
3807 fflush(outf);
3808 outp = output_buffer;
3809 }
3810
format_all_counters(PER_THREAD_PARAMS)3811 void format_all_counters(PER_THREAD_PARAMS)
3812 {
3813 static int count;
3814
3815 if ((!count || (header_iterations && !(count % header_iterations))) || !summary_only)
3816 print_header("\t");
3817
3818 format_counters(average.threads, average.cores, average.packages);
3819
3820 count++;
3821
3822 if (summary_only)
3823 return;
3824
3825 for_all_cpus(format_counters, t, c, p);
3826 }
3827
3828 #define DELTA_WRAP32(new, old) \
3829 old = ((((unsigned long long)new << 32) - ((unsigned long long)old << 32)) >> 32);
3830
delta_package(struct pkg_data * new,struct pkg_data * old)3831 int delta_package(struct pkg_data *new, struct pkg_data *old)
3832 {
3833 int i;
3834 struct msr_counter *mp;
3835 struct perf_counter_info *pp;
3836 struct pmt_counter *ppmt;
3837
3838 if (DO_BIC(BIC_Totl_c0))
3839 old->pkg_wtd_core_c0 = new->pkg_wtd_core_c0 - old->pkg_wtd_core_c0;
3840 if (DO_BIC(BIC_Any_c0))
3841 old->pkg_any_core_c0 = new->pkg_any_core_c0 - old->pkg_any_core_c0;
3842 if (DO_BIC(BIC_GFX_c0))
3843 old->pkg_any_gfxe_c0 = new->pkg_any_gfxe_c0 - old->pkg_any_gfxe_c0;
3844 if (DO_BIC(BIC_CPUGFX))
3845 old->pkg_both_core_gfxe_c0 = new->pkg_both_core_gfxe_c0 - old->pkg_both_core_gfxe_c0;
3846
3847 old->pc2 = new->pc2 - old->pc2;
3848 if (DO_BIC(BIC_Pkgpc3))
3849 old->pc3 = new->pc3 - old->pc3;
3850 if (DO_BIC(BIC_Pkgpc6))
3851 old->pc6 = new->pc6 - old->pc6;
3852 if (DO_BIC(BIC_Pkgpc7))
3853 old->pc7 = new->pc7 - old->pc7;
3854 old->pc8 = new->pc8 - old->pc8;
3855 old->pc9 = new->pc9 - old->pc9;
3856 old->pc10 = new->pc10 - old->pc10;
3857 old->die_c6 = new->die_c6 - old->die_c6;
3858 old->cpu_lpi = new->cpu_lpi - old->cpu_lpi;
3859 old->sys_lpi = new->sys_lpi - old->sys_lpi;
3860 old->pkg_temp_c = new->pkg_temp_c;
3861
3862 /* flag an error when rc6 counter resets/wraps */
3863 if (old->gfx_rc6_ms > new->gfx_rc6_ms)
3864 old->gfx_rc6_ms = -1;
3865 else
3866 old->gfx_rc6_ms = new->gfx_rc6_ms - old->gfx_rc6_ms;
3867
3868 old->uncore_mhz = new->uncore_mhz;
3869 old->gfx_mhz = new->gfx_mhz;
3870 old->gfx_act_mhz = new->gfx_act_mhz;
3871
3872 /* flag an error when mc6 counter resets/wraps */
3873 if (old->sam_mc6_ms > new->sam_mc6_ms)
3874 old->sam_mc6_ms = -1;
3875 else
3876 old->sam_mc6_ms = new->sam_mc6_ms - old->sam_mc6_ms;
3877
3878 old->sam_mhz = new->sam_mhz;
3879 old->sam_act_mhz = new->sam_act_mhz;
3880
3881 old->energy_pkg.raw_value = new->energy_pkg.raw_value - old->energy_pkg.raw_value;
3882 old->energy_cores.raw_value = new->energy_cores.raw_value - old->energy_cores.raw_value;
3883 old->energy_gfx.raw_value = new->energy_gfx.raw_value - old->energy_gfx.raw_value;
3884 old->energy_dram.raw_value = new->energy_dram.raw_value - old->energy_dram.raw_value;
3885 old->rapl_pkg_perf_status.raw_value = new->rapl_pkg_perf_status.raw_value - old->rapl_pkg_perf_status.raw_value;
3886 old->rapl_dram_perf_status.raw_value = new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value;
3887
3888 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3889 if (mp->format == FORMAT_RAW)
3890 old->counter[i] = new->counter[i];
3891 else if (mp->format == FORMAT_AVERAGE)
3892 old->counter[i] = new->counter[i];
3893 else
3894 old->counter[i] = new->counter[i] - old->counter[i];
3895 }
3896
3897 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3898 if (pp->format == FORMAT_RAW)
3899 old->perf_counter[i] = new->perf_counter[i];
3900 else if (pp->format == FORMAT_AVERAGE)
3901 old->perf_counter[i] = new->perf_counter[i];
3902 else
3903 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3904 }
3905
3906 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3907 if (ppmt->format == FORMAT_RAW)
3908 old->pmt_counter[i] = new->pmt_counter[i];
3909 else
3910 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3911 }
3912
3913 return 0;
3914 }
3915
delta_core(struct core_data * new,struct core_data * old)3916 void delta_core(struct core_data *new, struct core_data *old)
3917 {
3918 int i;
3919 struct msr_counter *mp;
3920 struct perf_counter_info *pp;
3921 struct pmt_counter *ppmt;
3922
3923 old->c3 = new->c3 - old->c3;
3924 old->c6 = new->c6 - old->c6;
3925 old->c7 = new->c7 - old->c7;
3926 old->core_temp_c = new->core_temp_c;
3927 old->core_throt_cnt = new->core_throt_cnt - old->core_throt_cnt;
3928 old->mc6_us = new->mc6_us - old->mc6_us;
3929
3930 DELTA_WRAP32(new->core_energy.raw_value, old->core_energy.raw_value);
3931
3932 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3933 if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE)
3934 old->counter[i] = new->counter[i];
3935 else
3936 old->counter[i] = new->counter[i] - old->counter[i];
3937 }
3938
3939 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3940 if (pp->format == FORMAT_RAW)
3941 old->perf_counter[i] = new->perf_counter[i];
3942 else
3943 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3944 }
3945
3946 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3947 if (ppmt->format == FORMAT_RAW)
3948 old->pmt_counter[i] = new->pmt_counter[i];
3949 else
3950 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3951 }
3952 }
3953
soft_c1_residency_display(int bic)3954 int soft_c1_residency_display(int bic)
3955 {
3956 if (!DO_BIC(BIC_CPU_c1) || platform->has_msr_core_c1_res)
3957 return 0;
3958
3959 return DO_BIC_READ(bic);
3960 }
3961
3962 /*
3963 * old = new - old
3964 */
delta_thread(struct thread_data * new,struct thread_data * old,struct core_data * core_delta)3965 int delta_thread(struct thread_data *new, struct thread_data *old, struct core_data *core_delta)
3966 {
3967 int i;
3968 struct msr_counter *mp;
3969 struct perf_counter_info *pp;
3970 struct pmt_counter *ppmt;
3971
3972 /* we run cpuid just the 1st time, copy the results */
3973 if (DO_BIC(BIC_APIC))
3974 new->apic_id = old->apic_id;
3975 if (DO_BIC(BIC_X2APIC))
3976 new->x2apic_id = old->x2apic_id;
3977
3978 /*
3979 * the timestamps from start of measurement interval are in "old"
3980 * the timestamp from end of measurement interval are in "new"
3981 * over-write old w/ new so we can print end of interval values
3982 */
3983
3984 timersub(&new->tv_begin, &old->tv_begin, &old->tv_delta);
3985 old->tv_begin = new->tv_begin;
3986 old->tv_end = new->tv_end;
3987
3988 old->tsc = new->tsc - old->tsc;
3989
3990 /* check for TSC < 1 Mcycles over interval */
3991 if (old->tsc < (1000 * 1000))
3992 errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n"
3993 "You can disable all c-states by booting with \"idle=poll\"\nor just the deep ones with \"processor.max_cstate=1\"");
3994
3995 old->c1 = new->c1 - old->c1;
3996
3997 if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz) || DO_BIC(BIC_IPC)
3998 || soft_c1_residency_display(BIC_Avg_MHz)) {
3999 if ((new->aperf > old->aperf) && (new->mperf > old->mperf)) {
4000 old->aperf = new->aperf - old->aperf;
4001 old->mperf = new->mperf - old->mperf;
4002 } else {
4003 return -1;
4004 }
4005 }
4006
4007 if (platform->has_msr_core_c1_res) {
4008 /*
4009 * Some models have a dedicated C1 residency MSR,
4010 * which should be more accurate than the derivation below.
4011 */
4012 } else {
4013 /*
4014 * As counter collection is not atomic,
4015 * it is possible for mperf's non-halted cycles + idle states
4016 * to exceed TSC's all cycles: show c1 = 0% in that case.
4017 */
4018 if ((old->mperf + core_delta->c3 + core_delta->c6 + core_delta->c7) > (old->tsc * tsc_tweak))
4019 old->c1 = 0;
4020 else {
4021 /* normal case, derive c1 */
4022 old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - core_delta->c6 - core_delta->c7;
4023 }
4024 }
4025
4026 if (old->mperf == 0) {
4027 if (debug > 1)
4028 fprintf(outf, "cpu%d MPERF 0!\n", old->cpu_id);
4029 old->mperf = 1; /* divide by 0 protection */
4030 }
4031
4032 if (DO_BIC(BIC_IPC))
4033 old->instr_count = new->instr_count - old->instr_count;
4034
4035 if (DO_BIC(BIC_IRQ))
4036 old->irq_count = new->irq_count - old->irq_count;
4037
4038 if (DO_BIC(BIC_NMI))
4039 old->nmi_count = new->nmi_count - old->nmi_count;
4040
4041 if (DO_BIC(BIC_SMI))
4042 old->smi_count = new->smi_count - old->smi_count;
4043
4044 if (DO_BIC(BIC_LLC_MRPS) || DO_BIC(BIC_LLC_HIT))
4045 old->llc.references = new->llc.references - old->llc.references;
4046
4047 if (DO_BIC(BIC_LLC_HIT))
4048 old->llc.misses = new->llc.misses - old->llc.misses;
4049
4050 if (DO_BIC(BIC_L2_MRPS) || DO_BIC(BIC_L2_HIT))
4051 old->l2.references = new->l2.references - old->l2.references;
4052
4053 if (DO_BIC(BIC_L2_HIT))
4054 old->l2.hits = new->l2.hits - old->l2.hits;
4055
4056 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4057 if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE)
4058 old->counter[i] = new->counter[i];
4059 else
4060 old->counter[i] = new->counter[i] - old->counter[i];
4061 }
4062
4063 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4064 if (pp->format == FORMAT_RAW)
4065 old->perf_counter[i] = new->perf_counter[i];
4066 else
4067 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
4068 }
4069
4070 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4071 if (ppmt->format == FORMAT_RAW)
4072 old->pmt_counter[i] = new->pmt_counter[i];
4073 else
4074 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
4075 }
4076
4077 return 0;
4078 }
4079
delta_cpu(struct thread_data * t,struct core_data * c,struct pkg_data * p,struct thread_data * t2,struct core_data * c2,struct pkg_data * p2)4080 int delta_cpu(struct thread_data *t, struct core_data *c, struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2)
4081 {
4082 int retval = 0;
4083
4084 /* calculate core delta only for 1st thread in core */
4085 if (is_cpu_first_thread_in_core(t, c))
4086 delta_core(c, c2);
4087
4088 /* always calculate thread delta */
4089 retval = delta_thread(t, t2, c2); /* c2 is core delta */
4090
4091 /* calculate package delta only for 1st core in package */
4092 if (is_cpu_first_core_in_package(t, p))
4093 retval |= delta_package(p, p2);
4094
4095 return retval;
4096 }
4097
delta_platform(struct platform_counters * new,struct platform_counters * old)4098 void delta_platform(struct platform_counters *new, struct platform_counters *old)
4099 {
4100 old->energy_psys.raw_value = new->energy_psys.raw_value - old->energy_psys.raw_value;
4101 }
4102
rapl_counter_clear(struct rapl_counter * c)4103 void rapl_counter_clear(struct rapl_counter *c)
4104 {
4105 c->raw_value = 0;
4106 c->scale = 0.0;
4107 c->unit = RAPL_UNIT_INVALID;
4108 }
4109
clear_counters(PER_THREAD_PARAMS)4110 void clear_counters(PER_THREAD_PARAMS)
4111 {
4112 int i;
4113 struct msr_counter *mp;
4114
4115 t->tv_begin.tv_sec = 0;
4116 t->tv_begin.tv_usec = 0;
4117 t->tv_end.tv_sec = 0;
4118 t->tv_end.tv_usec = 0;
4119 t->tv_delta.tv_sec = 0;
4120 t->tv_delta.tv_usec = 0;
4121
4122 t->tsc = 0;
4123 t->aperf = 0;
4124 t->mperf = 0;
4125 t->c1 = 0;
4126
4127 t->instr_count = 0;
4128
4129 t->irq_count = 0;
4130 t->nmi_count = 0;
4131 t->smi_count = 0;
4132
4133 t->llc.references = 0;
4134 t->llc.misses = 0;
4135
4136 t->l2.references = 0;
4137 t->l2.hits = 0;
4138
4139 c->c3 = 0;
4140 c->c6 = 0;
4141 c->c7 = 0;
4142 c->mc6_us = 0;
4143 c->core_temp_c = 0;
4144 rapl_counter_clear(&c->core_energy);
4145 c->core_throt_cnt = 0;
4146
4147 p->pkg_wtd_core_c0 = 0;
4148 p->pkg_any_core_c0 = 0;
4149 p->pkg_any_gfxe_c0 = 0;
4150 p->pkg_both_core_gfxe_c0 = 0;
4151
4152 p->pc2 = 0;
4153 if (DO_BIC(BIC_Pkgpc3))
4154 p->pc3 = 0;
4155 if (DO_BIC(BIC_Pkgpc6))
4156 p->pc6 = 0;
4157 if (DO_BIC(BIC_Pkgpc7))
4158 p->pc7 = 0;
4159 p->pc8 = 0;
4160 p->pc9 = 0;
4161 p->pc10 = 0;
4162 p->die_c6 = 0;
4163 p->cpu_lpi = 0;
4164 p->sys_lpi = 0;
4165
4166 rapl_counter_clear(&p->energy_pkg);
4167 rapl_counter_clear(&p->energy_dram);
4168 rapl_counter_clear(&p->energy_cores);
4169 rapl_counter_clear(&p->energy_gfx);
4170 rapl_counter_clear(&p->rapl_pkg_perf_status);
4171 rapl_counter_clear(&p->rapl_dram_perf_status);
4172 p->pkg_temp_c = 0;
4173
4174 p->gfx_rc6_ms = 0;
4175 p->uncore_mhz = 0;
4176 p->gfx_mhz = 0;
4177 p->gfx_act_mhz = 0;
4178 p->sam_mc6_ms = 0;
4179 p->sam_mhz = 0;
4180 p->sam_act_mhz = 0;
4181 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next)
4182 t->counter[i] = 0;
4183
4184 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next)
4185 c->counter[i] = 0;
4186
4187 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next)
4188 p->counter[i] = 0;
4189
4190 memset(&t->perf_counter[0], 0, sizeof(t->perf_counter));
4191 memset(&c->perf_counter[0], 0, sizeof(c->perf_counter));
4192 memset(&p->perf_counter[0], 0, sizeof(p->perf_counter));
4193
4194 memset(&t->pmt_counter[0], 0, ARRAY_SIZE(t->pmt_counter));
4195 memset(&c->pmt_counter[0], 0, ARRAY_SIZE(c->pmt_counter));
4196 memset(&p->pmt_counter[0], 0, ARRAY_SIZE(p->pmt_counter));
4197 }
4198
rapl_counter_accumulate(struct rapl_counter * dst,const struct rapl_counter * src)4199 void rapl_counter_accumulate(struct rapl_counter *dst, const struct rapl_counter *src)
4200 {
4201 /* Copy unit and scale from src if dst is not initialized */
4202 if (dst->unit == RAPL_UNIT_INVALID) {
4203 dst->unit = src->unit;
4204 dst->scale = src->scale;
4205 }
4206
4207 assert(dst->unit == src->unit);
4208 assert(dst->scale == src->scale);
4209
4210 dst->raw_value += src->raw_value;
4211 }
4212
sum_counters(PER_THREAD_PARAMS)4213 int sum_counters(PER_THREAD_PARAMS)
4214 {
4215 int i;
4216 struct msr_counter *mp;
4217 struct perf_counter_info *pp;
4218 struct pmt_counter *ppmt;
4219
4220 /* copy un-changing apic_id's */
4221 if (DO_BIC(BIC_APIC))
4222 average.threads->apic_id = t->apic_id;
4223 if (DO_BIC(BIC_X2APIC))
4224 average.threads->x2apic_id = t->x2apic_id;
4225
4226 /* remember first tv_begin */
4227 if (average.threads->tv_begin.tv_sec == 0)
4228 average.threads->tv_begin = procsysfs_tv_begin;
4229
4230 /* remember last tv_end */
4231 average.threads->tv_end = t->tv_end;
4232
4233 average.threads->tsc += t->tsc;
4234 average.threads->aperf += t->aperf;
4235 average.threads->mperf += t->mperf;
4236 average.threads->c1 += t->c1;
4237
4238 average.threads->instr_count += t->instr_count;
4239
4240 average.threads->irq_count += t->irq_count;
4241 average.threads->nmi_count += t->nmi_count;
4242 average.threads->smi_count += t->smi_count;
4243
4244 average.threads->llc.references += t->llc.references;
4245 average.threads->llc.misses += t->llc.misses;
4246
4247 average.threads->l2.references += t->l2.references;
4248 average.threads->l2.hits += t->l2.hits;
4249
4250 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4251 if (mp->format == FORMAT_RAW)
4252 continue;
4253 average.threads->counter[i] += t->counter[i];
4254 }
4255
4256 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4257 if (pp->format == FORMAT_RAW)
4258 continue;
4259 average.threads->perf_counter[i] += t->perf_counter[i];
4260 }
4261
4262 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4263 average.threads->pmt_counter[i] += t->pmt_counter[i];
4264 }
4265
4266 /* sum per-core values only for 1st thread in core */
4267 if (!is_cpu_first_thread_in_core(t, c))
4268 return 0;
4269
4270 average.cores->c3 += c->c3;
4271 average.cores->c6 += c->c6;
4272 average.cores->c7 += c->c7;
4273 average.cores->mc6_us += c->mc6_us;
4274
4275 average.cores->core_temp_c = MAX(average.cores->core_temp_c, c->core_temp_c);
4276 average.cores->core_throt_cnt = MAX(average.cores->core_throt_cnt, c->core_throt_cnt);
4277
4278 rapl_counter_accumulate(&average.cores->core_energy, &c->core_energy);
4279
4280 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
4281 if (mp->format == FORMAT_RAW)
4282 continue;
4283 average.cores->counter[i] += c->counter[i];
4284 }
4285
4286 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
4287 if (pp->format == FORMAT_RAW)
4288 continue;
4289 average.cores->perf_counter[i] += c->perf_counter[i];
4290 }
4291
4292 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
4293 average.cores->pmt_counter[i] += c->pmt_counter[i];
4294 }
4295
4296 /* sum per-pkg values only for 1st core in pkg */
4297 if (!is_cpu_first_core_in_package(t, p))
4298 return 0;
4299
4300 if (DO_BIC(BIC_Totl_c0))
4301 average.packages->pkg_wtd_core_c0 += p->pkg_wtd_core_c0;
4302 if (DO_BIC(BIC_Any_c0))
4303 average.packages->pkg_any_core_c0 += p->pkg_any_core_c0;
4304 if (DO_BIC(BIC_GFX_c0))
4305 average.packages->pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0;
4306 if (DO_BIC(BIC_CPUGFX))
4307 average.packages->pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0;
4308
4309 average.packages->pc2 += p->pc2;
4310 if (DO_BIC(BIC_Pkgpc3))
4311 average.packages->pc3 += p->pc3;
4312 if (DO_BIC(BIC_Pkgpc6))
4313 average.packages->pc6 += p->pc6;
4314 if (DO_BIC(BIC_Pkgpc7))
4315 average.packages->pc7 += p->pc7;
4316 average.packages->pc8 += p->pc8;
4317 average.packages->pc9 += p->pc9;
4318 average.packages->pc10 += p->pc10;
4319 average.packages->die_c6 += p->die_c6;
4320
4321 average.packages->cpu_lpi = p->cpu_lpi;
4322 average.packages->sys_lpi = p->sys_lpi;
4323
4324 rapl_counter_accumulate(&average.packages->energy_pkg, &p->energy_pkg);
4325 rapl_counter_accumulate(&average.packages->energy_dram, &p->energy_dram);
4326 rapl_counter_accumulate(&average.packages->energy_cores, &p->energy_cores);
4327 rapl_counter_accumulate(&average.packages->energy_gfx, &p->energy_gfx);
4328
4329 average.packages->gfx_rc6_ms = p->gfx_rc6_ms;
4330 average.packages->uncore_mhz = p->uncore_mhz;
4331 average.packages->gfx_mhz = p->gfx_mhz;
4332 average.packages->gfx_act_mhz = p->gfx_act_mhz;
4333 average.packages->sam_mc6_ms = p->sam_mc6_ms;
4334 average.packages->sam_mhz = p->sam_mhz;
4335 average.packages->sam_act_mhz = p->sam_act_mhz;
4336
4337 average.packages->pkg_temp_c = MAX(average.packages->pkg_temp_c, p->pkg_temp_c);
4338
4339 rapl_counter_accumulate(&average.packages->rapl_pkg_perf_status, &p->rapl_pkg_perf_status);
4340 rapl_counter_accumulate(&average.packages->rapl_dram_perf_status, &p->rapl_dram_perf_status);
4341
4342 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
4343 if ((mp->format == FORMAT_RAW) && (topo.num_packages == 0))
4344 average.packages->counter[i] = p->counter[i];
4345 else
4346 average.packages->counter[i] += p->counter[i];
4347 }
4348
4349 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
4350 if ((pp->format == FORMAT_RAW) && (topo.num_packages == 0))
4351 average.packages->perf_counter[i] = p->perf_counter[i];
4352 else
4353 average.packages->perf_counter[i] += p->perf_counter[i];
4354 }
4355
4356 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
4357 average.packages->pmt_counter[i] += p->pmt_counter[i];
4358 }
4359
4360 return 0;
4361 }
4362
4363 /*
4364 * sum the counters for all cpus in the system
4365 * compute the weighted average
4366 */
compute_average(PER_THREAD_PARAMS)4367 void compute_average(PER_THREAD_PARAMS)
4368 {
4369 int i;
4370 struct msr_counter *mp;
4371 struct perf_counter_info *pp;
4372 struct pmt_counter *ppmt;
4373
4374 clear_counters(average.threads, average.cores, average.packages);
4375
4376 for_all_cpus(sum_counters, t, c, p);
4377
4378 /* Use the global time delta for the average. */
4379 average.threads->tv_delta = tv_delta;
4380
4381 average.threads->tsc /= topo.allowed_cpus;
4382 average.threads->aperf /= topo.allowed_cpus;
4383 average.threads->mperf /= topo.allowed_cpus;
4384 average.threads->instr_count /= topo.allowed_cpus;
4385 average.threads->c1 /= topo.allowed_cpus;
4386
4387 if (average.threads->irq_count > 9999999)
4388 sums_need_wide_columns = 1;
4389 if (average.threads->nmi_count > 9999999)
4390 sums_need_wide_columns = 1;
4391
4392 average.cores->c3 /= topo.allowed_cores;
4393 average.cores->c6 /= topo.allowed_cores;
4394 average.cores->c7 /= topo.allowed_cores;
4395 average.cores->mc6_us /= topo.allowed_cores;
4396
4397 if (DO_BIC(BIC_Totl_c0))
4398 average.packages->pkg_wtd_core_c0 /= topo.allowed_packages;
4399 if (DO_BIC(BIC_Any_c0))
4400 average.packages->pkg_any_core_c0 /= topo.allowed_packages;
4401 if (DO_BIC(BIC_GFX_c0))
4402 average.packages->pkg_any_gfxe_c0 /= topo.allowed_packages;
4403 if (DO_BIC(BIC_CPUGFX))
4404 average.packages->pkg_both_core_gfxe_c0 /= topo.allowed_packages;
4405
4406 average.packages->pc2 /= topo.allowed_packages;
4407 if (DO_BIC(BIC_Pkgpc3))
4408 average.packages->pc3 /= topo.allowed_packages;
4409 if (DO_BIC(BIC_Pkgpc6))
4410 average.packages->pc6 /= topo.allowed_packages;
4411 if (DO_BIC(BIC_Pkgpc7))
4412 average.packages->pc7 /= topo.allowed_packages;
4413
4414 average.packages->pc8 /= topo.allowed_packages;
4415 average.packages->pc9 /= topo.allowed_packages;
4416 average.packages->pc10 /= topo.allowed_packages;
4417 average.packages->die_c6 /= topo.allowed_packages;
4418
4419 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4420 if (mp->format == FORMAT_RAW)
4421 continue;
4422 if (mp->type == COUNTER_ITEMS) {
4423 if (average.threads->counter[i] > 9999999)
4424 sums_need_wide_columns = 1;
4425 continue;
4426 }
4427 average.threads->counter[i] /= topo.allowed_cpus;
4428 }
4429 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
4430 if (mp->format == FORMAT_RAW)
4431 continue;
4432 if (mp->type == COUNTER_ITEMS) {
4433 if (average.cores->counter[i] > 9999999)
4434 sums_need_wide_columns = 1;
4435 }
4436 average.cores->counter[i] /= topo.allowed_cores;
4437 }
4438 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
4439 if (mp->format == FORMAT_RAW)
4440 continue;
4441 if (mp->type == COUNTER_ITEMS) {
4442 if (average.packages->counter[i] > 9999999)
4443 sums_need_wide_columns = 1;
4444 }
4445 average.packages->counter[i] /= topo.allowed_packages;
4446 }
4447
4448 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4449 if (pp->format == FORMAT_RAW)
4450 continue;
4451 if (pp->type == COUNTER_ITEMS) {
4452 if (average.threads->perf_counter[i] > 9999999)
4453 sums_need_wide_columns = 1;
4454 continue;
4455 }
4456 average.threads->perf_counter[i] /= topo.allowed_cpus;
4457 }
4458 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
4459 if (pp->format == FORMAT_RAW)
4460 continue;
4461 if (pp->type == COUNTER_ITEMS) {
4462 if (average.cores->perf_counter[i] > 9999999)
4463 sums_need_wide_columns = 1;
4464 }
4465 average.cores->perf_counter[i] /= topo.allowed_cores;
4466 }
4467 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
4468 if (pp->format == FORMAT_RAW)
4469 continue;
4470 if (pp->type == COUNTER_ITEMS) {
4471 if (average.packages->perf_counter[i] > 9999999)
4472 sums_need_wide_columns = 1;
4473 }
4474 average.packages->perf_counter[i] /= topo.allowed_packages;
4475 }
4476
4477 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4478 average.threads->pmt_counter[i] /= topo.allowed_cpus;
4479 }
4480 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
4481 average.cores->pmt_counter[i] /= topo.allowed_cores;
4482 }
4483 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
4484 average.packages->pmt_counter[i] /= topo.allowed_packages;
4485 }
4486 }
4487
rdtsc(void)4488 static unsigned long long rdtsc(void)
4489 {
4490 unsigned int low, high;
4491
4492 asm volatile ("rdtsc":"=a" (low), "=d"(high));
4493
4494 return low | ((unsigned long long)high) << 32;
4495 }
4496
4497 /*
4498 * Open a file, and exit on failure
4499 */
fopen_or_die(const char * path,const char * mode)4500 FILE *fopen_or_die(const char *path, const char *mode)
4501 {
4502 FILE *filep = fopen(path, mode);
4503
4504 if (!filep)
4505 err(1, "%s: open failed", path);
4506 return filep;
4507 }
4508
4509 /*
4510 * snapshot_sysfs_counter()
4511 *
4512 * return snapshot of given counter
4513 */
snapshot_sysfs_counter(char * path)4514 unsigned long long snapshot_sysfs_counter(char *path)
4515 {
4516 FILE *fp;
4517 int retval;
4518 unsigned long long counter;
4519
4520 fp = fopen_or_die(path, "r");
4521
4522 retval = fscanf(fp, "%lld", &counter);
4523 if (retval != 1)
4524 err(1, "snapshot_sysfs_counter(%s)", path);
4525
4526 fclose(fp);
4527
4528 return counter;
4529 }
4530
get_mp(int cpu,struct msr_counter * mp,unsigned long long * counterp,char * counter_path)4531 int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp, char *counter_path)
4532 {
4533 if (mp->msr_num != 0) {
4534 assert(!no_msr);
4535 if (get_msr(cpu, mp->msr_num, counterp))
4536 return -1;
4537 } else {
4538 char path[128 + PATH_BYTES];
4539
4540 if (mp->flags & SYSFS_PERCPU) {
4541 sprintf(path, "/sys/devices/system/cpu/cpu%d/%s", cpu, mp->sp->path);
4542
4543 *counterp = snapshot_sysfs_counter(path);
4544 } else {
4545 *counterp = snapshot_sysfs_counter(counter_path);
4546 }
4547 }
4548
4549 return 0;
4550 }
4551
get_legacy_uncore_mhz(int package)4552 unsigned long long get_legacy_uncore_mhz(int package)
4553 {
4554 char path[128];
4555 int die;
4556 static int warn_once;
4557
4558 /*
4559 * for this package, use the first die_id that exists
4560 */
4561 for (die = 0; die <= topo.max_die_id; ++die) {
4562
4563 sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", package, die);
4564
4565 if (access(path, R_OK) == 0)
4566 return (snapshot_sysfs_counter(path) / 1000);
4567 }
4568 if (!warn_once) {
4569 warnx("BUG: %s: No %s", __func__, path);
4570 warn_once = 1;
4571 }
4572
4573 return 0;
4574 }
4575
get_epb(int cpu)4576 int get_epb(int cpu)
4577 {
4578 char path[128 + PATH_BYTES];
4579 unsigned long long msr;
4580 int ret, epb = -1;
4581 FILE *fp;
4582
4583 sprintf(path, "/sys/devices/system/cpu/cpu%d/power/energy_perf_bias", cpu);
4584
4585 fp = fopen(path, "r");
4586 if (!fp)
4587 goto msr_fallback;
4588
4589 ret = fscanf(fp, "%d", &epb);
4590 if (ret != 1)
4591 err(1, "%s(%s)", __func__, path);
4592
4593 fclose(fp);
4594
4595 return epb;
4596
4597 msr_fallback:
4598 if (no_msr)
4599 return -1;
4600
4601 get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr);
4602
4603 return msr & 0xf;
4604 }
4605
get_apic_id(struct thread_data * t)4606 void get_apic_id(struct thread_data *t)
4607 {
4608 unsigned int eax, ebx, ecx, edx;
4609
4610 if (DO_BIC(BIC_APIC)) {
4611 eax = ebx = ecx = edx = 0;
4612 __cpuid(1, eax, ebx, ecx, edx);
4613
4614 t->apic_id = (ebx >> 24) & 0xff;
4615 }
4616
4617 if (!DO_BIC(BIC_X2APIC))
4618 return;
4619
4620 if (authentic_amd || hygon_genuine) {
4621 unsigned int topology_extensions;
4622
4623 if (max_extended_level < 0x8000001e)
4624 return;
4625
4626 eax = ebx = ecx = edx = 0;
4627 __cpuid(0x80000001, eax, ebx, ecx, edx);
4628 topology_extensions = ecx & (1 << 22);
4629
4630 if (topology_extensions == 0)
4631 return;
4632
4633 eax = ebx = ecx = edx = 0;
4634 __cpuid(0x8000001e, eax, ebx, ecx, edx);
4635
4636 t->x2apic_id = eax;
4637 return;
4638 }
4639
4640 if (!genuine_intel)
4641 return;
4642
4643 if (max_level < 0xb)
4644 return;
4645
4646 ecx = 0;
4647 __cpuid(0xb, eax, ebx, ecx, edx);
4648 t->x2apic_id = edx;
4649
4650 if (debug && (t->apic_id != (t->x2apic_id & 0xff)))
4651 fprintf(outf, "cpu%d: BIOS BUG: apic 0x%x x2apic 0x%x\n", t->cpu_id, t->apic_id, t->x2apic_id);
4652 }
4653
get_core_throt_cnt(int cpu,unsigned long long * cnt)4654 int get_core_throt_cnt(int cpu, unsigned long long *cnt)
4655 {
4656 char path[128 + PATH_BYTES];
4657 unsigned long long tmp;
4658 FILE *fp;
4659 int ret;
4660
4661 sprintf(path, "/sys/devices/system/cpu/cpu%d/thermal_throttle/core_throttle_count", cpu);
4662 fp = fopen(path, "r");
4663 if (!fp)
4664 return -1;
4665 ret = fscanf(fp, "%lld", &tmp);
4666 fclose(fp);
4667 if (ret != 1)
4668 return -1;
4669 *cnt = tmp;
4670
4671 return 0;
4672 }
4673
read_perf_counter_info(const char * const path,const char * const parse_format,void * value_ptr)4674 static int read_perf_counter_info(const char *const path, const char *const parse_format, void *value_ptr)
4675 {
4676 int fdmt;
4677 int bytes_read;
4678 char buf[64];
4679 int ret = -1;
4680
4681 fdmt = open(path, O_RDONLY, 0);
4682 if (fdmt == -1) {
4683 if (debug)
4684 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4685 ret = -1;
4686 goto cleanup_and_exit;
4687 }
4688
4689 bytes_read = read(fdmt, buf, sizeof(buf) - 1);
4690 if (bytes_read <= 0 || bytes_read >= (int)sizeof(buf)) {
4691 if (debug)
4692 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4693 ret = -1;
4694 goto cleanup_and_exit;
4695 }
4696
4697 buf[bytes_read] = '\0';
4698
4699 if (sscanf(buf, parse_format, value_ptr) != 1) {
4700 if (debug)
4701 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4702 ret = -1;
4703 goto cleanup_and_exit;
4704 }
4705
4706 ret = 0;
4707
4708 cleanup_and_exit:
4709 close(fdmt);
4710 return ret;
4711 }
4712
read_perf_counter_info_n(const char * const path,const char * const parse_format)4713 static unsigned int read_perf_counter_info_n(const char *const path, const char *const parse_format)
4714 {
4715 unsigned int v;
4716 int status;
4717
4718 status = read_perf_counter_info(path, parse_format, &v);
4719 if (status)
4720 v = -1;
4721
4722 return v;
4723 }
4724
read_perf_type(const char * subsys)4725 static unsigned int read_perf_type(const char *subsys)
4726 {
4727 const char *const path_format = "/sys/bus/event_source/devices/%s/type";
4728 const char *const format = "%u";
4729 char path[128];
4730
4731 snprintf(path, sizeof(path), path_format, subsys);
4732
4733 return read_perf_counter_info_n(path, format);
4734 }
4735
read_perf_config(const char * subsys,const char * event_name)4736 static unsigned int read_perf_config(const char *subsys, const char *event_name)
4737 {
4738 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s";
4739 FILE *fconfig = NULL;
4740 char path[128];
4741 char config_str[64];
4742 unsigned int config;
4743 unsigned int umask;
4744 bool has_config = false;
4745 bool has_umask = false;
4746 unsigned int ret = -1;
4747
4748 snprintf(path, sizeof(path), path_format, subsys, event_name);
4749
4750 fconfig = fopen(path, "r");
4751 if (!fconfig)
4752 return -1;
4753
4754 if (fgets(config_str, ARRAY_SIZE(config_str), fconfig) != config_str)
4755 goto cleanup_and_exit;
4756
4757 for (char *pconfig_str = &config_str[0]; pconfig_str;) {
4758 if (sscanf(pconfig_str, "event=%x", &config) == 1) {
4759 has_config = true;
4760 goto next;
4761 }
4762
4763 if (sscanf(pconfig_str, "umask=%x", &umask) == 1) {
4764 has_umask = true;
4765 goto next;
4766 }
4767
4768 next:
4769 pconfig_str = strchr(pconfig_str, ',');
4770 if (pconfig_str) {
4771 *pconfig_str = '\0';
4772 ++pconfig_str;
4773 }
4774 }
4775
4776 if (!has_umask)
4777 umask = 0;
4778
4779 if (has_config)
4780 ret = (umask << 8) | config;
4781
4782 cleanup_and_exit:
4783 fclose(fconfig);
4784 return ret;
4785 }
4786
read_perf_rapl_unit(const char * subsys,const char * event_name)4787 static unsigned int read_perf_rapl_unit(const char *subsys, const char *event_name)
4788 {
4789 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.unit";
4790 const char *const format = "%s";
4791 char path[128];
4792 char unit_buffer[16];
4793
4794 snprintf(path, sizeof(path), path_format, subsys, event_name);
4795
4796 read_perf_counter_info(path, format, &unit_buffer);
4797 if (strcmp("Joules", unit_buffer) == 0)
4798 return RAPL_UNIT_JOULES;
4799
4800 return RAPL_UNIT_INVALID;
4801 }
4802
read_perf_scale(const char * subsys,const char * event_name)4803 static double read_perf_scale(const char *subsys, const char *event_name)
4804 {
4805 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.scale";
4806 const char *const format = "%lf";
4807 char path[128];
4808 double scale;
4809
4810 snprintf(path, sizeof(path), path_format, subsys, event_name);
4811
4812 if (read_perf_counter_info(path, format, &scale))
4813 return 0.0;
4814
4815 return scale;
4816 }
4817
rapl_counter_info_count_perf(const struct rapl_counter_info_t * rci)4818 size_t rapl_counter_info_count_perf(const struct rapl_counter_info_t *rci)
4819 {
4820 size_t ret = 0;
4821
4822 for (int i = 0; i < NUM_RAPL_COUNTERS; ++i)
4823 if (rci->source[i] == COUNTER_SOURCE_PERF)
4824 ++ret;
4825
4826 return ret;
4827 }
4828
cstate_counter_info_count_perf(const struct cstate_counter_info_t * cci)4829 static size_t cstate_counter_info_count_perf(const struct cstate_counter_info_t *cci)
4830 {
4831 size_t ret = 0;
4832
4833 for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i)
4834 if (cci->source[i] == COUNTER_SOURCE_PERF)
4835 ++ret;
4836
4837 return ret;
4838 }
4839
write_rapl_counter(struct rapl_counter * rc,struct rapl_counter_info_t * rci,unsigned int idx)4840 void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci, unsigned int idx)
4841 {
4842 if (rci->source[idx] == COUNTER_SOURCE_NONE)
4843 return;
4844
4845 rc->raw_value = rci->data[idx];
4846 rc->unit = rci->unit[idx];
4847 rc->scale = rci->scale[idx];
4848 }
4849
get_rapl_counters(int cpu,unsigned int domain,struct core_data * c,struct pkg_data * p)4850 int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct pkg_data *p)
4851 {
4852 struct platform_counters *pplat_cnt = p == odd.packages ? &platform_counters_odd : &platform_counters_even;
4853 unsigned long long perf_data[NUM_RAPL_COUNTERS + 1];
4854 struct rapl_counter_info_t *rci;
4855
4856 if (debug >= 2)
4857 fprintf(stderr, "%s: cpu%d domain%d\n", __func__, cpu, domain);
4858
4859 assert(rapl_counter_info_perdomain);
4860 assert(domain < rapl_counter_info_perdomain_size);
4861
4862 rci = &rapl_counter_info_perdomain[domain];
4863
4864 /*
4865 * If we have any perf counters to read, read them all now, in bulk
4866 */
4867 if (rci->fd_perf != -1) {
4868 size_t num_perf_counters = rapl_counter_info_count_perf(rci);
4869 const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
4870 const ssize_t actual_read_size = read(rci->fd_perf, &perf_data[0], sizeof(perf_data));
4871
4872 if (actual_read_size != expected_read_size)
4873 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size);
4874 }
4875
4876 for (unsigned int i = 0, pi = 1; i < NUM_RAPL_COUNTERS; ++i) {
4877 switch (rci->source[i]) {
4878 case COUNTER_SOURCE_NONE:
4879 rci->data[i] = 0;
4880 break;
4881
4882 case COUNTER_SOURCE_PERF:
4883 assert(pi < ARRAY_SIZE(perf_data));
4884 assert(rci->fd_perf != -1);
4885
4886 if (debug >= 2)
4887 fprintf(stderr, "Reading rapl counter via perf at %u (%llu %e %lf)\n",
4888 i, perf_data[pi], rci->scale[i], perf_data[pi] * rci->scale[i]);
4889
4890 rci->data[i] = perf_data[pi];
4891
4892 ++pi;
4893 break;
4894
4895 case COUNTER_SOURCE_MSR:
4896 if (debug >= 2)
4897 fprintf(stderr, "Reading rapl counter via msr at %u\n", i);
4898
4899 assert(!no_msr);
4900 if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM) {
4901 if (get_msr_sum(cpu, rci->msr[i], &rci->data[i]))
4902 return -13 - i;
4903 } else {
4904 if (get_msr(cpu, rci->msr[i], &rci->data[i]))
4905 return -13 - i;
4906 }
4907
4908 rci->data[i] &= rci->msr_mask[i];
4909 if (rci->msr_shift[i] >= 0)
4910 rci->data[i] >>= abs(rci->msr_shift[i]);
4911 else
4912 rci->data[i] <<= abs(rci->msr_shift[i]);
4913
4914 break;
4915 }
4916 }
4917
4918 BUILD_BUG_ON(NUM_RAPL_COUNTERS != 8);
4919 write_rapl_counter(&p->energy_pkg, rci, RAPL_RCI_INDEX_ENERGY_PKG);
4920 write_rapl_counter(&p->energy_cores, rci, RAPL_RCI_INDEX_ENERGY_CORES);
4921 write_rapl_counter(&p->energy_dram, rci, RAPL_RCI_INDEX_DRAM);
4922 write_rapl_counter(&p->energy_gfx, rci, RAPL_RCI_INDEX_GFX);
4923 write_rapl_counter(&p->rapl_pkg_perf_status, rci, RAPL_RCI_INDEX_PKG_PERF_STATUS);
4924 write_rapl_counter(&p->rapl_dram_perf_status, rci, RAPL_RCI_INDEX_DRAM_PERF_STATUS);
4925 write_rapl_counter(&c->core_energy, rci, RAPL_RCI_INDEX_CORE_ENERGY);
4926 write_rapl_counter(&pplat_cnt->energy_psys, rci, RAPL_RCI_INDEX_ENERGY_PLATFORM);
4927
4928 return 0;
4929 }
4930
find_sysfs_path_by_id(struct sysfs_path * sp,int id)4931 char *find_sysfs_path_by_id(struct sysfs_path *sp, int id)
4932 {
4933 while (sp) {
4934 if (sp->id == id)
4935 return (sp->path);
4936 sp = sp->next;
4937 }
4938 if (debug)
4939 warnx("%s: id%d not found", __func__, id);
4940 return NULL;
4941 }
4942
get_cstate_counters(unsigned int cpu,PER_THREAD_PARAMS)4943 int get_cstate_counters(unsigned int cpu, PER_THREAD_PARAMS)
4944 {
4945 /*
4946 * Overcommit memory a little bit here,
4947 * but skip calculating exact sizes for the buffers.
4948 */
4949 unsigned long long perf_data[NUM_CSTATE_COUNTERS];
4950 unsigned long long perf_data_core[NUM_CSTATE_COUNTERS + 1];
4951 unsigned long long perf_data_pkg[NUM_CSTATE_COUNTERS + 1];
4952
4953 struct cstate_counter_info_t *cci;
4954
4955 if (debug >= 2)
4956 fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
4957
4958 assert(ccstate_counter_info);
4959 assert(cpu <= ccstate_counter_info_size);
4960
4961 ZERO_ARRAY(perf_data);
4962 ZERO_ARRAY(perf_data_core);
4963 ZERO_ARRAY(perf_data_pkg);
4964
4965 cci = &ccstate_counter_info[cpu];
4966
4967 /*
4968 * If we have any perf counters to read, read them all now, in bulk
4969 */
4970 const size_t num_perf_counters = cstate_counter_info_count_perf(cci);
4971 ssize_t expected_read_size = num_perf_counters * sizeof(unsigned long long);
4972 ssize_t actual_read_size_core = 0, actual_read_size_pkg = 0;
4973
4974 if (cci->fd_perf_core != -1) {
4975 /* Each descriptor read begins with number of counters read. */
4976 expected_read_size += sizeof(unsigned long long);
4977
4978 actual_read_size_core = read(cci->fd_perf_core, &perf_data_core[0], sizeof(perf_data_core));
4979
4980 if (actual_read_size_core <= 0)
4981 err(-1, "%s: read perf %s: %ld", __func__, "core", actual_read_size_core);
4982 }
4983
4984 if (cci->fd_perf_pkg != -1) {
4985 /* Each descriptor read begins with number of counters read. */
4986 expected_read_size += sizeof(unsigned long long);
4987
4988 actual_read_size_pkg = read(cci->fd_perf_pkg, &perf_data_pkg[0], sizeof(perf_data_pkg));
4989
4990 if (actual_read_size_pkg <= 0)
4991 err(-1, "%s: read perf %s: %ld", __func__, "pkg", actual_read_size_pkg);
4992 }
4993
4994 const ssize_t actual_read_size_total = actual_read_size_core + actual_read_size_pkg;
4995
4996 if (actual_read_size_total != expected_read_size)
4997 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size_total);
4998
4999 /*
5000 * Copy ccstate and pcstate data into unified buffer.
5001 *
5002 * Skip first element from core and pkg buffers.
5003 * Kernel puts there how many counters were read.
5004 */
5005 const size_t num_core_counters = perf_data_core[0];
5006 const size_t num_pkg_counters = perf_data_pkg[0];
5007
5008 assert(num_perf_counters == num_core_counters + num_pkg_counters);
5009
5010 /* Copy ccstate perf data */
5011 memcpy(&perf_data[0], &perf_data_core[1], num_core_counters * sizeof(unsigned long long));
5012
5013 /* Copy pcstate perf data */
5014 memcpy(&perf_data[num_core_counters], &perf_data_pkg[1], num_pkg_counters * sizeof(unsigned long long));
5015
5016 for (unsigned int i = 0, pi = 0; i < NUM_CSTATE_COUNTERS; ++i) {
5017 switch (cci->source[i]) {
5018 case COUNTER_SOURCE_NONE:
5019 break;
5020
5021 case COUNTER_SOURCE_PERF:
5022 assert(pi < ARRAY_SIZE(perf_data));
5023 assert(cci->fd_perf_core != -1 || cci->fd_perf_pkg != -1);
5024
5025 if (debug >= 2)
5026 fprintf(stderr, "cstate via %s %u: %llu\n", "perf", i, perf_data[pi]);
5027
5028 cci->data[i] = perf_data[pi];
5029
5030 ++pi;
5031 break;
5032
5033 case COUNTER_SOURCE_MSR:
5034 assert(!no_msr);
5035 if (get_msr(cpu, cci->msr[i], &cci->data[i]))
5036 return -13 - i;
5037
5038 if (debug >= 2)
5039 fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", "msr", cci->msr[i], i, cci->data[i]);
5040
5041 break;
5042 }
5043 }
5044
5045 /*
5046 * Helper to write the data only if the source of
5047 * the counter for the current cpu is not none.
5048 *
5049 * Otherwise we would overwrite core data with 0 (default value),
5050 * when invoked for the thread sibling.
5051 */
5052 #define PERF_COUNTER_WRITE_DATA(out_counter, index) do { \
5053 if (cci->source[index] != COUNTER_SOURCE_NONE) \
5054 out_counter = cci->data[index]; \
5055 } while (0)
5056
5057 BUILD_BUG_ON(NUM_CSTATE_COUNTERS != 11);
5058
5059 PERF_COUNTER_WRITE_DATA(t->c1, CCSTATE_RCI_INDEX_C1_RESIDENCY);
5060 PERF_COUNTER_WRITE_DATA(c->c3, CCSTATE_RCI_INDEX_C3_RESIDENCY);
5061 PERF_COUNTER_WRITE_DATA(c->c6, CCSTATE_RCI_INDEX_C6_RESIDENCY);
5062 PERF_COUNTER_WRITE_DATA(c->c7, CCSTATE_RCI_INDEX_C7_RESIDENCY);
5063
5064 PERF_COUNTER_WRITE_DATA(p->pc2, PCSTATE_RCI_INDEX_C2_RESIDENCY);
5065 PERF_COUNTER_WRITE_DATA(p->pc3, PCSTATE_RCI_INDEX_C3_RESIDENCY);
5066 PERF_COUNTER_WRITE_DATA(p->pc6, PCSTATE_RCI_INDEX_C6_RESIDENCY);
5067 PERF_COUNTER_WRITE_DATA(p->pc7, PCSTATE_RCI_INDEX_C7_RESIDENCY);
5068 PERF_COUNTER_WRITE_DATA(p->pc8, PCSTATE_RCI_INDEX_C8_RESIDENCY);
5069 PERF_COUNTER_WRITE_DATA(p->pc9, PCSTATE_RCI_INDEX_C9_RESIDENCY);
5070 PERF_COUNTER_WRITE_DATA(p->pc10, PCSTATE_RCI_INDEX_C10_RESIDENCY);
5071
5072 #undef PERF_COUNTER_WRITE_DATA
5073
5074 return 0;
5075 }
5076
msr_counter_info_count_perf(const struct msr_counter_info_t * mci)5077 size_t msr_counter_info_count_perf(const struct msr_counter_info_t *mci)
5078 {
5079 size_t ret = 0;
5080
5081 for (int i = 0; i < NUM_MSR_COUNTERS; ++i)
5082 if (mci->source[i] == COUNTER_SOURCE_PERF)
5083 ++ret;
5084
5085 return ret;
5086 }
5087
get_smi_aperf_mperf(unsigned int cpu,struct thread_data * t)5088 int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t)
5089 {
5090 unsigned long long perf_data[NUM_MSR_COUNTERS + 1];
5091
5092 struct msr_counter_info_t *mci;
5093
5094 if (debug >= 2)
5095 fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
5096
5097 assert(msr_counter_info);
5098 assert(cpu <= msr_counter_info_size);
5099
5100 mci = &msr_counter_info[cpu];
5101
5102 ZERO_ARRAY(perf_data);
5103 ZERO_ARRAY(mci->data);
5104
5105 if (mci->fd_perf != -1) {
5106 const size_t num_perf_counters = msr_counter_info_count_perf(mci);
5107 const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
5108 const ssize_t actual_read_size = read(mci->fd_perf, &perf_data[0], sizeof(perf_data));
5109
5110 if (actual_read_size != expected_read_size)
5111 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size);
5112 }
5113
5114 for (unsigned int i = 0, pi = 1; i < NUM_MSR_COUNTERS; ++i) {
5115 switch (mci->source[i]) {
5116 case COUNTER_SOURCE_NONE:
5117 break;
5118
5119 case COUNTER_SOURCE_PERF:
5120 assert(pi < ARRAY_SIZE(perf_data));
5121 assert(mci->fd_perf != -1);
5122
5123 if (debug >= 2)
5124 fprintf(stderr, "Reading msr counter via perf at %u: %llu\n", i, perf_data[pi]);
5125
5126 mci->data[i] = perf_data[pi];
5127
5128 ++pi;
5129 break;
5130
5131 case COUNTER_SOURCE_MSR:
5132 assert(!no_msr);
5133
5134 if (get_msr(cpu, mci->msr[i], &mci->data[i]))
5135 return -2 - i;
5136
5137 mci->data[i] &= mci->msr_mask[i];
5138
5139 if (debug >= 2)
5140 fprintf(stderr, "Reading msr counter via msr at %u: %llu\n", i, mci->data[i]);
5141
5142 break;
5143 }
5144 }
5145
5146 BUILD_BUG_ON(NUM_MSR_COUNTERS != 3);
5147 t->aperf = mci->data[MSR_RCI_INDEX_APERF];
5148 t->mperf = mci->data[MSR_RCI_INDEX_MPERF];
5149 t->smi_count = mci->data[MSR_RCI_INDEX_SMI];
5150
5151 return 0;
5152 }
5153
perf_counter_info_read_values(struct perf_counter_info * pp,int cpu,unsigned long long * out,size_t out_size)5154 int perf_counter_info_read_values(struct perf_counter_info *pp, int cpu, unsigned long long *out, size_t out_size)
5155 {
5156 unsigned int domain;
5157 unsigned long long value;
5158 int fd_counter;
5159
5160 for (size_t i = 0; pp; ++i, pp = pp->next) {
5161 domain = cpu_to_domain(pp, cpu);
5162 assert(domain < pp->num_domains);
5163
5164 fd_counter = pp->fd_perf_per_domain[domain];
5165
5166 if (fd_counter == -1)
5167 continue;
5168
5169 if (read(fd_counter, &value, sizeof(value)) != sizeof(value))
5170 return 1;
5171
5172 assert(i < out_size);
5173 out[i] = value * pp->scale;
5174 }
5175
5176 return 0;
5177 }
5178
pmt_gen_value_mask(unsigned int lsb,unsigned int msb)5179 unsigned long pmt_gen_value_mask(unsigned int lsb, unsigned int msb)
5180 {
5181 unsigned long mask;
5182
5183 if (msb == 63)
5184 mask = 0xffffffffffffffff;
5185 else
5186 mask = ((1 << (msb + 1)) - 1);
5187
5188 mask -= (1 << lsb) - 1;
5189
5190 return mask;
5191 }
5192
pmt_read_counter(struct pmt_counter * ppmt,unsigned int domain_id)5193 unsigned long pmt_read_counter(struct pmt_counter *ppmt, unsigned int domain_id)
5194 {
5195 if (domain_id >= ppmt->num_domains)
5196 return 0;
5197
5198 const unsigned long *pmmio = ppmt->domains[domain_id].pcounter;
5199 const unsigned long value = pmmio ? *pmmio : 0;
5200 const unsigned long value_mask = pmt_gen_value_mask(ppmt->lsb, ppmt->msb);
5201 const unsigned long value_shift = ppmt->lsb;
5202
5203 return (value & value_mask) >> value_shift;
5204 }
5205
5206 /* Rapl domain enumeration helpers */
get_rapl_num_domains(void)5207 static inline int get_rapl_num_domains(void)
5208 {
5209 if (!platform->has_per_core_rapl)
5210 return topo.num_packages;
5211
5212 return GLOBAL_CORE_ID(topo.max_core_id, topo.num_packages) + 1;
5213 }
5214
get_rapl_domain_id(int cpu)5215 static inline int get_rapl_domain_id(int cpu)
5216 {
5217 if (!platform->has_per_core_rapl)
5218 return cpus[cpu].package_id;
5219
5220 return GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id);
5221 }
5222
5223 /*
5224 * get_counters(...)
5225 * migrate to cpu
5226 * acquire and record local counters for that cpu
5227 */
get_counters(PER_THREAD_PARAMS)5228 int get_counters(PER_THREAD_PARAMS)
5229 {
5230 int cpu = t->cpu_id;
5231 unsigned long long msr;
5232 struct msr_counter *mp;
5233 struct pmt_counter *pp;
5234 int i;
5235 int status;
5236
5237 if (cpu_migrate(cpu)) {
5238 fprintf(outf, "%s: Could not migrate to CPU %d\n", __func__, cpu);
5239 return -1;
5240 }
5241
5242 gettimeofday(&t->tv_begin, (struct timezone *)NULL);
5243
5244 if (first_counter_read)
5245 get_apic_id(t);
5246
5247 t->tsc = rdtsc(); /* we are running on local CPU of interest */
5248
5249 get_smi_aperf_mperf(cpu, t);
5250
5251 if (DO_BIC(BIC_LLC_MRPS) || DO_BIC(BIC_LLC_HIT))
5252 get_perf_llc_stats(cpu, &t->llc);
5253
5254 if (DO_BIC(BIC_L2_MRPS) || DO_BIC(BIC_L2_HIT))
5255 get_perf_l2_stats(cpu, &t->l2);
5256
5257 if (DO_BIC(BIC_IPC))
5258 if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long))
5259 return -4;
5260
5261 if (DO_BIC(BIC_IRQ))
5262 t->irq_count = irqs_per_cpu[cpu];
5263 if (DO_BIC(BIC_NMI))
5264 t->nmi_count = nmi_per_cpu[cpu];
5265
5266 get_cstate_counters(cpu, t, c, p);
5267
5268 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
5269 if (get_mp(cpu, mp, &t->counter[i], mp->sp->path))
5270 return -10;
5271 }
5272
5273 if (perf_counter_info_read_values(sys.perf_tp, cpu, t->perf_counter, MAX_ADDED_THREAD_COUNTERS))
5274 return -10;
5275
5276 for (i = 0, pp = sys.pmt_tp; pp; i++, pp = pp->next)
5277 t->pmt_counter[i] = pmt_read_counter(pp, t->cpu_id);
5278
5279 /* collect core counters only for 1st thread in core */
5280 if (!is_cpu_first_thread_in_core(t, c))
5281 goto done;
5282
5283 if (platform->has_per_core_rapl) {
5284 status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p);
5285 if (status != 0)
5286 return status;
5287 }
5288
5289 if (DO_BIC(BIC_CPU_c7) && t->is_atom) {
5290 /*
5291 * For Atom CPUs that has core cstate deeper than c6,
5292 * MSR_CORE_C6_RESIDENCY returns residency of cc6 and deeper.
5293 * Minus CC7 (and deeper cstates) residency to get
5294 * accturate cc6 residency.
5295 */
5296 c->c6 -= c->c7;
5297 }
5298
5299 if (DO_BIC(BIC_Mod_c6))
5300 if (get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us))
5301 return -8;
5302
5303 if (DO_BIC(BIC_CoreTmp)) {
5304 if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
5305 return -9;
5306 c->core_temp_c = tj_max - ((msr >> 16) & 0x7F);
5307 }
5308
5309 if (DO_BIC(BIC_CORE_THROT_CNT))
5310 get_core_throt_cnt(cpu, &c->core_throt_cnt);
5311
5312 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
5313 if (get_mp(cpu, mp, &c->counter[i], mp->sp->path))
5314 return -10;
5315 }
5316
5317 if (perf_counter_info_read_values(sys.perf_cp, cpu, c->perf_counter, MAX_ADDED_CORE_COUNTERS))
5318 return -10;
5319
5320 for (i = 0, pp = sys.pmt_cp; pp; i++, pp = pp->next)
5321 c->pmt_counter[i] = pmt_read_counter(pp, cpus[t->cpu_id].core_id);
5322
5323 /* collect package counters only for 1st core in package */
5324 if (!is_cpu_first_core_in_package(t, p))
5325 goto done;
5326
5327 if (DO_BIC(BIC_Totl_c0)) {
5328 if (get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0))
5329 return -10;
5330 }
5331 if (DO_BIC(BIC_Any_c0)) {
5332 if (get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0))
5333 return -11;
5334 }
5335 if (DO_BIC(BIC_GFX_c0)) {
5336 if (get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0))
5337 return -12;
5338 }
5339 if (DO_BIC(BIC_CPUGFX)) {
5340 if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0))
5341 return -13;
5342 }
5343
5344 if (DO_BIC(BIC_CPU_LPI))
5345 p->cpu_lpi = cpuidle_cur_cpu_lpi_us;
5346 if (DO_BIC(BIC_SYS_LPI))
5347 p->sys_lpi = cpuidle_cur_sys_lpi_us;
5348
5349 if (!platform->has_per_core_rapl) {
5350 status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p);
5351 if (status != 0)
5352 return status;
5353 }
5354
5355 if (DO_BIC(BIC_PkgTmp)) {
5356 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
5357 return -17;
5358 p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F);
5359 }
5360
5361 if (DO_BIC(BIC_UNCORE_MHZ))
5362 p->uncore_mhz = get_legacy_uncore_mhz(cpus[t->cpu_id].package_id);
5363
5364 if (DO_BIC(BIC_GFX_rc6))
5365 p->gfx_rc6_ms = gfx_info[GFX_rc6].val_ull;
5366
5367 if (DO_BIC(BIC_GFXMHz))
5368 p->gfx_mhz = gfx_info[GFX_MHz].val;
5369
5370 if (DO_BIC(BIC_GFXACTMHz))
5371 p->gfx_act_mhz = gfx_info[GFX_ACTMHz].val;
5372
5373 if (DO_BIC(BIC_SAM_mc6))
5374 p->sam_mc6_ms = gfx_info[SAM_mc6].val_ull;
5375
5376 if (DO_BIC(BIC_SAMMHz))
5377 p->sam_mhz = gfx_info[SAM_MHz].val;
5378
5379 if (DO_BIC(BIC_SAMACTMHz))
5380 p->sam_act_mhz = gfx_info[SAM_ACTMHz].val;
5381
5382 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
5383 char *path = NULL;
5384
5385 if (mp->msr_num == 0) {
5386 path = find_sysfs_path_by_id(mp->sp, cpus[t->cpu_id].package_id);
5387 if (path == NULL) {
5388 warnx("%s: package_id %d not found", __func__, cpus[t->cpu_id].package_id);
5389 return -10;
5390 }
5391 }
5392 if (get_mp(cpu, mp, &p->counter[i], path))
5393 return -10;
5394 }
5395
5396 if (perf_counter_info_read_values(sys.perf_pp, cpu, p->perf_counter, MAX_ADDED_PACKAGE_COUNTERS))
5397 return -10;
5398
5399 for (i = 0, pp = sys.pmt_pp; pp; i++, pp = pp->next)
5400 p->pmt_counter[i] = pmt_read_counter(pp, cpus[t->cpu_id].package_id);
5401
5402 done:
5403 gettimeofday(&t->tv_end, (struct timezone *)NULL);
5404
5405 return 0;
5406 }
5407
5408 int pkg_cstate_limit = PCLUKN;
5409 char *pkg_cstate_limit_strings[] = { "unknown", "reserved", "pc0", "pc1", "pc2",
5410 "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited"
5411 };
5412
5413 int nhm_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5414 PCLRSV, PCLRSV
5415 };
5416
5417 int snb_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5418 PCLRSV, PCLRSV
5419 };
5420
5421 int hsw_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5422 PCLRSV, PCLRSV
5423 };
5424
5425 int slv_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5426 PCL__6, PCL__7
5427 };
5428
5429 int amt_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5430 PCLRSV, PCLRSV
5431 };
5432
5433 int phi_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5434 PCLRSV, PCLRSV
5435 };
5436
5437 int glm_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5438 PCLRSV, PCLRSV
5439 };
5440
5441 int skx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5442 PCLRSV, PCLRSV
5443 };
5444
5445 int icx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5446 PCLRSV, PCLRSV
5447 };
5448
probe_cst_limit(void)5449 void probe_cst_limit(void)
5450 {
5451 unsigned long long msr;
5452 int *pkg_cstate_limits;
5453
5454 if (!platform->has_nhm_msrs || no_msr)
5455 return;
5456
5457 switch (platform->cst_limit) {
5458 case CST_LIMIT_NHM:
5459 pkg_cstate_limits = nhm_pkg_cstate_limits;
5460 break;
5461 case CST_LIMIT_SNB:
5462 pkg_cstate_limits = snb_pkg_cstate_limits;
5463 break;
5464 case CST_LIMIT_HSW:
5465 pkg_cstate_limits = hsw_pkg_cstate_limits;
5466 break;
5467 case CST_LIMIT_SKX:
5468 pkg_cstate_limits = skx_pkg_cstate_limits;
5469 break;
5470 case CST_LIMIT_ICX:
5471 pkg_cstate_limits = icx_pkg_cstate_limits;
5472 break;
5473 case CST_LIMIT_SLV:
5474 pkg_cstate_limits = slv_pkg_cstate_limits;
5475 break;
5476 case CST_LIMIT_AMT:
5477 pkg_cstate_limits = amt_pkg_cstate_limits;
5478 break;
5479 case CST_LIMIT_KNL:
5480 pkg_cstate_limits = phi_pkg_cstate_limits;
5481 break;
5482 case CST_LIMIT_GMT:
5483 pkg_cstate_limits = glm_pkg_cstate_limits;
5484 break;
5485 default:
5486 return;
5487 }
5488
5489 get_msr(master_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
5490 pkg_cstate_limit = pkg_cstate_limits[msr & 0xF];
5491 }
5492
dump_platform_info(void)5493 static void dump_platform_info(void)
5494 {
5495 unsigned long long msr;
5496 unsigned int ratio;
5497
5498 if (!platform->has_nhm_msrs || no_msr)
5499 return;
5500
5501 get_msr(master_cpu, MSR_PLATFORM_INFO, &msr);
5502
5503 fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", master_cpu, msr);
5504
5505 ratio = (msr >> 40) & 0xFF;
5506 fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n", ratio, bclk, ratio * bclk);
5507
5508 ratio = (msr >> 8) & 0xFF;
5509 fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
5510 }
5511
dump_power_ctl(void)5512 static void dump_power_ctl(void)
5513 {
5514 unsigned long long msr;
5515
5516 if (!platform->has_nhm_msrs || no_msr)
5517 return;
5518
5519 get_msr(master_cpu, MSR_IA32_POWER_CTL, &msr);
5520 fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", master_cpu, msr, msr & 0x2 ? "EN" : "DIS");
5521
5522 /* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */
5523 if (platform->has_cst_prewake_bit)
5524 fprintf(outf, "C-state Pre-wake: %sabled\n", msr & 0x40000000 ? "DIS" : "EN");
5525
5526 return;
5527 }
5528
dump_turbo_ratio_limit2(void)5529 static void dump_turbo_ratio_limit2(void)
5530 {
5531 unsigned long long msr;
5532 unsigned int ratio;
5533
5534 get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT2, &msr);
5535
5536 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", master_cpu, msr);
5537
5538 ratio = (msr >> 8) & 0xFF;
5539 if (ratio)
5540 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 18 active cores\n", ratio, bclk, ratio * bclk);
5541
5542 ratio = (msr >> 0) & 0xFF;
5543 if (ratio)
5544 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 17 active cores\n", ratio, bclk, ratio * bclk);
5545 return;
5546 }
5547
dump_turbo_ratio_limit1(void)5548 static void dump_turbo_ratio_limit1(void)
5549 {
5550 unsigned long long msr;
5551 unsigned int ratio;
5552
5553 get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT1, &msr);
5554
5555 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", master_cpu, msr);
5556
5557 ratio = (msr >> 56) & 0xFF;
5558 if (ratio)
5559 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 16 active cores\n", ratio, bclk, ratio * bclk);
5560
5561 ratio = (msr >> 48) & 0xFF;
5562 if (ratio)
5563 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 15 active cores\n", ratio, bclk, ratio * bclk);
5564
5565 ratio = (msr >> 40) & 0xFF;
5566 if (ratio)
5567 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 14 active cores\n", ratio, bclk, ratio * bclk);
5568
5569 ratio = (msr >> 32) & 0xFF;
5570 if (ratio)
5571 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 13 active cores\n", ratio, bclk, ratio * bclk);
5572
5573 ratio = (msr >> 24) & 0xFF;
5574 if (ratio)
5575 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 12 active cores\n", ratio, bclk, ratio * bclk);
5576
5577 ratio = (msr >> 16) & 0xFF;
5578 if (ratio)
5579 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 11 active cores\n", ratio, bclk, ratio * bclk);
5580
5581 ratio = (msr >> 8) & 0xFF;
5582 if (ratio)
5583 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 10 active cores\n", ratio, bclk, ratio * bclk);
5584
5585 ratio = (msr >> 0) & 0xFF;
5586 if (ratio)
5587 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 9 active cores\n", ratio, bclk, ratio * bclk);
5588 return;
5589 }
5590
dump_turbo_ratio_limits(int trl_msr_offset)5591 static void dump_turbo_ratio_limits(int trl_msr_offset)
5592 {
5593 unsigned long long msr, core_counts;
5594 int shift;
5595
5596 get_msr(master_cpu, trl_msr_offset, &msr);
5597 fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n", master_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr);
5598
5599 if (platform->trl_msrs & TRL_CORECOUNT) {
5600 get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts);
5601 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", master_cpu, core_counts);
5602 } else {
5603 core_counts = 0x0807060504030201;
5604 }
5605
5606 for (shift = 56; shift >= 0; shift -= 8) {
5607 unsigned int ratio, group_size;
5608
5609 ratio = (msr >> shift) & 0xFF;
5610 group_size = (core_counts >> shift) & 0xFF;
5611 if (ratio)
5612 fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio, bclk, ratio * bclk, group_size);
5613 }
5614
5615 return;
5616 }
5617
dump_atom_turbo_ratio_limits(void)5618 static void dump_atom_turbo_ratio_limits(void)
5619 {
5620 unsigned long long msr;
5621 unsigned int ratio;
5622
5623 get_msr(master_cpu, MSR_ATOM_CORE_RATIOS, &msr);
5624 fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", master_cpu, msr & 0xFFFFFFFF);
5625
5626 ratio = (msr >> 0) & 0x3F;
5627 if (ratio)
5628 fprintf(outf, "%d * %.1f = %.1f MHz minimum operating frequency\n", ratio, bclk, ratio * bclk);
5629
5630 ratio = (msr >> 8) & 0x3F;
5631 if (ratio)
5632 fprintf(outf, "%d * %.1f = %.1f MHz low frequency mode (LFM)\n", ratio, bclk, ratio * bclk);
5633
5634 ratio = (msr >> 16) & 0x3F;
5635 if (ratio)
5636 fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
5637
5638 get_msr(master_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr);
5639 fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", master_cpu, msr & 0xFFFFFFFF);
5640
5641 ratio = (msr >> 24) & 0x3F;
5642 if (ratio)
5643 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 4 active cores\n", ratio, bclk, ratio * bclk);
5644
5645 ratio = (msr >> 16) & 0x3F;
5646 if (ratio)
5647 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 3 active cores\n", ratio, bclk, ratio * bclk);
5648
5649 ratio = (msr >> 8) & 0x3F;
5650 if (ratio)
5651 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 2 active cores\n", ratio, bclk, ratio * bclk);
5652
5653 ratio = (msr >> 0) & 0x3F;
5654 if (ratio)
5655 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 1 active core\n", ratio, bclk, ratio * bclk);
5656 }
5657
dump_knl_turbo_ratio_limits(void)5658 static void dump_knl_turbo_ratio_limits(void)
5659 {
5660 const unsigned int buckets_no = 7;
5661
5662 unsigned long long msr;
5663 int delta_cores, delta_ratio;
5664 int i, b_nr;
5665 unsigned int cores[buckets_no];
5666 unsigned int ratio[buckets_no];
5667
5668 get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT, &msr);
5669
5670 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", master_cpu, msr);
5671
5672 /*
5673 * Turbo encoding in KNL is as follows:
5674 * [0] -- Reserved
5675 * [7:1] -- Base value of number of active cores of bucket 1.
5676 * [15:8] -- Base value of freq ratio of bucket 1.
5677 * [20:16] -- +ve delta of number of active cores of bucket 2.
5678 * i.e. active cores of bucket 2 =
5679 * active cores of bucket 1 + delta
5680 * [23:21] -- Negative delta of freq ratio of bucket 2.
5681 * i.e. freq ratio of bucket 2 =
5682 * freq ratio of bucket 1 - delta
5683 * [28:24]-- +ve delta of number of active cores of bucket 3.
5684 * [31:29]-- -ve delta of freq ratio of bucket 3.
5685 * [36:32]-- +ve delta of number of active cores of bucket 4.
5686 * [39:37]-- -ve delta of freq ratio of bucket 4.
5687 * [44:40]-- +ve delta of number of active cores of bucket 5.
5688 * [47:45]-- -ve delta of freq ratio of bucket 5.
5689 * [52:48]-- +ve delta of number of active cores of bucket 6.
5690 * [55:53]-- -ve delta of freq ratio of bucket 6.
5691 * [60:56]-- +ve delta of number of active cores of bucket 7.
5692 * [63:61]-- -ve delta of freq ratio of bucket 7.
5693 */
5694
5695 b_nr = 0;
5696 cores[b_nr] = (msr & 0xFF) >> 1;
5697 ratio[b_nr] = (msr >> 8) & 0xFF;
5698
5699 for (i = 16; i < 64; i += 8) {
5700 delta_cores = (msr >> i) & 0x1F;
5701 delta_ratio = (msr >> (i + 5)) & 0x7;
5702
5703 cores[b_nr + 1] = cores[b_nr] + delta_cores;
5704 ratio[b_nr + 1] = ratio[b_nr] - delta_ratio;
5705 b_nr++;
5706 }
5707
5708 for (i = buckets_no - 1; i >= 0; i--)
5709 if (i > 0 ? ratio[i] != ratio[i - 1] : 1)
5710 fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio[i], bclk, ratio[i] * bclk, cores[i]);
5711 }
5712
dump_cst_cfg(void)5713 static void dump_cst_cfg(void)
5714 {
5715 unsigned long long msr;
5716
5717 if (!platform->has_nhm_msrs || no_msr)
5718 return;
5719
5720 get_msr(master_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
5721
5722 fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", master_cpu, msr);
5723
5724 fprintf(outf, " (%s%s%s%s%slocked, pkg-cstate-limit=%d (%s)",
5725 (msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "",
5726 (msr & SNB_C1_AUTO_UNDEMOTE) ? "UNdemote-C1, " : "",
5727 (msr & NHM_C3_AUTO_DEMOTE) ? "demote-C3, " : "",
5728 (msr & NHM_C1_AUTO_DEMOTE) ? "demote-C1, " : "",
5729 (msr & (1 << 15)) ? "" : "UN", (unsigned int)msr & 0xF, pkg_cstate_limit_strings[pkg_cstate_limit]);
5730
5731 #define AUTOMATIC_CSTATE_CONVERSION (1UL << 16)
5732 if (platform->has_cst_auto_convension) {
5733 fprintf(outf, ", automatic c-state conversion=%s", (msr & AUTOMATIC_CSTATE_CONVERSION) ? "on" : "off");
5734 }
5735
5736 fprintf(outf, ")\n");
5737
5738 return;
5739 }
5740
dump_config_tdp(void)5741 static void dump_config_tdp(void)
5742 {
5743 unsigned long long msr;
5744
5745 get_msr(master_cpu, MSR_CONFIG_TDP_NOMINAL, &msr);
5746 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", master_cpu, msr);
5747 fprintf(outf, " (base_ratio=%d)\n", (unsigned int)msr & 0xFF);
5748
5749 get_msr(master_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr);
5750 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", master_cpu, msr);
5751 if (msr) {
5752 fprintf(outf, "PKG_MIN_PWR_LVL1=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5753 fprintf(outf, "PKG_MAX_PWR_LVL1=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5754 fprintf(outf, "LVL1_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5755 fprintf(outf, "PKG_TDP_LVL1=%d", (unsigned int)(msr) & 0x7FFF);
5756 }
5757 fprintf(outf, ")\n");
5758
5759 get_msr(master_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr);
5760 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", master_cpu, msr);
5761 if (msr) {
5762 fprintf(outf, "PKG_MIN_PWR_LVL2=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5763 fprintf(outf, "PKG_MAX_PWR_LVL2=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5764 fprintf(outf, "LVL2_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5765 fprintf(outf, "PKG_TDP_LVL2=%d", (unsigned int)(msr) & 0x7FFF);
5766 }
5767 fprintf(outf, ")\n");
5768
5769 get_msr(master_cpu, MSR_CONFIG_TDP_CONTROL, &msr);
5770 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", master_cpu, msr);
5771 if ((msr) & 0x3)
5772 fprintf(outf, "TDP_LEVEL=%d ", (unsigned int)(msr) & 0x3);
5773 fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5774 fprintf(outf, ")\n");
5775
5776 get_msr(master_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr);
5777 fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", master_cpu, msr);
5778 fprintf(outf, "MAX_NON_TURBO_RATIO=%d", (unsigned int)(msr) & 0xFF);
5779 fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5780 fprintf(outf, ")\n");
5781 }
5782
5783 unsigned int irtl_time_units[] = { 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };
5784
print_irtl(void)5785 void print_irtl(void)
5786 {
5787 unsigned long long msr;
5788
5789 if (!platform->has_irtl_msrs || no_msr)
5790 return;
5791
5792 if (platform->supported_cstates & PC3) {
5793 get_msr(master_cpu, MSR_PKGC3_IRTL, &msr);
5794 fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", master_cpu, msr);
5795 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5796 }
5797
5798 if (platform->supported_cstates & PC6) {
5799 get_msr(master_cpu, MSR_PKGC6_IRTL, &msr);
5800 fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", master_cpu, msr);
5801 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5802 }
5803
5804 if (platform->supported_cstates & PC7) {
5805 get_msr(master_cpu, MSR_PKGC7_IRTL, &msr);
5806 fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", master_cpu, msr);
5807 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5808 }
5809
5810 if (platform->supported_cstates & PC8) {
5811 get_msr(master_cpu, MSR_PKGC8_IRTL, &msr);
5812 fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", master_cpu, msr);
5813 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5814 }
5815
5816 if (platform->supported_cstates & PC9) {
5817 get_msr(master_cpu, MSR_PKGC9_IRTL, &msr);
5818 fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", master_cpu, msr);
5819 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5820 }
5821
5822 if (platform->supported_cstates & PC10) {
5823 get_msr(master_cpu, MSR_PKGC10_IRTL, &msr);
5824 fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", master_cpu, msr);
5825 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5826 }
5827 }
5828
free_fd_percpu(void)5829 void free_fd_percpu(void)
5830 {
5831 int i;
5832
5833 if (!fd_percpu)
5834 return;
5835
5836 for (i = 0; i < topo.max_cpu_num + 1; ++i) {
5837 if (fd_percpu[i] != 0)
5838 close(fd_percpu[i]);
5839 }
5840
5841 free(fd_percpu);
5842 fd_percpu = NULL;
5843 }
5844
free_fd_instr_count_percpu(void)5845 void free_fd_instr_count_percpu(void)
5846 {
5847 if (!fd_instr_count_percpu)
5848 return;
5849
5850 for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5851 if (fd_instr_count_percpu[i] != 0)
5852 close(fd_instr_count_percpu[i]);
5853 }
5854
5855 free(fd_instr_count_percpu);
5856 fd_instr_count_percpu = NULL;
5857 }
5858
free_fd_llc_percpu(void)5859 void free_fd_llc_percpu(void)
5860 {
5861 if (!fd_llc_percpu)
5862 return;
5863
5864 for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5865 if (fd_llc_percpu[i] != 0)
5866 close(fd_llc_percpu[i]);
5867 }
5868
5869 free(fd_llc_percpu);
5870 fd_llc_percpu = NULL;
5871
5872 BIC_NOT_PRESENT(BIC_LLC_MRPS);
5873 BIC_NOT_PRESENT(BIC_LLC_HIT);
5874 }
5875
free_fd_l2_percpu(void)5876 void free_fd_l2_percpu(void)
5877 {
5878 if (!fd_l2_percpu)
5879 return;
5880
5881 for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5882 if (fd_l2_percpu[i] != 0)
5883 close(fd_l2_percpu[i]);
5884 }
5885
5886 free(fd_l2_percpu);
5887 fd_l2_percpu = NULL;
5888
5889 BIC_NOT_PRESENT(BIC_L2_MRPS);
5890 BIC_NOT_PRESENT(BIC_L2_HIT);
5891 }
5892
free_fd_cstate(void)5893 void free_fd_cstate(void)
5894 {
5895 if (!ccstate_counter_info)
5896 return;
5897
5898 const int counter_info_num = ccstate_counter_info_size;
5899
5900 for (int counter_id = 0; counter_id < counter_info_num; ++counter_id) {
5901 if (ccstate_counter_info[counter_id].fd_perf_core != -1)
5902 close(ccstate_counter_info[counter_id].fd_perf_core);
5903
5904 if (ccstate_counter_info[counter_id].fd_perf_pkg != -1)
5905 close(ccstate_counter_info[counter_id].fd_perf_pkg);
5906 }
5907
5908 free(ccstate_counter_info);
5909 ccstate_counter_info = NULL;
5910 ccstate_counter_info_size = 0;
5911 }
5912
free_fd_msr(void)5913 void free_fd_msr(void)
5914 {
5915 if (!msr_counter_info)
5916 return;
5917
5918 for (int cpu = 0; cpu < topo.max_cpu_num; ++cpu) {
5919 if (msr_counter_info[cpu].fd_perf != -1)
5920 close(msr_counter_info[cpu].fd_perf);
5921 }
5922
5923 free(msr_counter_info);
5924 msr_counter_info = NULL;
5925 msr_counter_info_size = 0;
5926 }
5927
free_fd_rapl_percpu(void)5928 void free_fd_rapl_percpu(void)
5929 {
5930 if (!rapl_counter_info_perdomain)
5931 return;
5932
5933 const int num_domains = rapl_counter_info_perdomain_size;
5934
5935 for (int domain_id = 0; domain_id < num_domains; ++domain_id) {
5936 if (rapl_counter_info_perdomain[domain_id].fd_perf != -1)
5937 close(rapl_counter_info_perdomain[domain_id].fd_perf);
5938 }
5939
5940 free(rapl_counter_info_perdomain);
5941 rapl_counter_info_perdomain = NULL;
5942 rapl_counter_info_perdomain_size = 0;
5943 }
5944
free_fd_added_perf_counters_(struct perf_counter_info * pp)5945 void free_fd_added_perf_counters_(struct perf_counter_info *pp)
5946 {
5947 if (!pp)
5948 return;
5949
5950 if (!pp->fd_perf_per_domain)
5951 return;
5952
5953 while (pp) {
5954 for (size_t domain = 0; domain < pp->num_domains; ++domain) {
5955 if (pp->fd_perf_per_domain[domain] != -1) {
5956 close(pp->fd_perf_per_domain[domain]);
5957 pp->fd_perf_per_domain[domain] = -1;
5958 }
5959 }
5960
5961 free(pp->fd_perf_per_domain);
5962 pp->fd_perf_per_domain = NULL;
5963
5964 pp = pp->next;
5965 }
5966 }
5967
free_fd_added_perf_counters(void)5968 void free_fd_added_perf_counters(void)
5969 {
5970 free_fd_added_perf_counters_(sys.perf_tp);
5971 free_fd_added_perf_counters_(sys.perf_cp);
5972 free_fd_added_perf_counters_(sys.perf_pp);
5973 }
5974
free_all_buffers(void)5975 void free_all_buffers(void)
5976 {
5977 int i;
5978
5979 CPU_FREE(cpu_present_set);
5980 cpu_present_set = NULL;
5981 cpu_present_setsize = 0;
5982
5983 CPU_FREE(cpu_effective_set);
5984 cpu_effective_set = NULL;
5985 cpu_effective_setsize = 0;
5986
5987 CPU_FREE(cpu_allowed_set);
5988 cpu_allowed_set = NULL;
5989 cpu_allowed_setsize = 0;
5990
5991 CPU_FREE(cpu_affinity_set);
5992 cpu_affinity_set = NULL;
5993 cpu_affinity_setsize = 0;
5994
5995 if (perf_pcore_set) {
5996 CPU_FREE(perf_pcore_set);
5997 perf_pcore_set = NULL;
5998 }
5999
6000 if (perf_ecore_set) {
6001 CPU_FREE(perf_ecore_set);
6002 perf_ecore_set = NULL;
6003 }
6004
6005 if (perf_lcore_set) {
6006 CPU_FREE(perf_lcore_set);
6007 perf_lcore_set = NULL;
6008 }
6009
6010 free(even.threads);
6011 free(even.cores);
6012 free(even.packages);
6013
6014 even.threads = NULL;
6015 even.cores = NULL;
6016 even.packages = NULL;
6017
6018 free(odd.threads);
6019 free(odd.cores);
6020 free(odd.packages);
6021
6022 odd.threads = NULL;
6023 odd.cores = NULL;
6024 odd.packages = NULL;
6025
6026 free(output_buffer);
6027 output_buffer = NULL;
6028 outp = NULL;
6029
6030 free_fd_percpu();
6031 free_fd_instr_count_percpu();
6032 free_fd_llc_percpu();
6033 free_fd_l2_percpu();
6034 free_fd_msr();
6035 free_fd_rapl_percpu();
6036 free_fd_cstate();
6037 free_fd_added_perf_counters();
6038
6039 free(irq_column_2_cpu);
6040 free(irqs_per_cpu);
6041 free(nmi_per_cpu);
6042
6043 for (i = 0; i <= topo.max_cpu_num; ++i) {
6044 if (cpus[i].put_ids)
6045 CPU_FREE(cpus[i].put_ids);
6046 }
6047 free(cpus);
6048 }
6049
6050 /*
6051 * Parse a file containing a single int.
6052 * Return 0 if file can not be opened
6053 * Exit if file can be opened, but can not be parsed
6054 */
parse_int_file(const char * fmt,...)6055 int parse_int_file(const char *fmt, ...)
6056 {
6057 va_list args;
6058 char path[PATH_MAX];
6059 FILE *filep;
6060 int value;
6061
6062 va_start(args, fmt);
6063 vsnprintf(path, sizeof(path), fmt, args);
6064 va_end(args);
6065 filep = fopen(path, "r");
6066 if (!filep)
6067 return 0;
6068 if (fscanf(filep, "%d", &value) != 1)
6069 err(1, "%s: failed to parse number from file", path);
6070 fclose(filep);
6071 return value;
6072 }
6073
6074 /*
6075 * cpu_is_first_core_in_package(cpu)
6076 * return 1 if given CPU is 1st core in package
6077 */
cpu_is_first_core_in_package(int cpu)6078 int cpu_is_first_core_in_package(int cpu)
6079 {
6080 return cpu == parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_siblings_list", cpu);
6081 }
6082
get_package_id(int cpu)6083 int get_package_id(int cpu)
6084 {
6085 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu);
6086 }
6087
get_die_id(int cpu)6088 int get_die_id(int cpu)
6089 {
6090 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/die_id", cpu);
6091 }
6092
get_l3_id(int cpu)6093 int get_l3_id(int cpu)
6094 {
6095 return parse_int_file("/sys/devices/system/cpu/cpu%d/cache/index3/id", cpu);
6096 }
6097
get_module_id(int cpu)6098 int get_module_id(int cpu)
6099 {
6100 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/cluster_id", cpu);
6101 }
6102
get_core_id(int cpu)6103 int get_core_id(int cpu)
6104 {
6105 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_id", cpu);
6106 }
6107
set_node_data(void)6108 void set_node_data(void)
6109 {
6110 int pkg, node, lnode, cpu, cpux;
6111 int cpu_count;
6112
6113 /* initialize logical_node_id */
6114 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu)
6115 cpus[cpu].logical_node_id = -1;
6116
6117 cpu_count = 0;
6118 for (pkg = 0; pkg < topo.num_packages; pkg++) {
6119 lnode = 0;
6120 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
6121 if (cpus[cpu].package_id != pkg)
6122 continue;
6123 /* find a cpu with an unset logical_node_id */
6124 if (cpus[cpu].logical_node_id != -1)
6125 continue;
6126 cpus[cpu].logical_node_id = lnode;
6127 node = cpus[cpu].physical_node_id;
6128 cpu_count++;
6129 /*
6130 * find all matching cpus on this pkg and set
6131 * the logical_node_id
6132 */
6133 for (cpux = cpu; cpux <= topo.max_cpu_num; cpux++) {
6134 if ((cpus[cpux].package_id == pkg) && (cpus[cpux].physical_node_id == node)) {
6135 cpus[cpux].logical_node_id = lnode;
6136 cpu_count++;
6137 }
6138 }
6139 lnode++;
6140 if (lnode > topo.nodes_per_pkg)
6141 topo.nodes_per_pkg = lnode;
6142 }
6143 if (cpu_count >= topo.max_cpu_num)
6144 break;
6145 }
6146 }
6147
get_physical_node_id(struct cpu_topology * thiscpu)6148 int get_physical_node_id(struct cpu_topology *thiscpu)
6149 {
6150 char path[80];
6151 FILE *filep;
6152 int i;
6153 int cpu = thiscpu->cpu_id;
6154
6155 for (i = 0; i <= topo.max_cpu_num; i++) {
6156 sprintf(path, "/sys/devices/system/cpu/cpu%d/node%i/cpulist", cpu, i);
6157 filep = fopen(path, "r");
6158 if (!filep)
6159 continue;
6160 fclose(filep);
6161 return i;
6162 }
6163 return -1;
6164 }
6165
parse_cpu_str(char * cpu_str,cpu_set_t * cpu_set,int cpu_set_size)6166 static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size)
6167 {
6168 unsigned int start, end;
6169 char *next = cpu_str;
6170
6171 while (next && *next) {
6172
6173 if (*next == '-') /* no negative cpu numbers */
6174 return 1;
6175
6176 if (*next == '\0' || *next == '\n')
6177 break;
6178
6179 start = strtoul(next, &next, 10);
6180
6181 if (start >= CPU_SUBSET_MAXCPUS)
6182 return 1;
6183 CPU_SET_S(start, cpu_set_size, cpu_set);
6184
6185 if (*next == '\0' || *next == '\n')
6186 break;
6187
6188 if (*next == ',') {
6189 next += 1;
6190 continue;
6191 }
6192
6193 if (*next == '-') {
6194 next += 1; /* start range */
6195 } else if (*next == '.') {
6196 next += 1;
6197 if (*next == '.')
6198 next += 1; /* start range */
6199 else
6200 return 1;
6201 }
6202
6203 end = strtoul(next, &next, 10);
6204 if (end <= start)
6205 return 1;
6206
6207 while (++start <= end) {
6208 if (start >= CPU_SUBSET_MAXCPUS)
6209 return 1;
6210 CPU_SET_S(start, cpu_set_size, cpu_set);
6211 }
6212
6213 if (*next == ',')
6214 next += 1;
6215 else if (*next != '\0' && *next != '\n')
6216 return 1;
6217 }
6218
6219 return 0;
6220 }
6221
6222 /*
6223 * run func(thread, core, package) in topology order
6224 * skip non-present cpus
6225 */
6226
for_all_cpus_2(int (func)(struct thread_data *,struct core_data *,struct pkg_data *,struct thread_data *,struct core_data *,struct pkg_data *),struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base,struct thread_data * thread_base2,struct core_data * core_base2,struct pkg_data * pkg_base2)6227 int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *,
6228 struct pkg_data *, struct thread_data *, struct core_data *,
6229 struct pkg_data *), struct thread_data *thread_base,
6230 struct core_data *core_base, struct pkg_data *pkg_base,
6231 struct thread_data *thread_base2, struct core_data *core_base2, struct pkg_data *pkg_base2)
6232 {
6233 int cpu, retval;
6234
6235 retval = 0;
6236
6237 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
6238 struct thread_data *t, *t2;
6239 struct core_data *c, *c2;
6240 struct pkg_data *p, *p2;
6241
6242 if (cpu_is_not_allowed(cpu))
6243 continue;
6244
6245 if (has_allowed_lower_ht_sibling(cpu)) /* skip HT sibling */
6246 continue;
6247
6248 t = &thread_base[cpu];
6249 t2 = &thread_base2[cpu];
6250 c = &core_base[GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id)];
6251 c2 = &core_base2[GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id)];
6252 p = &pkg_base[cpus[cpu].package_id];
6253 p2 = &pkg_base2[cpus[cpu].package_id];
6254
6255 retval |= func(t, c, p, t2, c2, p2);
6256
6257 /* Handle HT sibling now */
6258 int i;
6259
6260 for (i = 0; i <= MAX_HT_ID; ++i) {
6261 int sibling_cpu_id = cpus[cpu].ht_sibling_cpu_id[i];
6262
6263 if (sibling_cpu_id < 0)
6264 break;
6265
6266 if (sibling_cpu_id == cpu)
6267 continue;
6268
6269 if (cpu_is_not_allowed(sibling_cpu_id))
6270 continue;
6271
6272 t = &thread_base[sibling_cpu_id];
6273 t2 = &thread_base2[sibling_cpu_id];
6274
6275 retval |= func(t, c, p, t2, c2, p2);
6276 }
6277 }
6278 return retval;
6279 }
6280
6281 /*
6282 * run func(cpu) on every cpu in /proc/stat
6283 * return max_cpu number
6284 */
for_all_proc_cpus(int (func)(int))6285 int for_all_proc_cpus(int (func) (int))
6286 {
6287 FILE *fp;
6288 int cpu_num;
6289 int retval;
6290
6291 fp = fopen_or_die(proc_stat, "r");
6292
6293 retval = fscanf(fp, "cpu %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n");
6294 if (retval != 0)
6295 err(1, "%s: failed to parse format", proc_stat);
6296
6297 while (1) {
6298 retval = fscanf(fp, "cpu%u %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n", &cpu_num);
6299 if (retval != 1)
6300 break;
6301
6302 retval = func(cpu_num);
6303 if (retval) {
6304 fclose(fp);
6305 return (retval);
6306 }
6307 }
6308 fclose(fp);
6309 return 0;
6310 }
6311
6312 #define PATH_EFFECTIVE_CPUS "/sys/fs/cgroup/cpuset.cpus.effective"
6313
6314 static char cpu_effective_str[1024];
6315
update_effective_str(bool startup)6316 static int update_effective_str(bool startup)
6317 {
6318 FILE *fp;
6319 char *pos;
6320 char buf[1024];
6321 int ret;
6322
6323 if (cpu_effective_str[0] == '\0' && !startup)
6324 return 0;
6325
6326 fp = fopen(PATH_EFFECTIVE_CPUS, "r");
6327 if (!fp)
6328 return 0;
6329
6330 pos = fgets(buf, 1024, fp);
6331 if (!pos)
6332 err(1, "%s: file read failed", PATH_EFFECTIVE_CPUS);
6333
6334 fclose(fp);
6335
6336 ret = strncmp(cpu_effective_str, buf, 1024);
6337 if (!ret)
6338 return 0;
6339
6340 strncpy(cpu_effective_str, buf, 1024);
6341 return 1;
6342 }
6343
update_effective_set(bool startup)6344 static void update_effective_set(bool startup)
6345 {
6346 update_effective_str(startup);
6347
6348 if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize))
6349 err(1, "%s: cpu str malformat %s", PATH_EFFECTIVE_CPUS, cpu_effective_str);
6350 }
6351
6352 void linux_perf_init(void);
6353 void msr_perf_init(void);
6354 void rapl_perf_init(void);
6355 void cstate_perf_init(void);
6356 void perf_llc_init(void);
6357 void perf_l2_init(void);
6358 void added_perf_counters_init(void);
6359 void pmt_init(void);
6360
re_initialize(void)6361 void re_initialize(void)
6362 {
6363 free_all_buffers();
6364 setup_all_buffers(false);
6365 linux_perf_init();
6366 msr_perf_init();
6367 rapl_perf_init();
6368 cstate_perf_init();
6369 perf_llc_init();
6370 perf_l2_init();
6371 added_perf_counters_init();
6372 pmt_init();
6373 fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus);
6374 }
6375
set_max_cpu_num(void)6376 void set_max_cpu_num(void)
6377 {
6378 FILE *filep;
6379 int current_cpu;
6380 unsigned long dummy;
6381 char pathname[64];
6382
6383 current_cpu = sched_getcpu();
6384 if (current_cpu < 0)
6385 err(1, "cannot find calling cpu ID");
6386 sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", current_cpu);
6387
6388 filep = fopen_or_die(pathname, "r");
6389 topo.max_cpu_num = 0;
6390 while (fscanf(filep, "%lx,", &dummy) == 1)
6391 topo.max_cpu_num += BITMASK_SIZE;
6392 fclose(filep);
6393 topo.max_cpu_num--; /* 0 based */
6394 }
6395
6396 /*
6397 * count_cpus()
6398 * remember the last one seen, it will be the max
6399 */
count_cpus(int cpu)6400 int count_cpus(int cpu)
6401 {
6402 UNUSED(cpu);
6403
6404 topo.num_cpus++;
6405 return 0;
6406 }
6407
mark_cpu_present(int cpu)6408 int mark_cpu_present(int cpu)
6409 {
6410 CPU_SET_S(cpu, cpu_present_setsize, cpu_present_set);
6411 return 0;
6412 }
6413
clear_ht_id(int cpu)6414 int clear_ht_id(int cpu)
6415 {
6416 int i;
6417
6418 cpus[cpu].ht_id = -1;
6419 for (i = 0; i <= MAX_HT_ID; ++i)
6420 cpus[cpu].ht_sibling_cpu_id[i] = -1;
6421 return 0;
6422 }
6423
set_my_cpu_type(void)6424 int set_my_cpu_type(void)
6425 {
6426 unsigned int eax, ebx, ecx, edx;
6427 unsigned int max_level;
6428
6429 __cpuid(0, max_level, ebx, ecx, edx);
6430
6431 if (max_level < CPUID_LEAF_MODEL_ID)
6432 return 0;
6433
6434 __cpuid(CPUID_LEAF_MODEL_ID, eax, ebx, ecx, edx);
6435
6436 return (eax >> CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT);
6437 }
6438
set_cpu_hybrid_type(int cpu)6439 int set_cpu_hybrid_type(int cpu)
6440 {
6441 if (cpu_migrate(cpu))
6442 return -1;
6443
6444 int type = set_my_cpu_type();
6445
6446 cpus[cpu].type = type;
6447 return 0;
6448 }
6449
6450 /*
6451 * snapshot_proc_interrupts()
6452 *
6453 * read and record summary of /proc/interrupts
6454 *
6455 * return 1 if config change requires a restart, else return 0
6456 */
snapshot_proc_interrupts(void)6457 int snapshot_proc_interrupts(void)
6458 {
6459 static FILE *fp;
6460 int column, retval;
6461
6462 if (fp == NULL)
6463 fp = fopen_or_die("/proc/interrupts", "r");
6464 else
6465 rewind(fp);
6466
6467 /* read 1st line of /proc/interrupts to get cpu* name for each column */
6468 for (column = 0; column < topo.num_cpus; ++column) {
6469 int cpu_number;
6470
6471 retval = fscanf(fp, " CPU%d", &cpu_number);
6472 if (retval != 1)
6473 break;
6474
6475 if (cpu_number > topo.max_cpu_num) {
6476 warn("/proc/interrupts: cpu%d: > %d", cpu_number, topo.max_cpu_num);
6477 return 1;
6478 }
6479
6480 irq_column_2_cpu[column] = cpu_number;
6481 irqs_per_cpu[cpu_number] = 0;
6482 nmi_per_cpu[cpu_number] = 0;
6483 }
6484
6485 /* read /proc/interrupt count lines and sum up irqs per cpu */
6486 while (1) {
6487 int column;
6488 char buf[64];
6489 int this_row_is_nmi = 0;
6490
6491 retval = fscanf(fp, " %s:", buf); /* irq# "N:" */
6492 if (retval != 1)
6493 break;
6494
6495 if (strncmp(buf, "NMI", strlen("NMI")) == 0)
6496 this_row_is_nmi = 1;
6497
6498 /* read the count per cpu */
6499 for (column = 0; column < topo.num_cpus; ++column) {
6500
6501 int cpu_number, irq_count;
6502
6503 retval = fscanf(fp, " %d", &irq_count);
6504
6505 if (retval != 1)
6506 break;
6507
6508 cpu_number = irq_column_2_cpu[column];
6509 irqs_per_cpu[cpu_number] += irq_count;
6510 if (this_row_is_nmi)
6511 nmi_per_cpu[cpu_number] += irq_count;
6512 }
6513 while (getc(fp) != '\n') ; /* flush interrupt description */
6514
6515 }
6516 return 0;
6517 }
6518
6519 /*
6520 * snapshot_graphics()
6521 *
6522 * record snapshot of specified graphics sysfs knob
6523 *
6524 * return 1 if config change requires a restart, else return 0
6525 */
snapshot_graphics(int idx)6526 int snapshot_graphics(int idx)
6527 {
6528 int retval;
6529
6530 rewind(gfx_info[idx].fp);
6531 fflush(gfx_info[idx].fp);
6532
6533 switch (idx) {
6534 case GFX_rc6:
6535 case SAM_mc6:
6536 retval = fscanf(gfx_info[idx].fp, "%lld", &gfx_info[idx].val_ull);
6537 if (retval != 1)
6538 err(1, "rc6");
6539 return 0;
6540 case GFX_MHz:
6541 case GFX_ACTMHz:
6542 case SAM_MHz:
6543 case SAM_ACTMHz:
6544 retval = fscanf(gfx_info[idx].fp, "%d", &gfx_info[idx].val);
6545 if (retval != 1)
6546 err(1, "MHz");
6547 return 0;
6548 default:
6549 return -EINVAL;
6550 }
6551 }
6552
6553 /*
6554 * snapshot_cpu_lpi()
6555 *
6556 * record snapshot of
6557 * /sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us
6558 */
snapshot_cpu_lpi_us(void)6559 int snapshot_cpu_lpi_us(void)
6560 {
6561 FILE *fp;
6562 int retval;
6563
6564 fp = fopen_or_die("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", "r");
6565
6566 retval = fscanf(fp, "%lld", &cpuidle_cur_cpu_lpi_us);
6567 if (retval != 1) {
6568 fprintf(stderr, "Disabling Low Power Idle CPU output\n");
6569 BIC_NOT_PRESENT(BIC_CPU_LPI);
6570 fclose(fp);
6571 return -1;
6572 }
6573
6574 fclose(fp);
6575
6576 return 0;
6577 }
6578
6579 /*
6580 * snapshot_sys_lpi()
6581 *
6582 * record snapshot of sys_lpi_file
6583 */
snapshot_sys_lpi_us(void)6584 int snapshot_sys_lpi_us(void)
6585 {
6586 FILE *fp;
6587 int retval;
6588
6589 fp = fopen_or_die(sys_lpi_file, "r");
6590
6591 retval = fscanf(fp, "%lld", &cpuidle_cur_sys_lpi_us);
6592 if (retval != 1) {
6593 fprintf(stderr, "Disabling Low Power Idle System output\n");
6594 BIC_NOT_PRESENT(BIC_SYS_LPI);
6595 fclose(fp);
6596 return -1;
6597 }
6598 fclose(fp);
6599
6600 return 0;
6601 }
6602
6603 /*
6604 * snapshot /proc and /sys files
6605 *
6606 * return 1 if configuration restart needed, else return 0
6607 */
snapshot_proc_sysfs_files(void)6608 int snapshot_proc_sysfs_files(void)
6609 {
6610 gettimeofday(&procsysfs_tv_begin, (struct timezone *)NULL);
6611
6612 if (DO_BIC(BIC_IRQ) || DO_BIC(BIC_NMI))
6613 if (snapshot_proc_interrupts())
6614 return 1;
6615
6616 if (DO_BIC(BIC_GFX_rc6))
6617 snapshot_graphics(GFX_rc6);
6618
6619 if (DO_BIC(BIC_GFXMHz))
6620 snapshot_graphics(GFX_MHz);
6621
6622 if (DO_BIC(BIC_GFXACTMHz))
6623 snapshot_graphics(GFX_ACTMHz);
6624
6625 if (DO_BIC(BIC_SAM_mc6))
6626 snapshot_graphics(SAM_mc6);
6627
6628 if (DO_BIC(BIC_SAMMHz))
6629 snapshot_graphics(SAM_MHz);
6630
6631 if (DO_BIC(BIC_SAMACTMHz))
6632 snapshot_graphics(SAM_ACTMHz);
6633
6634 if (DO_BIC(BIC_CPU_LPI))
6635 snapshot_cpu_lpi_us();
6636
6637 if (DO_BIC(BIC_SYS_LPI))
6638 snapshot_sys_lpi_us();
6639
6640 return 0;
6641 }
6642
6643 int exit_requested;
6644
signal_handler(int signal)6645 static void signal_handler(int signal)
6646 {
6647 switch (signal) {
6648 case SIGINT:
6649 exit_requested = 1;
6650 if (debug)
6651 fprintf(stderr, " SIGINT\n");
6652 break;
6653 case SIGUSR1:
6654 if (debug > 1)
6655 fprintf(stderr, "SIGUSR1\n");
6656 break;
6657 }
6658 }
6659
setup_signal_handler(void)6660 void setup_signal_handler(void)
6661 {
6662 struct sigaction sa;
6663
6664 memset(&sa, 0, sizeof(sa));
6665
6666 sa.sa_handler = &signal_handler;
6667
6668 if (sigaction(SIGINT, &sa, NULL) < 0)
6669 err(1, "sigaction SIGINT");
6670 if (sigaction(SIGUSR1, &sa, NULL) < 0)
6671 err(1, "sigaction SIGUSR1");
6672 }
6673
do_sleep(void)6674 void do_sleep(void)
6675 {
6676 struct timeval tout;
6677 struct timespec rest;
6678 fd_set readfds;
6679 int retval;
6680
6681 FD_ZERO(&readfds);
6682 FD_SET(0, &readfds);
6683
6684 if (ignore_stdin) {
6685 nanosleep(&interval_ts, NULL);
6686 return;
6687 }
6688
6689 tout = interval_tv;
6690 retval = select(1, &readfds, NULL, NULL, &tout);
6691
6692 if (retval == 1) {
6693 switch (getc(stdin)) {
6694 case 'q':
6695 exit_requested = 1;
6696 break;
6697 case EOF:
6698 /*
6699 * 'stdin' is a pipe closed on the other end. There
6700 * won't be any further input.
6701 */
6702 ignore_stdin = 1;
6703 /* Sleep the rest of the time */
6704 rest.tv_sec = (tout.tv_sec + tout.tv_usec / 1000000);
6705 rest.tv_nsec = (tout.tv_usec % 1000000) * 1000;
6706 nanosleep(&rest, NULL);
6707 }
6708 }
6709 }
6710
get_msr_sum(int cpu,off_t offset,unsigned long long * msr)6711 int get_msr_sum(int cpu, off_t offset, unsigned long long *msr)
6712 {
6713 int ret, idx;
6714 unsigned long long msr_cur, msr_last;
6715
6716 assert(!no_msr);
6717
6718 if (!per_cpu_msr_sum)
6719 return 1;
6720
6721 idx = offset_to_idx(offset);
6722 if (idx < 0)
6723 return idx;
6724 /* get_msr_sum() = sum + (get_msr() - last) */
6725 ret = get_msr(cpu, offset, &msr_cur);
6726 if (ret)
6727 return ret;
6728 msr_last = per_cpu_msr_sum[cpu].entries[idx].last;
6729 DELTA_WRAP32(msr_cur, msr_last);
6730 *msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum;
6731
6732 return 0;
6733 }
6734
6735 timer_t timerid;
6736
6737 /* Timer callback, update the sum of MSRs periodically. */
update_msr_sum(PER_THREAD_PARAMS)6738 static int update_msr_sum(PER_THREAD_PARAMS)
6739 {
6740 int i, ret;
6741 int cpu = t->cpu_id;
6742
6743 UNUSED(c);
6744 UNUSED(p);
6745
6746 assert(!no_msr);
6747
6748 for (i = IDX_PKG_ENERGY; i < IDX_COUNT; i++) {
6749 unsigned long long msr_cur, msr_last;
6750 off_t offset;
6751
6752 if (!idx_valid(i))
6753 continue;
6754 offset = idx_to_offset(i);
6755 if (offset < 0)
6756 continue;
6757 ret = get_msr(cpu, offset, &msr_cur);
6758 if (ret) {
6759 fprintf(outf, "Can not update msr(0x%llx)\n", (unsigned long long)offset);
6760 continue;
6761 }
6762
6763 msr_last = per_cpu_msr_sum[cpu].entries[i].last;
6764 per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff;
6765
6766 DELTA_WRAP32(msr_cur, msr_last);
6767 per_cpu_msr_sum[cpu].entries[i].sum += msr_last;
6768 }
6769 return 0;
6770 }
6771
msr_record_handler(union sigval v)6772 static void msr_record_handler(union sigval v)
6773 {
6774 UNUSED(v);
6775
6776 for_all_cpus(update_msr_sum, EVEN_COUNTERS);
6777 }
6778
msr_sum_record(void)6779 void msr_sum_record(void)
6780 {
6781 struct itimerspec its;
6782 struct sigevent sev;
6783
6784 per_cpu_msr_sum = calloc(topo.max_cpu_num + 1, sizeof(struct msr_sum_array));
6785 if (!per_cpu_msr_sum) {
6786 fprintf(outf, "Can not allocate memory for long time MSR.\n");
6787 return;
6788 }
6789 /*
6790 * Signal handler might be restricted, so use thread notifier instead.
6791 */
6792 memset(&sev, 0, sizeof(struct sigevent));
6793 sev.sigev_notify = SIGEV_THREAD;
6794 sev.sigev_notify_function = msr_record_handler;
6795
6796 sev.sigev_value.sival_ptr = &timerid;
6797 if (timer_create(CLOCK_REALTIME, &sev, &timerid) == -1) {
6798 fprintf(outf, "Can not create timer.\n");
6799 goto release_msr;
6800 }
6801
6802 its.it_value.tv_sec = 0;
6803 its.it_value.tv_nsec = 1;
6804 /*
6805 * A wraparound time has been calculated early.
6806 * Some sources state that the peak power for a
6807 * microprocessor is usually 1.5 times the TDP rating,
6808 * use 2 * TDP for safety.
6809 */
6810 its.it_interval.tv_sec = rapl_joule_counter_range / 2;
6811 its.it_interval.tv_nsec = 0;
6812
6813 if (timer_settime(timerid, 0, &its, NULL) == -1) {
6814 fprintf(outf, "Can not set timer.\n");
6815 goto release_timer;
6816 }
6817 return;
6818
6819 release_timer:
6820 timer_delete(timerid);
6821 release_msr:
6822 free(per_cpu_msr_sum);
6823 per_cpu_msr_sum = NULL;
6824 }
6825
6826 /*
6827 * set_my_sched_priority(pri)
6828 * return previous priority on success
6829 * return value < -20 on failure
6830 */
set_my_sched_priority(int priority)6831 int set_my_sched_priority(int priority)
6832 {
6833 int retval;
6834 int original_priority;
6835
6836 errno = 0;
6837 original_priority = getpriority(PRIO_PROCESS, 0);
6838 if (errno && (original_priority == -1))
6839 return -21;
6840
6841 retval = setpriority(PRIO_PROCESS, 0, priority);
6842 if (retval)
6843 return -21;
6844
6845 errno = 0;
6846 retval = getpriority(PRIO_PROCESS, 0);
6847 if (retval != priority)
6848 return -21;
6849
6850 return original_priority;
6851 }
6852
turbostat_loop()6853 void turbostat_loop()
6854 {
6855 int retval;
6856 int restarted = 0;
6857 unsigned int done_iters = 0;
6858
6859 setup_signal_handler();
6860
6861 /*
6862 * elevate own priority for interval mode
6863 *
6864 * ignore on error - we probably don't have permission to set it, but
6865 * it's not a big deal
6866 */
6867 set_my_sched_priority(-20);
6868
6869 restart:
6870 restarted++;
6871
6872 snapshot_proc_sysfs_files();
6873 retval = for_all_cpus(get_counters, EVEN_COUNTERS);
6874 first_counter_read = 0;
6875 if (retval < -1) {
6876 exit(retval);
6877 } else if (retval == -1) {
6878 if (restarted > 10) {
6879 exit(retval);
6880 }
6881 re_initialize();
6882 goto restart;
6883 }
6884 restarted = 0;
6885 done_iters = 0;
6886 gettimeofday(&tv_even, (struct timezone *)NULL);
6887
6888 while (1) {
6889 if (for_all_proc_cpus(cpu_is_not_present)) {
6890 re_initialize();
6891 goto restart;
6892 }
6893 if (update_effective_str(false)) {
6894 re_initialize();
6895 goto restart;
6896 }
6897 do_sleep();
6898 if (snapshot_proc_sysfs_files())
6899 goto restart;
6900 retval = for_all_cpus(get_counters, ODD_COUNTERS);
6901 if (retval < -1) {
6902 exit(retval);
6903 } else if (retval == -1) {
6904 re_initialize();
6905 goto restart;
6906 }
6907 gettimeofday(&tv_odd, (struct timezone *)NULL);
6908 timersub(&tv_odd, &tv_even, &tv_delta);
6909 if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) {
6910 re_initialize();
6911 goto restart;
6912 }
6913 delta_platform(&platform_counters_odd, &platform_counters_even);
6914 compute_average(EVEN_COUNTERS);
6915 format_all_counters(EVEN_COUNTERS);
6916 flush_output_stdout();
6917 if (exit_requested)
6918 break;
6919 if (num_iterations && ++done_iters >= num_iterations)
6920 break;
6921 do_sleep();
6922 if (snapshot_proc_sysfs_files())
6923 goto restart;
6924 retval = for_all_cpus(get_counters, EVEN_COUNTERS);
6925 if (retval < -1) {
6926 exit(retval);
6927 } else if (retval == -1) {
6928 re_initialize();
6929 goto restart;
6930 }
6931 gettimeofday(&tv_even, (struct timezone *)NULL);
6932 timersub(&tv_even, &tv_odd, &tv_delta);
6933 if (for_all_cpus_2(delta_cpu, EVEN_COUNTERS, ODD_COUNTERS)) {
6934 re_initialize();
6935 goto restart;
6936 }
6937 delta_platform(&platform_counters_even, &platform_counters_odd);
6938 compute_average(ODD_COUNTERS);
6939 format_all_counters(ODD_COUNTERS);
6940 flush_output_stdout();
6941 if (exit_requested)
6942 break;
6943 if (num_iterations && ++done_iters >= num_iterations)
6944 break;
6945 }
6946 }
6947
probe_dev_msr(void)6948 int probe_dev_msr(void)
6949 {
6950 struct stat sb;
6951 char pathname[32];
6952
6953 sprintf(pathname, "/dev/msr%d", master_cpu);
6954 return !stat(pathname, &sb);
6955 }
6956
probe_dev_cpu_msr(void)6957 int probe_dev_cpu_msr(void)
6958 {
6959 struct stat sb;
6960 char pathname[32];
6961
6962 sprintf(pathname, "/dev/cpu/%d/msr", master_cpu);
6963 return !stat(pathname, &sb);
6964 }
6965
probe_msr_driver(void)6966 int probe_msr_driver(void)
6967 {
6968 if (probe_dev_msr()) {
6969 use_android_msr_path = 1;
6970 return 1;
6971 }
6972 return probe_dev_cpu_msr();
6973 }
6974
check_msr_driver(void)6975 void check_msr_driver(void)
6976 {
6977 if (probe_msr_driver())
6978 return;
6979
6980 if (system("/sbin/modprobe msr > /dev/null 2>&1"))
6981 no_msr = 1;
6982
6983 if (!probe_msr_driver())
6984 no_msr = 1;
6985 }
6986
6987 /*
6988 * check for CAP_SYS_RAWIO
6989 * return 0 on success
6990 * return 1 on fail
6991 */
check_for_cap_sys_rawio(void)6992 int check_for_cap_sys_rawio(void)
6993 {
6994 cap_t caps;
6995 cap_flag_value_t cap_flag_value;
6996 int ret = 0;
6997
6998 caps = cap_get_proc();
6999 if (caps == NULL) {
7000 /*
7001 * CONFIG_MULTIUSER=n kernels have no cap_get_proc()
7002 * Allow them to continue and attempt to access MSRs
7003 */
7004 if (errno == ENOSYS)
7005 return 0;
7006
7007 return 1;
7008 }
7009
7010 if (cap_get_flag(caps, CAP_SYS_RAWIO, CAP_EFFECTIVE, &cap_flag_value)) {
7011 ret = 1;
7012 goto free_and_exit;
7013 }
7014
7015 if (cap_flag_value != CAP_SET) {
7016 ret = 1;
7017 goto free_and_exit;
7018 }
7019
7020 free_and_exit:
7021 if (cap_free(caps) == -1)
7022 err(-6, "cap_free");
7023
7024 return ret;
7025 }
7026
check_msr_permission(void)7027 void check_msr_permission(void)
7028 {
7029 int failed = 0;
7030 char pathname[32];
7031
7032 if (no_msr)
7033 return;
7034
7035 /* check for CAP_SYS_RAWIO */
7036 failed += check_for_cap_sys_rawio();
7037
7038 /* test file permissions */
7039 sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", master_cpu);
7040 if (euidaccess(pathname, R_OK)) {
7041 failed++;
7042 }
7043
7044 if (failed) {
7045 warnx("Failed to access %s. Some of the counters may not be available\n"
7046 "\tRun as root to enable them or use %s to disable the access explicitly", pathname, "--no-msr");
7047 no_msr = 1;
7048 }
7049 }
7050
probe_bclk(void)7051 void probe_bclk(void)
7052 {
7053 unsigned long long msr;
7054 unsigned int base_ratio;
7055
7056 if (!platform->has_nhm_msrs || no_msr)
7057 return;
7058
7059 if (platform->bclk_freq == BCLK_100MHZ)
7060 bclk = 100.00;
7061 else if (platform->bclk_freq == BCLK_133MHZ)
7062 bclk = 133.33;
7063 else if (platform->bclk_freq == BCLK_SLV)
7064 bclk = slm_bclk();
7065 else
7066 return;
7067
7068 get_msr(master_cpu, MSR_PLATFORM_INFO, &msr);
7069 base_ratio = (msr >> 8) & 0xFF;
7070
7071 base_hz = base_ratio * bclk * 1000000;
7072 has_base_hz = 1;
7073
7074 if (platform->enable_tsc_tweak)
7075 tsc_tweak = base_hz / tsc_hz;
7076 }
7077
remove_underbar(char * s)7078 static void remove_underbar(char *s)
7079 {
7080 char *to = s;
7081
7082 while (*s) {
7083 if (*s != '_')
7084 *to++ = *s;
7085 s++;
7086 }
7087
7088 *to = 0;
7089 }
7090
dump_turbo_ratio_info(void)7091 static void dump_turbo_ratio_info(void)
7092 {
7093 if (!has_turbo)
7094 return;
7095
7096 if (!platform->has_nhm_msrs || no_msr)
7097 return;
7098
7099 if (platform->trl_msrs & TRL_LIMIT2)
7100 dump_turbo_ratio_limit2();
7101
7102 if (platform->trl_msrs & TRL_LIMIT1)
7103 dump_turbo_ratio_limit1();
7104
7105 if (platform->trl_msrs & TRL_BASE) {
7106 dump_turbo_ratio_limits(MSR_TURBO_RATIO_LIMIT);
7107
7108 if (is_hybrid)
7109 dump_turbo_ratio_limits(MSR_SECONDARY_TURBO_RATIO_LIMIT);
7110 }
7111
7112 if (platform->trl_msrs & TRL_ATOM)
7113 dump_atom_turbo_ratio_limits();
7114
7115 if (platform->trl_msrs & TRL_KNL)
7116 dump_knl_turbo_ratio_limits();
7117
7118 if (platform->has_config_tdp)
7119 dump_config_tdp();
7120 }
7121
read_sysfs_int(char * path)7122 static int read_sysfs_int(char *path)
7123 {
7124 FILE *input;
7125 int retval = -1;
7126
7127 input = fopen(path, "r");
7128 if (input == NULL) {
7129 if (debug)
7130 fprintf(outf, "NSFOD %s\n", path);
7131 return (-1);
7132 }
7133 if (fscanf(input, "%d", &retval) != 1)
7134 err(1, "%s: failed to read int from file", path);
7135 fclose(input);
7136
7137 return (retval);
7138 }
7139
dump_sysfs_file(char * path)7140 static void dump_sysfs_file(char *path)
7141 {
7142 FILE *input;
7143 char cpuidle_buf[64];
7144
7145 input = fopen(path, "r");
7146 if (input == NULL) {
7147 if (debug)
7148 fprintf(outf, "NSFOD %s\n", path);
7149 return;
7150 }
7151 if (!fgets(cpuidle_buf, sizeof(cpuidle_buf), input))
7152 err(1, "%s: failed to read file", path);
7153 fclose(input);
7154
7155 fprintf(outf, "%s: %s", strrchr(path, '/') + 1, cpuidle_buf);
7156 }
7157
probe_intel_uncore_frequency_legacy(void)7158 static void probe_intel_uncore_frequency_legacy(void)
7159 {
7160 int i, j;
7161 char path[256];
7162
7163 for (i = 0; i < topo.num_packages; ++i) {
7164 for (j = 0; j <= topo.max_die_id; ++j) {
7165 int k, l;
7166 char path_base[128];
7167
7168 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, j);
7169
7170 sprintf(path, "%s/current_freq_khz", path_base);
7171 if (access(path, R_OK))
7172 continue;
7173
7174 BIC_PRESENT(BIC_UNCORE_MHZ);
7175
7176 if (quiet)
7177 return;
7178
7179 sprintf(path, "%s/min_freq_khz", path_base);
7180 k = read_sysfs_int(path);
7181 sprintf(path, "%s/max_freq_khz", path_base);
7182 l = read_sysfs_int(path);
7183 fprintf(outf, "Uncore Frequency package%d die%d: %d - %d MHz ", i, j, k / 1000, l / 1000);
7184
7185 sprintf(path, "%s/initial_min_freq_khz", path_base);
7186 k = read_sysfs_int(path);
7187 sprintf(path, "%s/initial_max_freq_khz", path_base);
7188 l = read_sysfs_int(path);
7189 fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
7190
7191 sprintf(path, "%s/current_freq_khz", path_base);
7192 k = read_sysfs_int(path);
7193 fprintf(outf, " %d MHz\n", k / 1000);
7194 }
7195 }
7196 }
7197
probe_intel_uncore_frequency_cluster(void)7198 static void probe_intel_uncore_frequency_cluster(void)
7199 {
7200 int i, uncore_max_id;
7201 char path[256];
7202 char path_base[128];
7203
7204 if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00/current_freq_khz", R_OK))
7205 return;
7206
7207 for (uncore_max_id = 0;; ++uncore_max_id) {
7208
7209 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", uncore_max_id);
7210
7211 /* uncore## start at 00 and skips no numbers, so stop upon first missing */
7212 if (access(path_base, R_OK)) {
7213 uncore_max_id -= 1;
7214 break;
7215 }
7216 }
7217 for (i = uncore_max_id; i >= 0; --i) {
7218 int k, l;
7219 int unc_pkg_id, domain_id, cluster_id;
7220 char name_buf[16];
7221
7222 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i);
7223
7224 if (access(path_base, R_OK))
7225 err(1, "%s: %s", __func__, path_base);
7226
7227 sprintf(path, "%s/package_id", path_base);
7228 unc_pkg_id = read_sysfs_int(path);
7229
7230 sprintf(path, "%s/domain_id", path_base);
7231 domain_id = read_sysfs_int(path);
7232
7233 sprintf(path, "%s/fabric_cluster_id", path_base);
7234 cluster_id = read_sysfs_int(path);
7235
7236 sprintf(path, "%s/current_freq_khz", path_base);
7237 sprintf(name_buf, "UMHz%d.%d", domain_id, cluster_id);
7238
7239 /*
7240 * Once add_couter() is called, that counter is always read
7241 * and reported -- So it is effectively (enabled & present).
7242 * Only call add_counter() here if legacy BIC_UNCORE_MHZ (UncMHz)
7243 * is (enabled). Since we are in this routine, we
7244 * know we will not probe and set (present) the legacy counter.
7245 *
7246 * This allows "--show/--hide UncMHz" to be effective for
7247 * the clustered MHz counters, as a group.
7248 */
7249 if BIC_IS_ENABLED
7250 (BIC_UNCORE_MHZ)
7251 add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, unc_pkg_id);
7252
7253 if (quiet)
7254 continue;
7255
7256 sprintf(path, "%s/min_freq_khz", path_base);
7257 k = read_sysfs_int(path);
7258 sprintf(path, "%s/max_freq_khz", path_base);
7259 l = read_sysfs_int(path);
7260 fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", unc_pkg_id, domain_id, cluster_id, k / 1000, l / 1000);
7261
7262 sprintf(path, "%s/initial_min_freq_khz", path_base);
7263 k = read_sysfs_int(path);
7264 sprintf(path, "%s/initial_max_freq_khz", path_base);
7265 l = read_sysfs_int(path);
7266 fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
7267
7268 sprintf(path, "%s/current_freq_khz", path_base);
7269 k = read_sysfs_int(path);
7270 fprintf(outf, " %d MHz\n", k / 1000);
7271 }
7272 }
7273
probe_intel_uncore_frequency(void)7274 static void probe_intel_uncore_frequency(void)
7275 {
7276 if (!genuine_intel)
7277 return;
7278
7279 if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00", R_OK) == 0)
7280 probe_intel_uncore_frequency_cluster();
7281 else
7282 probe_intel_uncore_frequency_legacy();
7283 }
7284
set_graphics_fp(char * path,int idx)7285 static void set_graphics_fp(char *path, int idx)
7286 {
7287 if (!access(path, R_OK))
7288 gfx_info[idx].fp = fopen_or_die(path, "r");
7289 }
7290
7291 /* Enlarge this if there are /sys/class/drm/card2 ... */
7292 #define GFX_MAX_CARDS 2
7293
probe_graphics(void)7294 static void probe_graphics(void)
7295 {
7296 char path[PATH_MAX];
7297 int i;
7298
7299 /* Xe graphics sysfs knobs */
7300 if (!access("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", R_OK)) {
7301 FILE *fp;
7302 char buf[8];
7303 bool gt0_is_gt;
7304
7305 fp = fopen("/sys/class/drm/card0/device/tile0/gt0/gtidle/name", "r");
7306 if (!fp)
7307 goto next;
7308
7309 if (!fread(buf, sizeof(char), 7, fp)) {
7310 fclose(fp);
7311 goto next;
7312 }
7313 fclose(fp);
7314
7315 if (!strncmp(buf, "gt0-rc", strlen("gt0-rc")))
7316 gt0_is_gt = true;
7317 else if (!strncmp(buf, "gt0-mc", strlen("gt0-mc")))
7318 gt0_is_gt = false;
7319 else
7320 goto next;
7321
7322 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", gt0_is_gt ? GFX_rc6 : SAM_mc6);
7323
7324 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/cur_freq", gt0_is_gt ? GFX_MHz : SAM_MHz);
7325
7326 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/act_freq", gt0_is_gt ? GFX_ACTMHz : SAM_ACTMHz);
7327
7328 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/gtidle/idle_residency_ms", gt0_is_gt ? SAM_mc6 : GFX_rc6);
7329
7330 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/cur_freq", gt0_is_gt ? SAM_MHz : GFX_MHz);
7331
7332 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/act_freq", gt0_is_gt ? SAM_ACTMHz : GFX_ACTMHz);
7333
7334 goto end;
7335 }
7336
7337 next:
7338 /* New i915 graphics sysfs knobs */
7339 for (i = 0; i < GFX_MAX_CARDS; i++) {
7340 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i);
7341 if (!access(path, R_OK))
7342 break;
7343 }
7344
7345 if (i == GFX_MAX_CARDS)
7346 goto legacy_i915;
7347
7348 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i);
7349 set_graphics_fp(path, GFX_rc6);
7350
7351 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rps_cur_freq_mhz", i);
7352 set_graphics_fp(path, GFX_MHz);
7353
7354 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rps_act_freq_mhz", i);
7355 set_graphics_fp(path, GFX_ACTMHz);
7356
7357 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rc6_residency_ms", i);
7358 set_graphics_fp(path, SAM_mc6);
7359
7360 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rps_cur_freq_mhz", i);
7361 set_graphics_fp(path, SAM_MHz);
7362
7363 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rps_act_freq_mhz", i);
7364 set_graphics_fp(path, SAM_ACTMHz);
7365
7366 goto end;
7367
7368 legacy_i915:
7369 /* Fall back to traditional i915 graphics sysfs knobs */
7370 set_graphics_fp("/sys/class/drm/card0/power/rc6_residency_ms", GFX_rc6);
7371
7372 set_graphics_fp("/sys/class/drm/card0/gt_cur_freq_mhz", GFX_MHz);
7373 if (!gfx_info[GFX_MHz].fp)
7374 set_graphics_fp("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", GFX_MHz);
7375
7376 set_graphics_fp("/sys/class/drm/card0/gt_act_freq_mhz", GFX_ACTMHz);
7377 if (!gfx_info[GFX_ACTMHz].fp)
7378 set_graphics_fp("/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz", GFX_ACTMHz);
7379
7380 end:
7381 if (gfx_info[GFX_rc6].fp)
7382 BIC_PRESENT(BIC_GFX_rc6);
7383 if (gfx_info[GFX_MHz].fp)
7384 BIC_PRESENT(BIC_GFXMHz);
7385 if (gfx_info[GFX_ACTMHz].fp)
7386 BIC_PRESENT(BIC_GFXACTMHz);
7387 if (gfx_info[SAM_mc6].fp)
7388 BIC_PRESENT(BIC_SAM_mc6);
7389 if (gfx_info[SAM_MHz].fp)
7390 BIC_PRESENT(BIC_SAMMHz);
7391 if (gfx_info[SAM_ACTMHz].fp)
7392 BIC_PRESENT(BIC_SAMACTMHz);
7393 }
7394
dump_sysfs_cstate_config(void)7395 static void dump_sysfs_cstate_config(void)
7396 {
7397 char path[64];
7398 char name_buf[16];
7399 char desc[64];
7400 FILE *input;
7401 int state;
7402 char *sp;
7403
7404 if (access("/sys/devices/system/cpu/cpuidle", R_OK)) {
7405 fprintf(outf, "cpuidle not loaded\n");
7406 return;
7407 }
7408
7409 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_driver");
7410 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor");
7411 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor_ro");
7412
7413 for (state = 0; state < 10; ++state) {
7414
7415 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state);
7416 input = fopen(path, "r");
7417 if (input == NULL)
7418 continue;
7419 if (!fgets(name_buf, sizeof(name_buf), input))
7420 err(1, "%s: failed to read file", path);
7421
7422 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
7423 sp = strchr(name_buf, '-');
7424 if (!sp)
7425 sp = strchrnul(name_buf, '\n');
7426 *sp = '\0';
7427 fclose(input);
7428
7429 remove_underbar(name_buf);
7430
7431 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", master_cpu, state);
7432 input = fopen(path, "r");
7433 if (input == NULL)
7434 continue;
7435 if (!fgets(desc, sizeof(desc), input))
7436 err(1, "%s: failed to read file", path);
7437
7438 fprintf(outf, "cpu%d: %s: %s", master_cpu, name_buf, desc);
7439 fclose(input);
7440 }
7441 }
7442
dump_sysfs_pstate_config(void)7443 static void dump_sysfs_pstate_config(void)
7444 {
7445 char path[64];
7446 char driver_buf[64];
7447 char governor_buf[64];
7448 FILE *input;
7449 int turbo;
7450
7451 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", master_cpu);
7452 input = fopen(path, "r");
7453 if (input == NULL) {
7454 fprintf(outf, "NSFOD %s\n", path);
7455 return;
7456 }
7457 if (!fgets(driver_buf, sizeof(driver_buf), input))
7458 err(1, "%s: failed to read file", path);
7459 fclose(input);
7460
7461 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", master_cpu);
7462 input = fopen(path, "r");
7463 if (input == NULL) {
7464 fprintf(outf, "NSFOD %s\n", path);
7465 return;
7466 }
7467 if (!fgets(governor_buf, sizeof(governor_buf), input))
7468 err(1, "%s: failed to read file", path);
7469 fclose(input);
7470
7471 fprintf(outf, "cpu%d: cpufreq driver: %s", master_cpu, driver_buf);
7472 fprintf(outf, "cpu%d: cpufreq governor: %s", master_cpu, governor_buf);
7473
7474 sprintf(path, "/sys/devices/system/cpu/cpufreq/boost");
7475 input = fopen(path, "r");
7476 if (input != NULL) {
7477 if (fscanf(input, "%d", &turbo) != 1)
7478 err(1, "%s: failed to parse number from file", path);
7479 fprintf(outf, "cpufreq boost: %d\n", turbo);
7480 fclose(input);
7481 }
7482
7483 sprintf(path, "/sys/devices/system/cpu/intel_pstate/no_turbo");
7484 input = fopen(path, "r");
7485 if (input != NULL) {
7486 if (fscanf(input, "%d", &turbo) != 1)
7487 err(1, "%s: failed to parse number from file", path);
7488 fprintf(outf, "cpufreq intel_pstate no_turbo: %d\n", turbo);
7489 fclose(input);
7490 }
7491 }
7492
7493 /*
7494 * print_epb()
7495 * Decode the ENERGY_PERF_BIAS MSR
7496 */
print_epb(PER_THREAD_PARAMS)7497 int print_epb(PER_THREAD_PARAMS)
7498 {
7499 char *epb_string;
7500 int cpu, epb;
7501
7502 UNUSED(c);
7503 UNUSED(p);
7504
7505 if (!has_epb)
7506 return 0;
7507
7508 cpu = t->cpu_id;
7509
7510 /* EPB is per-package */
7511 if (!is_cpu_first_thread_in_package(t, c, p))
7512 return 0;
7513
7514 if (cpu_migrate(cpu)) {
7515 fprintf(outf, "print_epb: Could not migrate to CPU %d\n", cpu);
7516 return -1;
7517 }
7518
7519 epb = get_epb(cpu);
7520 if (epb < 0)
7521 return 0;
7522
7523 switch (epb) {
7524 case ENERGY_PERF_BIAS_PERFORMANCE:
7525 epb_string = "performance";
7526 break;
7527 case ENERGY_PERF_BIAS_NORMAL:
7528 epb_string = "balanced";
7529 break;
7530 case ENERGY_PERF_BIAS_POWERSAVE:
7531 epb_string = "powersave";
7532 break;
7533 default:
7534 epb_string = "custom";
7535 break;
7536 }
7537 fprintf(outf, "cpu%d: EPB: %d (%s)\n", cpu, epb, epb_string);
7538
7539 return 0;
7540 }
7541
7542 /*
7543 * print_hwp()
7544 * Decode the MSR_HWP_CAPABILITIES
7545 */
print_hwp(PER_THREAD_PARAMS)7546 int print_hwp(PER_THREAD_PARAMS)
7547 {
7548 unsigned long long msr;
7549 int cpu;
7550
7551 UNUSED(c);
7552 UNUSED(p);
7553
7554 if (no_msr)
7555 return 0;
7556
7557 if (!has_hwp)
7558 return 0;
7559
7560 cpu = t->cpu_id;
7561
7562 /* MSR_HWP_CAPABILITIES is per-package */
7563 if (!is_cpu_first_thread_in_package(t, c, p))
7564 return 0;
7565
7566 if (cpu_migrate(cpu)) {
7567 fprintf(outf, "print_hwp: Could not migrate to CPU %d\n", cpu);
7568 return -1;
7569 }
7570
7571 if (get_msr(cpu, MSR_PM_ENABLE, &msr))
7572 return 0;
7573
7574 fprintf(outf, "cpu%d: MSR_PM_ENABLE: 0x%08llx (%sHWP)\n", cpu, msr, (msr & (1 << 0)) ? "" : "No-");
7575
7576 /* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */
7577 if ((msr & (1 << 0)) == 0)
7578 return 0;
7579
7580 if (get_msr(cpu, MSR_HWP_CAPABILITIES, &msr))
7581 return 0;
7582
7583 fprintf(outf, "cpu%d: MSR_HWP_CAPABILITIES: 0x%08llx "
7584 "(high %d guar %d eff %d low %d)\n",
7585 cpu, msr,
7586 (unsigned int)HWP_HIGHEST_PERF(msr),
7587 (unsigned int)HWP_GUARANTEED_PERF(msr), (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr));
7588
7589 if (get_msr(cpu, MSR_HWP_REQUEST, &msr))
7590 return 0;
7591
7592 fprintf(outf, "cpu%d: MSR_HWP_REQUEST: 0x%08llx "
7593 "(min %d max %d des %d epp 0x%x window 0x%x pkg 0x%x)\n",
7594 cpu, msr,
7595 (unsigned int)(((msr) >> 0) & 0xff),
7596 (unsigned int)(((msr) >> 8) & 0xff),
7597 (unsigned int)(((msr) >> 16) & 0xff),
7598 (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1));
7599
7600 if (has_hwp_pkg) {
7601 if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr))
7602 return 0;
7603
7604 fprintf(outf, "cpu%d: MSR_HWP_REQUEST_PKG: 0x%08llx "
7605 "(min %d max %d des %d epp 0x%x window 0x%x)\n",
7606 cpu, msr,
7607 (unsigned int)(((msr) >> 0) & 0xff),
7608 (unsigned int)(((msr) >> 8) & 0xff),
7609 (unsigned int)(((msr) >> 16) & 0xff), (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3));
7610 }
7611 if (has_hwp_notify) {
7612 if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr))
7613 return 0;
7614
7615 fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx "
7616 "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis");
7617 }
7618 if (get_msr(cpu, MSR_HWP_STATUS, &msr))
7619 return 0;
7620
7621 fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx "
7622 "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-");
7623
7624 return 0;
7625 }
7626
7627 /*
7628 * print_perf_limit()
7629 */
print_perf_limit(PER_THREAD_PARAMS)7630 int print_perf_limit(PER_THREAD_PARAMS)
7631 {
7632 unsigned long long msr;
7633 int cpu;
7634
7635 UNUSED(c);
7636 UNUSED(p);
7637
7638 if (no_msr)
7639 return 0;
7640
7641 cpu = t->cpu_id;
7642
7643 /* per-package */
7644 if (!is_cpu_first_thread_in_package(t, c, p))
7645 return 0;
7646
7647 if (cpu_migrate(cpu)) {
7648 fprintf(outf, "print_perf_limit: Could not migrate to CPU %d\n", cpu);
7649 return -1;
7650 }
7651
7652 if (platform->plr_msrs & PLR_CORE) {
7653 get_msr(cpu, MSR_CORE_PERF_LIMIT_REASONS, &msr);
7654 fprintf(outf, "cpu%d: MSR_CORE_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7655 fprintf(outf, " (Active: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)",
7656 (msr & 1 << 15) ? "bit15, " : "",
7657 (msr & 1 << 14) ? "bit14, " : "",
7658 (msr & 1 << 13) ? "Transitions, " : "",
7659 (msr & 1 << 12) ? "MultiCoreTurbo, " : "",
7660 (msr & 1 << 11) ? "PkgPwrL2, " : "",
7661 (msr & 1 << 10) ? "PkgPwrL1, " : "",
7662 (msr & 1 << 9) ? "CorePwr, " : "",
7663 (msr & 1 << 8) ? "Amps, " : "",
7664 (msr & 1 << 6) ? "VR-Therm, " : "",
7665 (msr & 1 << 5) ? "Auto-HWP, " : "",
7666 (msr & 1 << 4) ? "Graphics, " : "",
7667 (msr & 1 << 2) ? "bit2, " : "", (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : "");
7668 fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n",
7669 (msr & 1 << 31) ? "bit31, " : "",
7670 (msr & 1 << 30) ? "bit30, " : "",
7671 (msr & 1 << 29) ? "Transitions, " : "",
7672 (msr & 1 << 28) ? "MultiCoreTurbo, " : "",
7673 (msr & 1 << 27) ? "PkgPwrL2, " : "",
7674 (msr & 1 << 26) ? "PkgPwrL1, " : "",
7675 (msr & 1 << 25) ? "CorePwr, " : "",
7676 (msr & 1 << 24) ? "Amps, " : "",
7677 (msr & 1 << 22) ? "VR-Therm, " : "",
7678 (msr & 1 << 21) ? "Auto-HWP, " : "",
7679 (msr & 1 << 20) ? "Graphics, " : "",
7680 (msr & 1 << 18) ? "bit18, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : "");
7681
7682 }
7683 if (platform->plr_msrs & PLR_GFX) {
7684 get_msr(cpu, MSR_GFX_PERF_LIMIT_REASONS, &msr);
7685 fprintf(outf, "cpu%d: MSR_GFX_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7686 fprintf(outf, " (Active: %s%s%s%s%s%s%s%s)",
7687 (msr & 1 << 0) ? "PROCHOT, " : "",
7688 (msr & 1 << 1) ? "ThermStatus, " : "",
7689 (msr & 1 << 4) ? "Graphics, " : "",
7690 (msr & 1 << 6) ? "VR-Therm, " : "",
7691 (msr & 1 << 8) ? "Amps, " : "",
7692 (msr & 1 << 9) ? "GFXPwr, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
7693 fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s)\n",
7694 (msr & 1 << 16) ? "PROCHOT, " : "",
7695 (msr & 1 << 17) ? "ThermStatus, " : "",
7696 (msr & 1 << 20) ? "Graphics, " : "",
7697 (msr & 1 << 22) ? "VR-Therm, " : "",
7698 (msr & 1 << 24) ? "Amps, " : "",
7699 (msr & 1 << 25) ? "GFXPwr, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
7700 }
7701 if (platform->plr_msrs & PLR_RING) {
7702 get_msr(cpu, MSR_RING_PERF_LIMIT_REASONS, &msr);
7703 fprintf(outf, "cpu%d: MSR_RING_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7704 fprintf(outf, " (Active: %s%s%s%s%s%s)",
7705 (msr & 1 << 0) ? "PROCHOT, " : "",
7706 (msr & 1 << 1) ? "ThermStatus, " : "",
7707 (msr & 1 << 6) ? "VR-Therm, " : "",
7708 (msr & 1 << 8) ? "Amps, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
7709 fprintf(outf, " (Logged: %s%s%s%s%s%s)\n",
7710 (msr & 1 << 16) ? "PROCHOT, " : "",
7711 (msr & 1 << 17) ? "ThermStatus, " : "",
7712 (msr & 1 << 22) ? "VR-Therm, " : "",
7713 (msr & 1 << 24) ? "Amps, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
7714 }
7715 return 0;
7716 }
7717
7718 #define RAPL_POWER_GRANULARITY 0x7FFF /* 15 bit power granularity */
7719 #define RAPL_TIME_GRANULARITY 0x3F /* 6 bit time granularity */
7720
get_quirk_tdp(void)7721 double get_quirk_tdp(void)
7722 {
7723 if (platform->rapl_quirk_tdp)
7724 return platform->rapl_quirk_tdp;
7725
7726 return 135.0;
7727 }
7728
get_tdp_intel(void)7729 double get_tdp_intel(void)
7730 {
7731 unsigned long long msr;
7732
7733 if (valid_rapl_msrs & RAPL_PKG_POWER_INFO)
7734 if (!get_msr(master_cpu, MSR_PKG_POWER_INFO, &msr))
7735 return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units;
7736 return get_quirk_tdp();
7737 }
7738
get_tdp_amd(void)7739 double get_tdp_amd(void)
7740 {
7741 return get_quirk_tdp();
7742 }
7743
rapl_probe_intel(void)7744 void rapl_probe_intel(void)
7745 {
7746 unsigned long long msr;
7747 unsigned int time_unit;
7748 double tdp;
7749
7750 if (rapl_joules) {
7751 CLR_BIC(BIC_SysWatt, &bic_enabled);
7752 CLR_BIC(BIC_PkgWatt, &bic_enabled);
7753 CLR_BIC(BIC_CorWatt, &bic_enabled);
7754 CLR_BIC(BIC_RAMWatt, &bic_enabled);
7755 CLR_BIC(BIC_GFXWatt, &bic_enabled);
7756 } else {
7757 CLR_BIC(BIC_Sys_J, &bic_enabled);
7758 CLR_BIC(BIC_Pkg_J, &bic_enabled);
7759 CLR_BIC(BIC_Cor_J, &bic_enabled);
7760 CLR_BIC(BIC_RAM_J, &bic_enabled);
7761 CLR_BIC(BIC_GFX_J, &bic_enabled);
7762 }
7763
7764 if (!valid_rapl_msrs || no_msr)
7765 return;
7766
7767 if (!(valid_rapl_msrs & RAPL_PKG_PERF_STATUS))
7768 CLR_BIC(BIC_PKG__, &bic_enabled);
7769 if (!(valid_rapl_msrs & RAPL_DRAM_PERF_STATUS))
7770 CLR_BIC(BIC_RAM__, &bic_enabled);
7771
7772 /* units on package 0, verify later other packages match */
7773 if (get_msr(master_cpu, MSR_RAPL_POWER_UNIT, &msr))
7774 return;
7775
7776 rapl_power_units = 1.0 / (1 << (msr & 0xF));
7777 if (platform->has_rapl_divisor)
7778 rapl_energy_units = 1.0 * (1 << (msr >> 8 & 0x1F)) / 1000000;
7779 else
7780 rapl_energy_units = 1.0 / (1 << (msr >> 8 & 0x1F));
7781
7782 if (platform->has_fixed_rapl_unit)
7783 rapl_dram_energy_units = (15.3 / 1000000);
7784 else
7785 rapl_dram_energy_units = rapl_energy_units;
7786
7787 if (platform->has_fixed_rapl_psys_unit)
7788 rapl_psys_energy_units = 1.0;
7789 else
7790 rapl_psys_energy_units = rapl_energy_units;
7791
7792 time_unit = msr >> 16 & 0xF;
7793 if (time_unit == 0)
7794 time_unit = 0xA;
7795
7796 rapl_time_units = 1.0 / (1 << (time_unit));
7797
7798 tdp = get_tdp_intel();
7799
7800 rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7801 if (!quiet)
7802 fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7803 }
7804
rapl_probe_amd(void)7805 void rapl_probe_amd(void)
7806 {
7807 unsigned long long msr;
7808 double tdp;
7809
7810 if (rapl_joules) {
7811 CLR_BIC(BIC_SysWatt, &bic_enabled);
7812 CLR_BIC(BIC_CorWatt, &bic_enabled);
7813 } else {
7814 CLR_BIC(BIC_Pkg_J, &bic_enabled);
7815 CLR_BIC(BIC_Cor_J, &bic_enabled);
7816 }
7817
7818 if (!valid_rapl_msrs || no_msr)
7819 return;
7820
7821 if (get_msr(master_cpu, MSR_RAPL_PWR_UNIT, &msr))
7822 return;
7823
7824 rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf));
7825 rapl_energy_units = ldexp(1.0, -(msr >> 8 & 0x1f));
7826 rapl_power_units = ldexp(1.0, -(msr & 0xf));
7827
7828 tdp = get_tdp_amd();
7829
7830 rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7831 if (!quiet)
7832 fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7833 }
7834
print_power_limit_msr(int cpu,unsigned long long msr,char * label)7835 void print_power_limit_msr(int cpu, unsigned long long msr, char *label)
7836 {
7837 fprintf(outf, "cpu%d: %s: %sabled (%0.3f Watts, %f sec, clamp %sabled)\n",
7838 cpu, label,
7839 ((msr >> 15) & 1) ? "EN" : "DIS",
7840 ((msr >> 0) & 0x7FFF) * rapl_power_units,
7841 (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, (((msr >> 16) & 1) ? "EN" : "DIS"));
7842
7843 return;
7844 }
7845
fread_int(char * path,int * val)7846 static int fread_int(char *path, int *val)
7847 {
7848 FILE *filep;
7849 int ret;
7850
7851 filep = fopen(path, "r");
7852 if (!filep)
7853 return -1;
7854
7855 ret = fscanf(filep, "%d", val);
7856 fclose(filep);
7857 return ret;
7858 }
7859
fread_ull(char * path,unsigned long long * val)7860 static int fread_ull(char *path, unsigned long long *val)
7861 {
7862 FILE *filep;
7863 int ret;
7864
7865 filep = fopen(path, "r");
7866 if (!filep)
7867 return -1;
7868
7869 ret = fscanf(filep, "%llu", val);
7870 fclose(filep);
7871 return ret;
7872 }
7873
fread_str(char * path,char * buf,int size)7874 static int fread_str(char *path, char *buf, int size)
7875 {
7876 FILE *filep;
7877 int ret;
7878 char *cp;
7879
7880 filep = fopen(path, "r");
7881 if (!filep)
7882 return -1;
7883
7884 ret = fread(buf, 1, size, filep);
7885 fclose(filep);
7886
7887 /* replace '\n' with '\0' */
7888 cp = strchr(buf, '\n');
7889 if (cp != NULL)
7890 *cp = '\0';
7891
7892 return ret;
7893 }
7894
7895 #define PATH_RAPL_SYSFS "/sys/class/powercap"
7896
dump_one_domain(char * domain_path)7897 static int dump_one_domain(char *domain_path)
7898 {
7899 char path[PATH_MAX];
7900 char str[PATH_MAX];
7901 unsigned long long val;
7902 int constraint;
7903 int enable;
7904 int ret;
7905
7906 snprintf(path, PATH_MAX, "%s/name", domain_path);
7907 ret = fread_str(path, str, PATH_MAX);
7908 if (ret <= 0)
7909 return -1;
7910
7911 fprintf(outf, "%s: %s", domain_path + strlen(PATH_RAPL_SYSFS) + 1, str);
7912
7913 snprintf(path, PATH_MAX, "%s/enabled", domain_path);
7914 ret = fread_int(path, &enable);
7915 if (ret <= 0)
7916 return -1;
7917
7918 if (!enable) {
7919 fputs(" disabled\n", outf);
7920 return 0;
7921 }
7922
7923 for (constraint = 0;; constraint++) {
7924 snprintf(path, PATH_MAX, "%s/constraint_%d_time_window_us", domain_path, constraint);
7925 ret = fread_ull(path, &val);
7926 if (ret <= 0)
7927 break;
7928
7929 if (val > 1000000)
7930 fprintf(outf, " %0.1fs", (double)val / 1000000);
7931 else if (val > 1000)
7932 fprintf(outf, " %0.1fms", (double)val / 1000);
7933 else
7934 fprintf(outf, " %0.1fus", (double)val);
7935
7936 snprintf(path, PATH_MAX, "%s/constraint_%d_power_limit_uw", domain_path, constraint);
7937 ret = fread_ull(path, &val);
7938 if (ret > 0 && val)
7939 fprintf(outf, ":%lluW", val / 1000000);
7940
7941 snprintf(path, PATH_MAX, "%s/constraint_%d_max_power_uw", domain_path, constraint);
7942 ret = fread_ull(path, &val);
7943 if (ret > 0 && val)
7944 fprintf(outf, ",max:%lluW", val / 1000000);
7945 }
7946 fputc('\n', outf);
7947
7948 return 0;
7949 }
7950
print_rapl_sysfs(void)7951 static int print_rapl_sysfs(void)
7952 {
7953 DIR *dir, *cdir;
7954 struct dirent *entry, *centry;
7955 char path[PATH_MAX];
7956 char str[PATH_MAX];
7957
7958 if ((dir = opendir(PATH_RAPL_SYSFS)) == NULL) {
7959 warn("open %s failed", PATH_RAPL_SYSFS);
7960 return 1;
7961 }
7962
7963 while ((entry = readdir(dir)) != NULL) {
7964 if (strlen(entry->d_name) > 100)
7965 continue;
7966
7967 if (strncmp(entry->d_name, "intel-rapl", strlen("intel-rapl")))
7968 continue;
7969
7970 snprintf(path, PATH_MAX, "%s/%s/name", PATH_RAPL_SYSFS, entry->d_name);
7971
7972 /* Parse top level domains first, including package and psys */
7973 fread_str(path, str, PATH_MAX);
7974 if (strncmp(str, "package", strlen("package")) && strncmp(str, "psys", strlen("psys")))
7975 continue;
7976
7977 snprintf(path, PATH_MAX, "%s/%s", PATH_RAPL_SYSFS, entry->d_name);
7978 if ((cdir = opendir(path)) == NULL) {
7979 perror("opendir() error");
7980 return 1;
7981 }
7982
7983 dump_one_domain(path);
7984
7985 while ((centry = readdir(cdir)) != NULL) {
7986 if (strncmp(centry->d_name, "intel-rapl", strlen("intel-rapl")))
7987 continue;
7988 snprintf(path, PATH_MAX, "%s/%s/%s", PATH_RAPL_SYSFS, entry->d_name, centry->d_name);
7989 dump_one_domain(path);
7990 }
7991 closedir(cdir);
7992 }
7993
7994 closedir(dir);
7995 return 0;
7996 }
7997
print_rapl(PER_THREAD_PARAMS)7998 int print_rapl(PER_THREAD_PARAMS)
7999 {
8000 unsigned long long msr;
8001 const char *msr_name;
8002 int cpu;
8003
8004 UNUSED(c);
8005 UNUSED(p);
8006
8007 if (!valid_rapl_msrs)
8008 return 0;
8009
8010 /* RAPL counters are per package, so print only for 1st thread/package */
8011 if (!is_cpu_first_thread_in_package(t, c, p))
8012 return 0;
8013
8014 cpu = t->cpu_id;
8015 if (cpu_migrate(cpu)) {
8016 fprintf(outf, "print_rapl: Could not migrate to CPU %d\n", cpu);
8017 return -1;
8018 }
8019
8020 if (valid_rapl_msrs & RAPL_AMD_F17H) {
8021 msr_name = "MSR_RAPL_PWR_UNIT";
8022 if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr))
8023 return -1;
8024 } else {
8025 msr_name = "MSR_RAPL_POWER_UNIT";
8026 if (get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr))
8027 return -1;
8028 }
8029
8030 fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, rapl_power_units, rapl_energy_units, rapl_time_units);
8031
8032 if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) {
8033
8034 if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr))
8035 return -5;
8036
8037 fprintf(outf, "cpu%d: MSR_PKG_POWER_INFO: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
8038 cpu, msr,
8039 ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
8040 ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
8041 ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
8042
8043 }
8044 if (valid_rapl_msrs & RAPL_PKG) {
8045
8046 if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr))
8047 return -9;
8048
8049 fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 63) & 1 ? "" : "UN");
8050
8051 print_power_limit_msr(cpu, msr, "PKG Limit #1");
8052 fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%0.3f Watts, %f* sec, clamp %sabled)\n",
8053 cpu,
8054 ((msr >> 47) & 1) ? "EN" : "DIS",
8055 ((msr >> 32) & 0x7FFF) * rapl_power_units,
8056 (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, ((msr >> 48) & 1) ? "EN" : "DIS");
8057
8058 if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr))
8059 return -9;
8060
8061 fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr);
8062 fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN");
8063 }
8064
8065 if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) {
8066 if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr))
8067 return -6;
8068
8069 fprintf(outf, "cpu%d: MSR_DRAM_POWER_INFO,: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
8070 cpu, msr,
8071 ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
8072 ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
8073 ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
8074 }
8075 if (valid_rapl_msrs & RAPL_DRAM) {
8076 if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr))
8077 return -9;
8078 fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
8079
8080 print_power_limit_msr(cpu, msr, "DRAM Limit");
8081 }
8082 if (valid_rapl_msrs & RAPL_CORE_POLICY) {
8083 if (get_msr(cpu, MSR_PP0_POLICY, &msr))
8084 return -7;
8085
8086 fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF);
8087 }
8088 if (valid_rapl_msrs & RAPL_CORE_POWER_LIMIT) {
8089 if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr))
8090 return -9;
8091 fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
8092 print_power_limit_msr(cpu, msr, "Cores Limit");
8093 }
8094 if (valid_rapl_msrs & RAPL_GFX) {
8095 if (get_msr(cpu, MSR_PP1_POLICY, &msr))
8096 return -8;
8097
8098 fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF);
8099
8100 if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr))
8101 return -9;
8102 fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
8103 print_power_limit_msr(cpu, msr, "GFX Limit");
8104 }
8105 return 0;
8106 }
8107
8108 /*
8109 * probe_rapl_msrs
8110 *
8111 * initialize global valid_rapl_msrs to platform->plat_rapl_msrs
8112 * only if PKG_ENERGY counter is enumerated and reads non-zero
8113 */
probe_rapl_msrs(void)8114 void probe_rapl_msrs(void)
8115 {
8116 int ret;
8117 off_t offset;
8118 unsigned long long msr_value;
8119
8120 if (no_msr)
8121 return;
8122
8123 if ((platform->plat_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H)) == 0)
8124 return;
8125
8126 offset = idx_to_offset(IDX_PKG_ENERGY);
8127 if (offset < 0)
8128 return;
8129
8130 ret = get_msr(master_cpu, offset, &msr_value);
8131 if (ret) {
8132 if (debug)
8133 fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset);
8134 return;
8135 }
8136 if (msr_value == 0) {
8137 if (debug)
8138 fprintf(outf, "RAPL_PKG_ENERGY MSR(0x%llx) == ZERO: disabling all RAPL MSRs\n", (unsigned long long)offset);
8139 return;
8140 }
8141
8142 valid_rapl_msrs = platform->plat_rapl_msrs; /* success */
8143 }
8144
8145 /*
8146 * probe_rapl()
8147 *
8148 * sets rapl_power_units, rapl_energy_units, rapl_time_units
8149 */
probe_rapl(void)8150 void probe_rapl(void)
8151 {
8152 probe_rapl_msrs();
8153
8154 if (genuine_intel)
8155 rapl_probe_intel();
8156 if (authentic_amd || hygon_genuine)
8157 rapl_probe_amd();
8158
8159 if (quiet)
8160 return;
8161
8162 print_rapl_sysfs();
8163
8164 if (!valid_rapl_msrs || no_msr)
8165 return;
8166
8167 for_all_cpus(print_rapl, ODD_COUNTERS);
8168 }
8169
8170 /*
8171 * MSR_IA32_TEMPERATURE_TARGET indicates the temperature where
8172 * the Thermal Control Circuit (TCC) activates.
8173 * This is usually equal to tjMax.
8174 *
8175 * Older processors do not have this MSR, so there we guess,
8176 * but also allow cmdline over-ride with -T.
8177 *
8178 * Several MSR temperature values are in units of degrees-C
8179 * below this value, including the Digital Thermal Sensor (DTS),
8180 * Package Thermal Management Sensor (PTM), and thermal event thresholds.
8181 */
set_temperature_target(PER_THREAD_PARAMS)8182 int set_temperature_target(PER_THREAD_PARAMS)
8183 {
8184 unsigned long long msr;
8185 unsigned int tcc_default, tcc_offset;
8186 int cpu;
8187
8188 UNUSED(c);
8189 UNUSED(p);
8190
8191 /* tj_max is used only for dts or ptm */
8192 if (!(do_dts || do_ptm))
8193 return 0;
8194
8195 /* this is a per-package concept */
8196 if (!is_cpu_first_thread_in_package(t, c, p))
8197 return 0;
8198
8199 cpu = t->cpu_id;
8200 if (cpu_migrate(cpu)) {
8201 fprintf(outf, "Could not migrate to CPU %d\n", cpu);
8202 return -1;
8203 }
8204
8205 if (tj_max_override != 0) {
8206 tj_max = tj_max_override;
8207 fprintf(outf, "cpu%d: Using cmdline TCC Target (%d C)\n", cpu, tj_max);
8208 return 0;
8209 }
8210
8211 /* Temperature Target MSR is Nehalem and newer only */
8212 if (!platform->has_nhm_msrs || no_msr)
8213 goto guess;
8214
8215 if (get_msr(master_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr))
8216 goto guess;
8217
8218 tcc_default = (msr >> 16) & 0xFF;
8219
8220 if (!quiet) {
8221 int bits = platform->tcc_offset_bits;
8222 unsigned long long enabled = 0;
8223
8224 if (bits && !get_msr(master_cpu, MSR_PLATFORM_INFO, &enabled))
8225 enabled = (enabled >> 30) & 1;
8226
8227 if (bits && enabled) {
8228 tcc_offset = (msr >> 24) & GENMASK(bits - 1, 0);
8229 fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C) (%d default - %d offset)\n",
8230 cpu, msr, tcc_default - tcc_offset, tcc_default, tcc_offset);
8231 } else {
8232 fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C)\n", cpu, msr, tcc_default);
8233 }
8234 }
8235
8236 if (!tcc_default)
8237 goto guess;
8238
8239 tj_max = tcc_default;
8240
8241 return 0;
8242
8243 guess:
8244 tj_max = TJMAX_DEFAULT;
8245 fprintf(outf, "cpu%d: Guessing tjMax %d C, Please use -T to specify\n", cpu, tj_max);
8246
8247 return 0;
8248 }
8249
print_thermal(PER_THREAD_PARAMS)8250 int print_thermal(PER_THREAD_PARAMS)
8251 {
8252 unsigned long long msr;
8253 unsigned int dts, dts2;
8254 int cpu;
8255
8256 UNUSED(c);
8257 UNUSED(p);
8258
8259 if (no_msr)
8260 return 0;
8261
8262 if (!(do_dts || do_ptm))
8263 return 0;
8264
8265 cpu = t->cpu_id;
8266
8267 /* DTS is per-core, no need to print for each thread */
8268 if (!is_cpu_first_thread_in_core(t, c))
8269 return 0;
8270
8271 if (cpu_migrate(cpu)) {
8272 fprintf(outf, "print_thermal: Could not migrate to CPU %d\n", cpu);
8273 return -1;
8274 }
8275
8276 if (do_ptm && is_cpu_first_core_in_package(t, p)) {
8277 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
8278 return 0;
8279
8280 dts = (msr >> 16) & 0x7F;
8281 fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_STATUS: 0x%08llx (%d C)\n", cpu, msr, tj_max - dts);
8282
8283 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr))
8284 return 0;
8285
8286 dts = (msr >> 16) & 0x7F;
8287 dts2 = (msr >> 8) & 0x7F;
8288 fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
8289 }
8290
8291 if (do_dts && debug) {
8292 unsigned int resolution;
8293
8294 if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
8295 return 0;
8296
8297 dts = (msr >> 16) & 0x7F;
8298 resolution = (msr >> 27) & 0xF;
8299 fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", cpu, msr, tj_max - dts, resolution);
8300
8301 if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr))
8302 return 0;
8303
8304 dts = (msr >> 16) & 0x7F;
8305 dts2 = (msr >> 8) & 0x7F;
8306 fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
8307 }
8308
8309 return 0;
8310 }
8311
probe_thermal(void)8312 void probe_thermal(void)
8313 {
8314 if (!access("/sys/devices/system/cpu/cpu0/thermal_throttle/core_throttle_count", R_OK))
8315 BIC_PRESENT(BIC_CORE_THROT_CNT);
8316 else
8317 BIC_NOT_PRESENT(BIC_CORE_THROT_CNT);
8318
8319 for_all_cpus(set_temperature_target, ODD_COUNTERS);
8320
8321 if (quiet)
8322 return;
8323
8324 for_all_cpus(print_thermal, ODD_COUNTERS);
8325 }
8326
get_cpu_type(PER_THREAD_PARAMS)8327 int get_cpu_type(PER_THREAD_PARAMS)
8328 {
8329 unsigned int eax, ebx, ecx, edx;
8330
8331 UNUSED(c);
8332 UNUSED(p);
8333
8334 if (!genuine_intel)
8335 return 0;
8336
8337 if (cpu_migrate(t->cpu_id)) {
8338 fprintf(outf, "Could not migrate to CPU %d\n", t->cpu_id);
8339 return -1;
8340 }
8341
8342 if (max_level < 0x1a)
8343 return 0;
8344
8345 __cpuid(0x1a, eax, ebx, ecx, edx);
8346 eax = (eax >> 24) & 0xFF;
8347 if (eax == 0x20)
8348 t->is_atom = true;
8349 return 0;
8350 }
8351
decode_feature_control_msr(void)8352 void decode_feature_control_msr(void)
8353 {
8354 unsigned long long msr;
8355
8356 if (no_msr)
8357 return;
8358
8359 if (quiet)
8360 return;
8361
8362 if (!get_msr(master_cpu, MSR_IA32_FEAT_CTL, &msr))
8363 fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n",
8364 master_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : "");
8365 }
8366
decode_misc_enable_msr(void)8367 void decode_misc_enable_msr(void)
8368 {
8369 unsigned long long msr;
8370
8371 if (no_msr)
8372 return;
8373
8374 if (!genuine_intel)
8375 return;
8376
8377 if (!get_msr(master_cpu, MSR_IA32_MISC_ENABLE, &msr))
8378 fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n",
8379 master_cpu, msr,
8380 msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-",
8381 msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-",
8382 msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-",
8383 msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : "");
8384 }
8385
decode_misc_feature_control(void)8386 void decode_misc_feature_control(void)
8387 {
8388 unsigned long long msr;
8389
8390 if (no_msr)
8391 return;
8392
8393 if (!platform->has_msr_misc_feature_control)
8394 return;
8395
8396 if (!get_msr(master_cpu, MSR_MISC_FEATURE_CONTROL, &msr))
8397 fprintf(outf,
8398 "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n",
8399 master_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : "");
8400 }
8401
8402 /*
8403 * Decode MSR_MISC_PWR_MGMT
8404 *
8405 * Decode the bits according to the Nehalem documentation
8406 * bit[0] seems to continue to have same meaning going forward
8407 * bit[1] less so...
8408 */
decode_misc_pwr_mgmt_msr(void)8409 void decode_misc_pwr_mgmt_msr(void)
8410 {
8411 unsigned long long msr;
8412
8413 if (no_msr)
8414 return;
8415
8416 if (!platform->has_msr_misc_pwr_mgmt)
8417 return;
8418
8419 if (!get_msr(master_cpu, MSR_MISC_PWR_MGMT, &msr))
8420 fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n",
8421 master_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS");
8422 }
8423
8424 /*
8425 * Decode MSR_CC6_DEMOTION_POLICY_CONFIG, MSR_MC6_DEMOTION_POLICY_CONFIG
8426 *
8427 * This MSRs are present on Silvermont processors,
8428 * Intel Atom processor E3000 series (Baytrail), and friends.
8429 */
decode_c6_demotion_policy_msr(void)8430 void decode_c6_demotion_policy_msr(void)
8431 {
8432 unsigned long long msr;
8433
8434 if (no_msr)
8435 return;
8436
8437 if (!platform->has_msr_c6_demotion_policy_config)
8438 return;
8439
8440 if (!get_msr(master_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr))
8441 fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
8442
8443 if (!get_msr(master_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr))
8444 fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
8445 }
8446
print_dev_latency(void)8447 void print_dev_latency(void)
8448 {
8449 char *path = "/dev/cpu_dma_latency";
8450 int fd;
8451 int value;
8452 int retval;
8453
8454 fd = open(path, O_RDONLY);
8455 if (fd < 0) {
8456 if (debug)
8457 warnx("Read %s failed", path);
8458 return;
8459 }
8460
8461 retval = read(fd, (void *)&value, sizeof(int));
8462 if (retval != sizeof(int)) {
8463 warn("read failed %s", path);
8464 close(fd);
8465 return;
8466 }
8467 fprintf(outf, "/dev/cpu_dma_latency: %d usec (%s)\n", value, value == 2000000000 ? "default" : "constrained");
8468
8469 close(fd);
8470 }
8471
has_perf_instr_count_access(void)8472 static int has_perf_instr_count_access(void)
8473 {
8474 int fd;
8475
8476 if (no_perf)
8477 return 0;
8478
8479 fd = open_perf_counter(master_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0);
8480 if (fd != -1)
8481 close(fd);
8482
8483 if (fd == -1)
8484 warnx("Failed to access %s. Some of the counters may not be available\n"
8485 "\tRun as root to enable them or use %s to disable the access explicitly", "perf instructions retired counter",
8486 "'--hide IPC' or '--no-perf'");
8487
8488 return (fd != -1);
8489 }
8490
add_rapl_perf_counter(int cpu,struct rapl_counter_info_t * rci,const struct rapl_counter_arch_info * cai,double * scale_,enum rapl_unit * unit_)8491 int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, double *scale_, enum rapl_unit *unit_)
8492 {
8493 int ret = -1;
8494
8495 if (no_perf)
8496 return -1;
8497
8498 if (!cai->perf_name)
8499 return -1;
8500
8501 const double scale = read_perf_scale(cai->perf_subsys, cai->perf_name);
8502
8503 if (scale == 0.0)
8504 goto end;
8505
8506 const enum rapl_unit unit = read_perf_rapl_unit(cai->perf_subsys, cai->perf_name);
8507
8508 if (unit == RAPL_UNIT_INVALID)
8509 goto end;
8510
8511 const unsigned int rapl_type = read_perf_type(cai->perf_subsys);
8512 const unsigned int rapl_energy_pkg_config = read_perf_config(cai->perf_subsys, cai->perf_name);
8513
8514 ret = open_perf_counter(cpu, rapl_type, rapl_energy_pkg_config, rci->fd_perf, PERF_FORMAT_GROUP);
8515 if (ret == -1)
8516 goto end;
8517
8518 /* If it's the first counter opened, make it a group descriptor */
8519 if (rci->fd_perf == -1)
8520 rci->fd_perf = ret;
8521
8522 *scale_ = scale;
8523 *unit_ = unit;
8524
8525 end:
8526 if (debug >= 2)
8527 fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
8528
8529 return ret;
8530 }
8531
8532 char cpuset_buf[1024];
initialize_cpu_set_from_sysfs(cpu_set_t * cpu_set,char * sysfs_path,char * sysfs_file)8533 int initialize_cpu_set_from_sysfs(cpu_set_t *cpu_set, char *sysfs_path, char *sysfs_file)
8534 {
8535 FILE *fp;
8536 char path[128];
8537
8538 if (snprintf(path, 128, "%s/%s", sysfs_path, sysfs_file) > 128)
8539 err(-1, "%s %s", sysfs_path, sysfs_file);
8540
8541 fp = fopen(path, "r");
8542 if (!fp) {
8543 warn("open %s", path);
8544 return -1;
8545 }
8546 if (fread(cpuset_buf, sizeof(char), 1024, fp) == 0) {
8547 warn("read %s", sysfs_path);
8548 goto err;
8549 }
8550 if (parse_cpu_str(cpuset_buf, cpu_set, cpu_possible_setsize)) {
8551 warnx("%s: cpu str malformat %s\n", sysfs_path, cpu_effective_str);
8552 goto err;
8553 }
8554 return 0;
8555
8556 err:
8557 fclose(fp);
8558 return -1;
8559 }
8560
print_cpu_set(char * s,cpu_set_t * set)8561 void print_cpu_set(char *s, cpu_set_t *set)
8562 {
8563 int i;
8564
8565 assert(MAX_BIC < CPU_SETSIZE);
8566
8567 printf("%s:", s);
8568
8569 for (i = 0; i <= topo.max_cpu_num; ++i)
8570 if (CPU_ISSET(i, set))
8571 printf(" %d", i);
8572 putchar('\n');
8573 }
8574
linux_perf_init_hybrid_cpus(void)8575 void linux_perf_init_hybrid_cpus(void)
8576 {
8577 char *perf_cpu_pcore_path = "/sys/devices/cpu_core";
8578 char *perf_cpu_ecore_path = "/sys/devices/cpu_atom";
8579 char *perf_cpu_lcore_path = "/sys/devices/cpu_lowpower";
8580 char path[128];
8581
8582 if (!access(perf_cpu_pcore_path, F_OK)) {
8583 perf_pcore_set = CPU_ALLOC((topo.max_cpu_num + 1));
8584 if (perf_pcore_set == NULL)
8585 err(3, "CPU_ALLOC");
8586 CPU_ZERO_S(cpu_possible_setsize, perf_pcore_set);
8587 initialize_cpu_set_from_sysfs(perf_pcore_set, perf_cpu_pcore_path, "cpus");
8588 if (debug)
8589 print_cpu_set("perf pcores", perf_pcore_set);
8590 sprintf(path, "%s/%s", perf_cpu_pcore_path, "type");
8591 perf_pmu_types.pcore = snapshot_sysfs_counter(path);
8592 }
8593
8594 if (!access(perf_cpu_ecore_path, F_OK)) {
8595 perf_ecore_set = CPU_ALLOC((topo.max_cpu_num + 1));
8596 if (perf_ecore_set == NULL)
8597 err(3, "CPU_ALLOC");
8598 CPU_ZERO_S(cpu_possible_setsize, perf_ecore_set);
8599 initialize_cpu_set_from_sysfs(perf_ecore_set, perf_cpu_ecore_path, "cpus");
8600 if (debug)
8601 print_cpu_set("perf ecores", perf_ecore_set);
8602 sprintf(path, "%s/%s", perf_cpu_ecore_path, "type");
8603 perf_pmu_types.ecore = snapshot_sysfs_counter(path);
8604 }
8605
8606 if (!access(perf_cpu_lcore_path, F_OK)) {
8607 perf_lcore_set = CPU_ALLOC((topo.max_cpu_num + 1));
8608 if (perf_lcore_set == NULL)
8609 err(3, "CPU_ALLOC");
8610 CPU_ZERO_S(cpu_possible_setsize, perf_lcore_set);
8611 initialize_cpu_set_from_sysfs(perf_lcore_set, perf_cpu_lcore_path, "cpus");
8612 if (debug)
8613 print_cpu_set("perf lcores", perf_lcore_set);
8614 sprintf(path, "%s/%s", perf_cpu_lcore_path, "type");
8615 perf_pmu_types.lcore = snapshot_sysfs_counter(path);
8616 }
8617 }
8618
8619 /*
8620 * Linux-perf related initialization
8621 */
linux_perf_init(void)8622 void linux_perf_init(void)
8623 {
8624 char path[128];
8625 char *perf_cpu_path = "/sys/devices/cpu";
8626
8627 if (access("/proc/sys/kernel/perf_event_paranoid", F_OK))
8628 return;
8629
8630 if (!access(perf_cpu_path, F_OK)) {
8631 sprintf(path, "%s/%s", perf_cpu_path, "type");
8632 perf_pmu_types.uniform = snapshot_sysfs_counter(path);
8633 } else {
8634 linux_perf_init_hybrid_cpus();
8635 }
8636
8637 if (BIC_IS_ENABLED(BIC_IPC) && cpuid_has_aperf_mperf) {
8638 fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8639 if (fd_instr_count_percpu == NULL)
8640 err(-1, "calloc fd_instr_count_percpu");
8641 }
8642 if (BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT)) {
8643 fd_llc_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8644 if (fd_llc_percpu == NULL)
8645 err(-1, "calloc fd_llc_percpu");
8646 }
8647 if (BIC_IS_ENABLED(BIC_L2_MRPS) || BIC_IS_ENABLED(BIC_L2_HIT)) {
8648 fd_l2_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8649 if (fd_l2_percpu == NULL)
8650 err(-1, "calloc fd_l2_percpu");
8651 }
8652 }
8653
rapl_perf_init(void)8654 void rapl_perf_init(void)
8655 {
8656 const unsigned int num_domains = get_rapl_num_domains();
8657 bool *domain_visited = calloc(num_domains, sizeof(bool));
8658
8659 rapl_counter_info_perdomain = calloc(num_domains, sizeof(*rapl_counter_info_perdomain));
8660 if (rapl_counter_info_perdomain == NULL)
8661 err(-1, "calloc rapl_counter_info_percpu");
8662 rapl_counter_info_perdomain_size = num_domains;
8663
8664 /*
8665 * Initialize rapl_counter_info_percpu
8666 */
8667 for (unsigned int domain_id = 0; domain_id < num_domains; ++domain_id) {
8668 struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[domain_id];
8669
8670 rci->fd_perf = -1;
8671 for (size_t i = 0; i < NUM_RAPL_COUNTERS; ++i) {
8672 rci->data[i] = 0;
8673 rci->source[i] = COUNTER_SOURCE_NONE;
8674 }
8675 }
8676
8677 /*
8678 * Open/probe the counters
8679 * If can't get it via perf, fallback to MSR
8680 */
8681 for (size_t i = 0; i < ARRAY_SIZE(rapl_counter_arch_infos); ++i) {
8682
8683 const struct rapl_counter_arch_info *const cai = &rapl_counter_arch_infos[i];
8684 bool has_counter = 0;
8685 double scale;
8686 enum rapl_unit unit;
8687 unsigned int next_domain;
8688
8689 if (!BIC_IS_ENABLED(cai->bic_number))
8690 continue;
8691
8692 memset(domain_visited, 0, num_domains * sizeof(*domain_visited));
8693
8694 for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
8695
8696 if (cpu_is_not_allowed(cpu))
8697 continue;
8698
8699 /* Skip already seen and handled RAPL domains */
8700 next_domain = get_rapl_domain_id(cpu);
8701
8702 assert(next_domain < num_domains);
8703
8704 if (domain_visited[next_domain])
8705 continue;
8706
8707 domain_visited[next_domain] = 1;
8708
8709 if ((cai->flags & RAPL_COUNTER_FLAG_PLATFORM_COUNTER) && (cpu != master_cpu))
8710 continue;
8711
8712 struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[next_domain];
8713
8714 /*
8715 * rapl_counter_arch_infos[] can have multiple entries describing the same
8716 * counter, due to the difference from different platforms/Vendors.
8717 * E.g. rapl_counter_arch_infos[0] and rapl_counter_arch_infos[1] share the
8718 * same perf_subsys and perf_name, but with different MSR address.
8719 * rapl_counter_arch_infos[0] is for Intel and rapl_counter_arch_infos[1]
8720 * is for AMD.
8721 * In this case, it is possible that multiple rapl_counter_arch_infos[]
8722 * entries are probed just because their perf/msr is duplicate and valid.
8723 *
8724 * Thus need a check to avoid re-probe the same counters.
8725 */
8726 if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE)
8727 break;
8728
8729 /* Use perf API for this counter */
8730 if (add_rapl_perf_counter(cpu, rci, cai, &scale, &unit) != -1) {
8731 rci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8732 rci->scale[cai->rci_index] = scale * cai->compat_scale;
8733 rci->unit[cai->rci_index] = unit;
8734 rci->flags[cai->rci_index] = cai->flags;
8735
8736 /* Use MSR for this counter */
8737 } else if (add_rapl_msr_counter(cpu, cai) >= 0) {
8738 rci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8739 rci->msr[cai->rci_index] = cai->msr;
8740 rci->msr_mask[cai->rci_index] = cai->msr_mask;
8741 rci->msr_shift[cai->rci_index] = cai->msr_shift;
8742 rci->unit[cai->rci_index] = RAPL_UNIT_JOULES;
8743 rci->scale[cai->rci_index] = *cai->platform_rapl_msr_scale * cai->compat_scale;
8744 rci->flags[cai->rci_index] = cai->flags;
8745 }
8746
8747 if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE)
8748 has_counter = 1;
8749 }
8750
8751 /* If any CPU has access to the counter, make it present */
8752 if (has_counter)
8753 BIC_PRESENT(cai->bic_number);
8754 }
8755
8756 free(domain_visited);
8757 }
8758
8759 /* Assumes msr_counter_info is populated */
has_amperf_access(void)8760 static int has_amperf_access(void)
8761 {
8762 return cpuid_has_aperf_mperf && msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present && msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present;
8763 }
8764
get_cstate_perf_group_fd(struct cstate_counter_info_t * cci,const char * group_name)8765 int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *group_name)
8766 {
8767 if (strcmp(group_name, "cstate_core") == 0)
8768 return &cci->fd_perf_core;
8769
8770 if (strcmp(group_name, "cstate_pkg") == 0)
8771 return &cci->fd_perf_pkg;
8772
8773 return NULL;
8774 }
8775
add_cstate_perf_counter(int cpu,struct cstate_counter_info_t * cci,const struct cstate_counter_arch_info * cai)8776 int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai)
8777 {
8778 int ret = -1;
8779
8780 if (no_perf)
8781 return -1;
8782
8783 if (!cai->perf_name)
8784 return -1;
8785
8786 int *pfd_group = get_cstate_perf_group_fd(cci, cai->perf_subsys);
8787
8788 if (pfd_group == NULL)
8789 goto end;
8790
8791 const unsigned int type = read_perf_type(cai->perf_subsys);
8792 const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name);
8793
8794 ret = open_perf_counter(cpu, type, config, *pfd_group, PERF_FORMAT_GROUP);
8795
8796 if (ret == -1)
8797 goto end;
8798
8799 /* If it's the first counter opened, make it a group descriptor */
8800 if (*pfd_group == -1)
8801 *pfd_group = ret;
8802
8803 end:
8804 if (debug >= 2)
8805 fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
8806
8807 return ret;
8808 }
8809
add_msr_perf_counter(int cpu,struct msr_counter_info_t * cci,const struct msr_counter_arch_info * cai)8810 int add_msr_perf_counter(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai)
8811 {
8812 int ret = -1;
8813
8814 if (no_perf)
8815 return -1;
8816
8817 if (!cai->perf_name)
8818 return -1;
8819
8820 const unsigned int type = read_perf_type(cai->perf_subsys);
8821 const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name);
8822
8823 ret = open_perf_counter(cpu, type, config, cci->fd_perf, PERF_FORMAT_GROUP);
8824
8825 if (ret == -1)
8826 goto end;
8827
8828 /* If it's the first counter opened, make it a group descriptor */
8829 if (cci->fd_perf == -1)
8830 cci->fd_perf = ret;
8831
8832 end:
8833 if (debug)
8834 fprintf(stderr, "%s: %s/%s: %d (cpu: %d)\n", __func__, cai->perf_subsys, cai->perf_name, ret, cpu);
8835
8836 return ret;
8837 }
8838
msr_perf_init_(void)8839 void msr_perf_init_(void)
8840 {
8841 const int mci_num = topo.max_cpu_num + 1;
8842
8843 msr_counter_info = calloc(mci_num, sizeof(*msr_counter_info));
8844 if (!msr_counter_info)
8845 err(1, "calloc msr_counter_info");
8846 msr_counter_info_size = mci_num;
8847
8848 for (int cpu = 0; cpu < mci_num; ++cpu)
8849 msr_counter_info[cpu].fd_perf = -1;
8850
8851 for (int cidx = 0; cidx < NUM_MSR_COUNTERS; ++cidx) {
8852
8853 struct msr_counter_arch_info *cai = &msr_counter_arch_infos[cidx];
8854
8855 cai->present = false;
8856
8857 for (int cpu = 0; cpu < mci_num; ++cpu) {
8858
8859 struct msr_counter_info_t *const cci = &msr_counter_info[cpu];
8860
8861 if (cpu_is_not_allowed(cpu))
8862 continue;
8863
8864 if (cai->needed) {
8865 /* Use perf API for this counter */
8866 if (add_msr_perf_counter(cpu, cci, cai) != -1) {
8867 cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8868 cai->present = true;
8869
8870 /* User MSR for this counter */
8871 } else if (add_msr_counter(cpu, cai->msr) >= 0) {
8872 cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8873 cci->msr[cai->rci_index] = cai->msr;
8874 cci->msr_mask[cai->rci_index] = cai->msr_mask;
8875 cai->present = true;
8876 }
8877 }
8878 }
8879 }
8880 }
8881
8882 /* Initialize data for reading perf counters from the MSR group. */
msr_perf_init(void)8883 void msr_perf_init(void)
8884 {
8885 bool need_amperf = false, need_smi = false;
8886 const bool need_soft_c1 = (!platform->has_msr_core_c1_res) && (platform->supported_cstates & CC1);
8887
8888 need_amperf = BIC_IS_ENABLED(BIC_Avg_MHz) || BIC_IS_ENABLED(BIC_Busy) || BIC_IS_ENABLED(BIC_Bzy_MHz)
8889 || BIC_IS_ENABLED(BIC_IPC) || need_soft_c1;
8890
8891 if (BIC_IS_ENABLED(BIC_SMI))
8892 need_smi = true;
8893
8894 /* Enable needed counters */
8895 msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].needed = need_amperf;
8896 msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].needed = need_amperf;
8897 msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].needed = need_smi;
8898
8899 msr_perf_init_();
8900
8901 const bool has_amperf = has_amperf_access();
8902 const bool has_smi = msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].present;
8903
8904 has_aperf_access = has_amperf;
8905
8906 if (has_amperf) {
8907 BIC_PRESENT(BIC_Avg_MHz);
8908 BIC_PRESENT(BIC_Busy);
8909 BIC_PRESENT(BIC_Bzy_MHz);
8910 BIC_PRESENT(BIC_SMI);
8911 }
8912
8913 if (has_smi)
8914 BIC_PRESENT(BIC_SMI);
8915 }
8916
cstate_perf_init_(bool soft_c1)8917 void cstate_perf_init_(bool soft_c1)
8918 {
8919 bool has_counter;
8920 bool *cores_visited = NULL, *pkg_visited = NULL;
8921 const int cores_visited_elems = topo.max_core_id + 1;
8922 const int pkg_visited_elems = topo.max_package_id + 1;
8923 const int cci_num = topo.max_cpu_num + 1;
8924
8925 ccstate_counter_info = calloc(cci_num, sizeof(*ccstate_counter_info));
8926 if (!ccstate_counter_info)
8927 err(1, "calloc ccstate_counter_arch_info");
8928 ccstate_counter_info_size = cci_num;
8929
8930 cores_visited = calloc(cores_visited_elems, sizeof(*cores_visited));
8931 if (!cores_visited)
8932 err(1, "calloc cores_visited");
8933
8934 pkg_visited = calloc(pkg_visited_elems, sizeof(*pkg_visited));
8935 if (!pkg_visited)
8936 err(1, "calloc pkg_visited");
8937
8938 /* Initialize cstate_counter_info_percpu */
8939 for (int cpu = 0; cpu < cci_num; ++cpu) {
8940 ccstate_counter_info[cpu].fd_perf_core = -1;
8941 ccstate_counter_info[cpu].fd_perf_pkg = -1;
8942 }
8943
8944 for (int cidx = 0; cidx < NUM_CSTATE_COUNTERS; ++cidx) {
8945 has_counter = false;
8946 memset(cores_visited, 0, cores_visited_elems * sizeof(*cores_visited));
8947 memset(pkg_visited, 0, pkg_visited_elems * sizeof(*pkg_visited));
8948
8949 const struct cstate_counter_arch_info *cai = &ccstate_counter_arch_infos[cidx];
8950
8951 for (int cpu = 0; cpu < cci_num; ++cpu) {
8952
8953 struct cstate_counter_info_t *const cci = &ccstate_counter_info[cpu];
8954
8955 if (cpu_is_not_allowed(cpu))
8956 continue;
8957
8958 const int core_id = cpus[cpu].core_id;
8959 const int pkg_id = cpus[cpu].package_id;
8960
8961 assert(core_id < cores_visited_elems);
8962 assert(pkg_id < pkg_visited_elems);
8963
8964 const bool per_thread = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD;
8965 const bool per_core = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_CORE;
8966
8967 if (!per_thread && cores_visited[core_id])
8968 continue;
8969
8970 if (!per_core && pkg_visited[pkg_id])
8971 continue;
8972
8973 const bool counter_needed = BIC_IS_ENABLED(cai->bic_number) || (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY));
8974 const bool counter_supported = (platform->supported_cstates & cai->feature_mask);
8975
8976 if (counter_needed && counter_supported) {
8977 /* Use perf API for this counter */
8978 if (add_cstate_perf_counter(cpu, cci, cai) != -1) {
8979
8980 cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8981
8982 /* User MSR for this counter */
8983 } else if (pkg_cstate_limit >= cai->pkg_cstate_limit && add_msr_counter(cpu, cai->msr) >= 0) {
8984 cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8985 cci->msr[cai->rci_index] = cai->msr;
8986 }
8987 }
8988
8989 if (cci->source[cai->rci_index] != COUNTER_SOURCE_NONE) {
8990 has_counter = true;
8991 cores_visited[core_id] = true;
8992 pkg_visited[pkg_id] = true;
8993 }
8994 }
8995
8996 /* If any CPU has access to the counter, make it present */
8997 if (has_counter)
8998 BIC_PRESENT(cai->bic_number);
8999 }
9000
9001 free(cores_visited);
9002 free(pkg_visited);
9003 }
9004
cstate_perf_init(void)9005 void cstate_perf_init(void)
9006 {
9007 /*
9008 * If we don't have a C1 residency MSR, we calculate it "in software",
9009 * but we need APERF, MPERF too.
9010 */
9011 const bool soft_c1 = !platform->has_msr_core_c1_res && has_amperf_access()
9012 && platform->supported_cstates & CC1;
9013
9014 if (soft_c1)
9015 BIC_PRESENT(BIC_CPU_c1);
9016
9017 cstate_perf_init_(soft_c1);
9018 }
9019
probe_cstates(void)9020 void probe_cstates(void)
9021 {
9022 probe_cst_limit();
9023
9024 if (platform->has_msr_module_c6_res_ms)
9025 BIC_PRESENT(BIC_Mod_c6);
9026
9027 if (platform->has_ext_cst_msrs && !no_msr) {
9028 BIC_PRESENT(BIC_Totl_c0);
9029 BIC_PRESENT(BIC_Any_c0);
9030 BIC_PRESENT(BIC_GFX_c0);
9031 BIC_PRESENT(BIC_CPUGFX);
9032 }
9033
9034 if (quiet)
9035 return;
9036
9037 dump_power_ctl();
9038 dump_cst_cfg();
9039 decode_c6_demotion_policy_msr();
9040 print_dev_latency();
9041 dump_sysfs_cstate_config();
9042 print_irtl();
9043 }
9044
probe_lpi(void)9045 void probe_lpi(void)
9046 {
9047 if (!access("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", R_OK))
9048 BIC_PRESENT(BIC_CPU_LPI);
9049 else
9050 BIC_NOT_PRESENT(BIC_CPU_LPI);
9051
9052 if (!access(sys_lpi_file_sysfs, R_OK)) {
9053 sys_lpi_file = sys_lpi_file_sysfs;
9054 BIC_PRESENT(BIC_SYS_LPI);
9055 } else if (!access(sys_lpi_file_debugfs, R_OK)) {
9056 sys_lpi_file = sys_lpi_file_debugfs;
9057 BIC_PRESENT(BIC_SYS_LPI);
9058 } else {
9059 sys_lpi_file_sysfs = NULL;
9060 BIC_NOT_PRESENT(BIC_SYS_LPI);
9061 }
9062
9063 }
9064
probe_pstates(void)9065 void probe_pstates(void)
9066 {
9067 probe_bclk();
9068
9069 if (quiet)
9070 return;
9071
9072 dump_platform_info();
9073 dump_turbo_ratio_info();
9074 dump_sysfs_pstate_config();
9075 decode_misc_pwr_mgmt_msr();
9076
9077 for_all_cpus(print_hwp, ODD_COUNTERS);
9078 for_all_cpus(print_epb, ODD_COUNTERS);
9079 for_all_cpus(print_perf_limit, ODD_COUNTERS);
9080 }
9081
dump_word_chars(unsigned int word)9082 void dump_word_chars(unsigned int word)
9083 {
9084 int i;
9085
9086 for (i = 0; i < 4; ++i)
9087 fprintf(outf, "%c", (word >> (i * 8)) & 0xFF);
9088 }
9089
dump_cpuid_hypervisor(void)9090 void dump_cpuid_hypervisor(void)
9091 {
9092 unsigned int ebx = 0;
9093 unsigned int ecx = 0;
9094 unsigned int edx = 0;
9095
9096 __cpuid(0x40000000, max_extended_level, ebx, ecx, edx);
9097
9098 fprintf(outf, "Hypervisor: ");
9099 dump_word_chars(ebx);
9100 dump_word_chars(ecx);
9101 dump_word_chars(edx);
9102 fprintf(outf, "\n");
9103 }
9104
process_cpuid()9105 void process_cpuid()
9106 {
9107 unsigned int eax, ebx, ecx, edx;
9108 unsigned int fms, family, model, stepping, ecx_flags, edx_flags;
9109 unsigned long long ucode_patch = 0;
9110 bool ucode_patch_valid = false;
9111
9112 eax = ebx = ecx = edx = 0;
9113
9114 __cpuid(0, max_level, ebx, ecx, edx);
9115
9116 if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
9117 genuine_intel = 1;
9118 else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
9119 authentic_amd = 1;
9120 else if (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e)
9121 hygon_genuine = 1;
9122
9123 if (!quiet)
9124 fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", (char *)&ebx, (char *)&edx, (char *)&ecx, max_level);
9125
9126 __cpuid(1, fms, ebx, ecx, edx);
9127 family = (fms >> 8) & 0xf;
9128 model = (fms >> 4) & 0xf;
9129 stepping = fms & 0xf;
9130 if (family == 0xf)
9131 family += (fms >> 20) & 0xff;
9132 if (family >= 6)
9133 model += ((fms >> 16) & 0xf) << 4;
9134 ecx_flags = ecx;
9135 edx_flags = edx;
9136 cpuid_has_hv = ecx_flags & (1 << 31);
9137
9138 if (!no_msr) {
9139 if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) {
9140 warnx("get_msr(UCODE)");
9141 } else {
9142 ucode_patch_valid = true;
9143 if (!authentic_amd && !hygon_genuine)
9144 ucode_patch >>= 32;
9145 }
9146 }
9147
9148 /*
9149 * check max extended function levels of CPUID.
9150 * This is needed to check for invariant TSC.
9151 * This check is valid for both Intel and AMD.
9152 */
9153 ebx = ecx = edx = 0;
9154 __cpuid(0x80000000, max_extended_level, ebx, ecx, edx);
9155
9156 if (!quiet) {
9157 fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", family, model, stepping, family, model, stepping);
9158 if (ucode_patch_valid)
9159 fprintf(outf, " microcode 0x%x", (unsigned int)ucode_patch);
9160 fputc('\n', outf);
9161
9162 fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level);
9163 fprintf(outf, "CPUID(1): %sSSE3 %sMONITOR %sSMX %sEIST %sTM2 %sHV %sTSC %sMSR %sACPI-TM %sHT %sTM\n",
9164 ecx_flags & (1 << 0) ? "" : "No-",
9165 ecx_flags & (1 << 3) ? "" : "No-",
9166 ecx_flags & (1 << 6) ? "" : "No-",
9167 ecx_flags & (1 << 7) ? "" : "No-",
9168 ecx_flags & (1 << 8) ? "" : "No-",
9169 cpuid_has_hv ? "" : "No-",
9170 edx_flags & (1 << 4) ? "" : "No-",
9171 edx_flags & (1 << 5) ? "" : "No-",
9172 edx_flags & (1 << 22) ? "" : "No-", edx_flags & (1 << 28) ? "" : "No-", edx_flags & (1 << 29) ? "" : "No-");
9173 }
9174 if (!quiet && cpuid_has_hv)
9175 dump_cpuid_hypervisor();
9176
9177 probe_platform_features(family, model);
9178 init_perf_model_support(family, model);
9179
9180 if (!(edx_flags & (1 << 5)))
9181 errx(1, "CPUID: no MSR");
9182
9183 if (max_extended_level >= 0x80000007) {
9184
9185 /*
9186 * Non-Stop TSC is advertised by CPUID.EAX=0x80000007: EDX.bit8
9187 * this check is valid for both Intel and AMD
9188 */
9189 __cpuid(0x80000007, eax, ebx, ecx, edx);
9190 has_invariant_tsc = edx & (1 << 8);
9191 }
9192
9193 /*
9194 * APERF/MPERF is advertised by CPUID.EAX=0x6: ECX.bit0
9195 * this check is valid for both Intel and AMD
9196 */
9197
9198 __cpuid(0x6, eax, ebx, ecx, edx);
9199 cpuid_has_aperf_mperf = ecx & (1 << 0);
9200 do_dts = eax & (1 << 0);
9201 if (do_dts)
9202 BIC_PRESENT(BIC_CoreTmp);
9203 has_turbo = eax & (1 << 1);
9204 do_ptm = eax & (1 << 6);
9205 if (do_ptm)
9206 BIC_PRESENT(BIC_PkgTmp);
9207 has_hwp = eax & (1 << 7);
9208 has_hwp_notify = eax & (1 << 8);
9209 has_hwp_activity_window = eax & (1 << 9);
9210 has_hwp_epp = eax & (1 << 10);
9211 has_hwp_pkg = eax & (1 << 11);
9212 has_epb = ecx & (1 << 3);
9213
9214 if (!quiet)
9215 fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, "
9216 "%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n",
9217 cpuid_has_aperf_mperf ? "" : "No-",
9218 has_turbo ? "" : "No-",
9219 do_dts ? "" : "No-",
9220 do_ptm ? "" : "No-",
9221 has_hwp ? "" : "No-",
9222 has_hwp_notify ? "" : "No-",
9223 has_hwp_activity_window ? "" : "No-", has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-");
9224
9225 if (!quiet)
9226 decode_misc_enable_msr();
9227
9228 if (max_level >= 0x7) {
9229 int has_sgx;
9230
9231 ecx = 0;
9232
9233 __cpuid_count(0x7, 0, eax, ebx, ecx, edx);
9234
9235 has_sgx = ebx & (1 << 2);
9236
9237 is_hybrid = !!(edx & (1 << 15));
9238
9239 if (!quiet)
9240 fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-");
9241
9242 if (has_sgx)
9243 decode_feature_control_msr();
9244 }
9245
9246 if (max_level >= 0x15) {
9247 unsigned int eax_crystal;
9248 unsigned int ebx_tsc;
9249
9250 /*
9251 * CPUID 15H TSC/Crystal ratio, possibly Crystal Hz
9252 */
9253 eax_crystal = ebx_tsc = crystal_hz = edx = 0;
9254 __cpuid(0x15, eax_crystal, ebx_tsc, crystal_hz, edx);
9255
9256 if (ebx_tsc != 0) {
9257 if (!quiet && (ebx != 0))
9258 fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", eax_crystal, ebx_tsc, crystal_hz);
9259
9260 if (crystal_hz == 0)
9261 crystal_hz = platform->crystal_freq;
9262
9263 if (crystal_hz) {
9264 tsc_hz = (unsigned long long)crystal_hz *ebx_tsc / eax_crystal;
9265 if (!quiet)
9266 fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n", tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal);
9267 }
9268 }
9269 }
9270 if (max_level >= 0x16) {
9271 unsigned int base_mhz, max_mhz, bus_mhz, edx;
9272
9273 /*
9274 * CPUID 16H Base MHz, Max MHz, Bus MHz
9275 */
9276 base_mhz = max_mhz = bus_mhz = edx = 0;
9277
9278 __cpuid(0x16, base_mhz, max_mhz, bus_mhz, edx);
9279
9280 bclk = bus_mhz;
9281
9282 base_hz = base_mhz * 1000000;
9283 has_base_hz = 1;
9284
9285 if (platform->enable_tsc_tweak)
9286 tsc_tweak = base_hz / tsc_hz;
9287
9288 if (!quiet)
9289 fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", base_mhz, max_mhz, bus_mhz);
9290 }
9291
9292 if (cpuid_has_aperf_mperf)
9293 aperf_mperf_multiplier = platform->need_perf_multiplier ? 1024 : 1;
9294
9295 BIC_PRESENT(BIC_IRQ);
9296 BIC_PRESENT(BIC_NMI);
9297 BIC_PRESENT(BIC_TSC_MHz);
9298 }
9299
counter_info_init(void)9300 static void counter_info_init(void)
9301 {
9302 for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i) {
9303 struct cstate_counter_arch_info *const cai = &ccstate_counter_arch_infos[i];
9304
9305 if (platform->has_msr_knl_core_c6_residency && cai->msr == MSR_CORE_C6_RESIDENCY)
9306 cai->msr = MSR_KNL_CORE_C6_RESIDENCY;
9307
9308 if (!platform->has_msr_core_c1_res && cai->msr == MSR_CORE_C1_RES)
9309 cai->msr = 0;
9310
9311 if (platform->has_msr_atom_pkg_c6_residency && cai->msr == MSR_PKG_C6_RESIDENCY)
9312 cai->msr = MSR_ATOM_PKG_C6_RESIDENCY;
9313 }
9314
9315 for (int i = 0; i < NUM_MSR_COUNTERS; ++i) {
9316 msr_counter_arch_infos[i].present = false;
9317 msr_counter_arch_infos[i].needed = false;
9318 }
9319 }
9320
probe_pm_features(void)9321 void probe_pm_features(void)
9322 {
9323 probe_pstates();
9324
9325 probe_cstates();
9326
9327 probe_lpi();
9328
9329 probe_intel_uncore_frequency();
9330
9331 probe_graphics();
9332
9333 probe_rapl();
9334
9335 probe_thermal();
9336
9337 if (platform->has_nhm_msrs && !no_msr)
9338 BIC_PRESENT(BIC_SMI);
9339
9340 if (!quiet)
9341 decode_misc_feature_control();
9342 }
9343
9344 /*
9345 * has_perf_llc_access()
9346 *
9347 * return 1 on success, else 0
9348 */
has_perf_llc_access(void)9349 int has_perf_llc_access(void)
9350 {
9351 int fd;
9352
9353 if (no_perf)
9354 return 0;
9355
9356 fd = open_perf_counter(master_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP);
9357 if (fd != -1)
9358 close(fd);
9359
9360 if (fd == -1)
9361 warnx("Failed to access %s. Some of the counters may not be available\n"
9362 "\tRun as root to enable them or use %s to disable the access explicitly", "perf LLC counters", "'--hide LLC' or '--no-perf'");
9363
9364 return (fd != -1);
9365 }
9366
perf_llc_init(void)9367 void perf_llc_init(void)
9368 {
9369 int cpu;
9370 int retval;
9371
9372 if (no_perf)
9373 return;
9374 if (!(BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT)))
9375 return;
9376
9377 assert(fd_llc_percpu != 0);
9378
9379 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
9380
9381 if (cpu_is_not_allowed(cpu))
9382 continue;
9383
9384 fd_llc_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP);
9385 if (fd_llc_percpu[cpu] == -1) {
9386 warnx("%s: perf REFS: failed to open counter on cpu%d", __func__, cpu);
9387 free_fd_llc_percpu();
9388 return;
9389 }
9390 retval = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, fd_llc_percpu[cpu], PERF_FORMAT_GROUP);
9391 if (retval == -1) {
9392 warnx("%s: perf MISS: failed to open counter on cpu%d", __func__, cpu);
9393 free_fd_llc_percpu();
9394 return;
9395 }
9396 }
9397 BIC_PRESENT(BIC_LLC_MRPS);
9398 BIC_PRESENT(BIC_LLC_HIT);
9399 }
9400
perf_l2_init(void)9401 void perf_l2_init(void)
9402 {
9403 int cpu;
9404 int retval;
9405
9406 if (no_perf)
9407 return;
9408 if (!(BIC_IS_ENABLED(BIC_L2_MRPS) || BIC_IS_ENABLED(BIC_L2_HIT)))
9409 return;
9410 if (perf_model_support == NULL)
9411 return;
9412
9413 assert(fd_l2_percpu != 0);
9414
9415 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
9416
9417 if (cpu_is_not_allowed(cpu))
9418 continue;
9419
9420 if (!is_hybrid) {
9421 fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.uniform, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP);
9422 if (fd_l2_percpu[cpu] == -1) {
9423 warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.uniform, perf_model_support->first.refs);
9424 free_fd_l2_percpu();
9425 return;
9426 }
9427 retval = open_perf_counter(cpu, perf_pmu_types.uniform, perf_model_support->first.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP);
9428 if (retval == -1) {
9429 warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.uniform, perf_model_support->first.hits);
9430 free_fd_l2_percpu();
9431 return;
9432 }
9433 continue;
9434 }
9435 if (perf_pcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_pcore_set)) {
9436 fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP);
9437 if (fd_l2_percpu[cpu] == -1) {
9438 warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.refs);
9439 free_fd_l2_percpu();
9440 return;
9441 }
9442 retval = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP);
9443 if (retval == -1) {
9444 warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.hits);
9445 free_fd_l2_percpu();
9446 return;
9447 }
9448 } else if (perf_ecore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_ecore_set)) {
9449 fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.refs, -1, PERF_FORMAT_GROUP);
9450 if (fd_l2_percpu[cpu] == -1) {
9451 warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.ecore, perf_model_support->second.refs);
9452 free_fd_l2_percpu();
9453 return;
9454 }
9455 retval = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP);
9456 if (retval == -1) {
9457 warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.ecore, perf_model_support->second.hits);
9458 free_fd_l2_percpu();
9459 return;
9460 }
9461 } else if (perf_lcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_lcore_set)) {
9462 fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.refs, -1, PERF_FORMAT_GROUP);
9463 if (fd_l2_percpu[cpu] == -1) {
9464 warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.lcore, perf_model_support->third.refs);
9465 free_fd_l2_percpu();
9466 return;
9467 }
9468 retval = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP);
9469 if (retval == -1) {
9470 warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.lcore, perf_model_support->third.hits);
9471 free_fd_l2_percpu();
9472 return;
9473 }
9474 } else
9475 err(-1, "%s: cpu%d: type %d", __func__, cpu, cpus[cpu].type);
9476 }
9477 BIC_PRESENT(BIC_L2_MRPS);
9478 BIC_PRESENT(BIC_L2_HIT);
9479 }
9480
9481 /*
9482 * in /dev/cpu/ return success for names that are numbers
9483 * ie. filter out ".", "..", "microcode".
9484 */
dir_filter(const struct dirent * dirp)9485 int dir_filter(const struct dirent *dirp)
9486 {
9487 if (isdigit(dirp->d_name[0]))
9488 return 1;
9489 else
9490 return 0;
9491 }
9492
set_thread_siblings(struct cpu_topology * thiscpu)9493 int set_thread_siblings(struct cpu_topology *thiscpu)
9494 {
9495 char path[80];
9496 int cpu = thiscpu->cpu_id;
9497 size_t size;
9498 int ht_id = 0;
9499 int i;
9500
9501 thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1));
9502 if (thiscpu->ht_id < 0)
9503 thiscpu->ht_id = 0; /* first CPU in core */
9504 if (!thiscpu->put_ids)
9505 return -1;
9506
9507 size = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9508 CPU_ZERO_S(size, thiscpu->put_ids);
9509
9510 sprintf(path, "/sys/devices/system/cpu/cpu%d/topology", cpu);
9511
9512 initialize_cpu_set_from_sysfs(thiscpu->put_ids, path, "thread_siblings_list");
9513
9514 for (i = 0; i <= topo.max_cpu_num; ++i)
9515 if (CPU_ISSET_S(i, size, thiscpu->put_ids)) {
9516 cpus[i].ht_id = ht_id;
9517 cpus[cpu].ht_sibling_cpu_id[ht_id] = i;
9518 ht_id += 1;
9519 }
9520
9521 return (ht_id - 1);
9522 }
9523
topology_probe(bool startup)9524 void topology_probe(bool startup)
9525 {
9526 int i;
9527 int max_core_id = 0;
9528 int max_package_id = 0;
9529 int max_siblings = 0;
9530
9531 /* Initialize num_cpus, max_cpu_num */
9532 set_max_cpu_num();
9533 topo.num_cpus = 0;
9534 for_all_proc_cpus(count_cpus);
9535 if (!summary_only)
9536 BIC_PRESENT(BIC_CPU);
9537
9538 if (debug > 1)
9539 fprintf(outf, "num_cpus %d max_cpu_num %d\n", topo.num_cpus, topo.max_cpu_num);
9540
9541 cpus = calloc(1, (topo.max_cpu_num + 1) * sizeof(struct cpu_topology));
9542 if (cpus == NULL)
9543 err(1, "calloc cpus");
9544
9545 /*
9546 * Allocate and initialize cpu_present_set
9547 */
9548 cpu_present_set = CPU_ALLOC((topo.max_cpu_num + 1));
9549 if (cpu_present_set == NULL)
9550 err(3, "CPU_ALLOC");
9551 cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9552 CPU_ZERO_S(cpu_present_setsize, cpu_present_set);
9553 for_all_proc_cpus(mark_cpu_present);
9554 if (debug)
9555 print_cpu_set("present set", cpu_present_set);
9556
9557 /*
9558 * Allocate and initialize cpu_possible_set
9559 */
9560 cpu_possible_set = CPU_ALLOC((topo.max_cpu_num + 1));
9561 if (cpu_possible_set == NULL)
9562 err(3, "CPU_ALLOC");
9563 cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9564 CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set);
9565 initialize_cpu_set_from_sysfs(cpu_possible_set, "/sys/devices/system/cpu", "possible");
9566 if (debug)
9567 print_cpu_set("possible set", cpu_possible_set);
9568
9569 /*
9570 * Allocate and initialize cpu_effective_set
9571 */
9572 cpu_effective_set = CPU_ALLOC((topo.max_cpu_num + 1));
9573 if (cpu_effective_set == NULL)
9574 err(3, "CPU_ALLOC");
9575 cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9576 CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set);
9577 update_effective_set(startup);
9578 if (debug)
9579 print_cpu_set("effective set", cpu_effective_set);
9580
9581 /*
9582 * Allocate and initialize cpu_allowed_set
9583 */
9584 cpu_allowed_set = CPU_ALLOC((topo.max_cpu_num + 1));
9585 if (cpu_allowed_set == NULL)
9586 err(3, "CPU_ALLOC");
9587 cpu_allowed_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9588 CPU_ZERO_S(cpu_allowed_setsize, cpu_allowed_set);
9589
9590 /*
9591 * Validate and update cpu_allowed_set.
9592 *
9593 * Make sure all cpus in cpu_subset are also in cpu_present_set during startup.
9594 * Give a warning when cpus in cpu_subset become unavailable at runtime.
9595 * Give a warning when cpus are not effective because of cgroup setting.
9596 *
9597 * cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpu_subset.
9598 */
9599 for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) {
9600 if (cpu_subset && !CPU_ISSET_S(i, cpu_subset_size, cpu_subset))
9601 continue;
9602
9603 if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set)) {
9604 if (cpu_subset) {
9605 /* cpus in cpu_subset must be in cpu_present_set during startup */
9606 if (startup)
9607 err(1, "cpu%d not present", i);
9608 else
9609 fprintf(stderr, "cpu%d not present\n", i);
9610 }
9611 continue;
9612 }
9613
9614 if (CPU_COUNT_S(cpu_effective_setsize, cpu_effective_set)) {
9615 if (!CPU_ISSET_S(i, cpu_effective_setsize, cpu_effective_set)) {
9616 fprintf(stderr, "cpu%d not effective\n", i);
9617 continue;
9618 }
9619 }
9620
9621 CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set);
9622 }
9623 if (debug)
9624 print_cpu_set("allowed set", cpu_allowed_set);
9625
9626 if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set))
9627 err(-ENODEV, "No valid cpus found");
9628 sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set);
9629
9630 /*
9631 * Allocate and initialize cpu_affinity_set
9632 */
9633 cpu_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1));
9634 if (cpu_affinity_set == NULL)
9635 err(3, "CPU_ALLOC");
9636 cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9637 CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
9638
9639 for_all_proc_cpus(clear_ht_id);
9640
9641 for_all_proc_cpus(set_cpu_hybrid_type);
9642
9643 /*
9644 * For online cpus
9645 * find max_core_id, max_package_id, num_cores (per system)
9646 */
9647 topo.min_module_id = 0x7FFFFFFF;
9648 for (i = 0; i <= topo.max_cpu_num; ++i) {
9649 int siblings;
9650
9651 if (cpu_is_not_present(i)) {
9652 if (debug > 1)
9653 fprintf(outf, "cpu%d NOT PRESENT\n", i);
9654 continue;
9655 }
9656
9657 cpus[i].cpu_id = i;
9658
9659 /* get package information */
9660 cpus[i].package_id = get_package_id(i);
9661 if (cpus[i].package_id > max_package_id)
9662 max_package_id = cpus[i].package_id;
9663
9664 /* get die information */
9665 cpus[i].die_id = get_die_id(i);
9666 if (cpus[i].die_id > topo.max_die_id)
9667 topo.max_die_id = cpus[i].die_id;
9668
9669 /* get l3 information */
9670 cpus[i].l3_id = get_l3_id(i);
9671 if (cpus[i].l3_id > topo.max_l3_id)
9672 topo.max_l3_id = cpus[i].l3_id;
9673
9674 /* get numa node information */
9675 cpus[i].physical_node_id = get_physical_node_id(&cpus[i]);
9676 if (cpus[i].physical_node_id > topo.max_node_num)
9677 topo.max_node_num = cpus[i].physical_node_id;
9678
9679 /* get module information */
9680 cpus[i].module_id = get_module_id(i);
9681 if (cpus[i].module_id > topo.max_module_id)
9682 topo.max_module_id = cpus[i].module_id;
9683 if (cpus[i].module_id < topo.min_module_id)
9684 topo.min_module_id = cpus[i].module_id;
9685
9686 /* get core information */
9687 cpus[i].core_id = get_core_id(i);
9688 if (cpus[i].core_id > max_core_id)
9689 max_core_id = cpus[i].core_id;
9690
9691 /* get thread information */
9692 siblings = set_thread_siblings(&cpus[i]);
9693 if (siblings > max_siblings)
9694 max_siblings = siblings;
9695 if (cpus[i].ht_id == 0)
9696 topo.num_cores++;
9697 }
9698 topo.max_core_id = max_core_id; /* within a package */
9699 topo.max_package_id = max_package_id;
9700
9701 topo.cores_per_pkg = max_core_id + 1;
9702 if (debug > 1)
9703 fprintf(outf, "max_core_id %d, sizing for %d cores per package\n", max_core_id, topo.cores_per_pkg);
9704 if (!summary_only)
9705 BIC_PRESENT(BIC_Core);
9706
9707 if (debug > 1)
9708 fprintf(outf, "min_module_id %d max_module_id %d\n", topo.min_module_id, topo.max_module_id);
9709 if (!summary_only && (topo.min_module_id != topo.max_module_id))
9710 BIC_PRESENT(BIC_Module);
9711
9712 topo.num_die = topo.max_die_id + 1;
9713 if (debug > 1)
9714 fprintf(outf, "max_die_id %d, sizing for %d die\n", topo.max_die_id, topo.num_die);
9715 if (!summary_only && topo.num_die > 1)
9716 BIC_PRESENT(BIC_Die);
9717
9718 if (!summary_only && topo.max_l3_id > 0)
9719 BIC_PRESENT(BIC_L3);
9720
9721 topo.num_packages = max_package_id + 1;
9722 if (debug > 1)
9723 fprintf(outf, "max_package_id %d, sizing for %d packages\n", max_package_id, topo.num_packages);
9724 if (!summary_only && topo.num_packages > 1)
9725 BIC_PRESENT(BIC_Package);
9726
9727 set_node_data();
9728 if (debug > 1)
9729 fprintf(outf, "nodes_per_pkg %d\n", topo.nodes_per_pkg);
9730 if (!summary_only && topo.nodes_per_pkg > 1)
9731 BIC_PRESENT(BIC_Node);
9732
9733 topo.threads_per_core = max_siblings;
9734 if (debug > 1)
9735 fprintf(outf, "max_siblings %d\n", max_siblings);
9736
9737 if (debug < 1)
9738 return;
9739
9740 for (i = 0; i <= topo.max_cpu_num; ++i) {
9741 int ht_id;
9742
9743 if (cpu_is_not_present(i))
9744 continue;
9745 fprintf(outf,
9746 "cpu %d pkg %d die %d l3 %d node %d lnode %d module 0x%x core %d ht_id %d",
9747 i, cpus[i].package_id, cpus[i].die_id, cpus[i].l3_id,
9748 cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].module_id, cpus[i].core_id, cpus[i].ht_id);
9749 fprintf(outf, " siblings");
9750 for (ht_id = 0; ht_id <= MAX_HT_ID; ++ht_id)
9751 fprintf(outf, " %d", cpus[i].ht_sibling_cpu_id[ht_id]);
9752 fprintf(outf, "\n");
9753 }
9754
9755 }
9756
allocate_counters_1(struct counters * counters)9757 void allocate_counters_1(struct counters *counters)
9758 {
9759 counters->threads = calloc(1, sizeof(struct thread_data));
9760 if (counters->threads == NULL)
9761 goto error;
9762
9763 counters->cores = calloc(1, sizeof(struct core_data));
9764 if (counters->cores == NULL)
9765 goto error;
9766
9767 counters->packages = calloc(1, sizeof(struct pkg_data));
9768 if (counters->packages == NULL)
9769 goto error;
9770
9771 return;
9772 error:
9773 err(1, "calloc counters_1");
9774 }
9775
allocate_counters(struct counters * counters)9776 void allocate_counters(struct counters *counters)
9777 {
9778 int i;
9779 int num_cores = topo.cores_per_pkg * topo.num_packages;
9780
9781 counters->threads = calloc(topo.max_cpu_num + 1, sizeof(struct thread_data));
9782 if (counters->threads == NULL)
9783 goto error;
9784
9785 for (i = 0; i < topo.max_cpu_num + 1; i++)
9786 (counters->threads)[i].cpu_id = -1;
9787
9788 counters->cores = calloc(num_cores, sizeof(struct core_data));
9789 if (counters->cores == NULL)
9790 goto error;
9791
9792 for (i = 0; i < num_cores; i++)
9793 (counters->cores)[i].first_cpu = -1;
9794
9795 counters->packages = calloc(topo.num_packages, sizeof(struct pkg_data));
9796 if (counters->packages == NULL)
9797 goto error;
9798
9799 for (i = 0; i < topo.num_packages; i++)
9800 (counters->packages)[i].first_cpu = -1;
9801
9802 return;
9803 error:
9804 err(1, "calloc counters");
9805 }
9806
9807 /*
9808 * init_counter()
9809 *
9810 * set t->cpu_id, FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE
9811 */
init_counter(struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base,int cpu_id)9812 void init_counter(struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base, int cpu_id)
9813 {
9814 int pkg_id = cpus[cpu_id].package_id;
9815 int node_id = cpus[cpu_id].logical_node_id;
9816 int core_id = cpus[cpu_id].core_id;
9817 struct thread_data *t;
9818 struct core_data *c;
9819
9820 /* Workaround for systems where physical_node_id==-1
9821 * and logical_node_id==(-1 - topo.num_cpus)
9822 */
9823 if (node_id < 0)
9824 node_id = 0;
9825
9826 t = &thread_base[cpu_id];
9827 c = &core_base[GLOBAL_CORE_ID(core_id, pkg_id)];
9828
9829 t->cpu_id = cpu_id;
9830 if (!cpu_is_not_allowed(cpu_id)) {
9831
9832 if (c->first_cpu < 0)
9833 c->first_cpu = t->cpu_id;
9834 if (pkg_base[pkg_id].first_cpu < 0)
9835 pkg_base[pkg_id].first_cpu = t->cpu_id;
9836 }
9837 }
9838
initialize_counters(int cpu_id)9839 int initialize_counters(int cpu_id)
9840 {
9841 init_counter(EVEN_COUNTERS, cpu_id);
9842 init_counter(ODD_COUNTERS, cpu_id);
9843 return 0;
9844 }
9845
allocate_output_buffer()9846 void allocate_output_buffer()
9847 {
9848 output_buffer = calloc(1, (1 + topo.num_cpus) * 2048);
9849 outp = output_buffer;
9850 if (outp == NULL)
9851 err(-1, "calloc output buffer");
9852 }
9853
allocate_fd_percpu(void)9854 void allocate_fd_percpu(void)
9855 {
9856 fd_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9857 if (fd_percpu == NULL)
9858 err(-1, "calloc fd_percpu");
9859 }
9860
allocate_irq_buffers(void)9861 void allocate_irq_buffers(void)
9862 {
9863 irq_column_2_cpu = calloc(topo.num_cpus, sizeof(int));
9864 if (irq_column_2_cpu == NULL)
9865 err(-1, "calloc %d", topo.num_cpus);
9866
9867 irqs_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9868 if (irqs_per_cpu == NULL)
9869 err(-1, "calloc %d IRQ", topo.max_cpu_num + 1);
9870
9871 nmi_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9872 if (nmi_per_cpu == NULL)
9873 err(-1, "calloc %d NMI", topo.max_cpu_num + 1);
9874 }
9875
update_topo(PER_THREAD_PARAMS)9876 int update_topo(PER_THREAD_PARAMS)
9877 {
9878 topo.allowed_cpus++;
9879 if ((int)t->cpu_id == c->first_cpu)
9880 topo.allowed_cores++;
9881 if ((int)t->cpu_id == p->first_cpu)
9882 topo.allowed_packages++;
9883
9884 return 0;
9885 }
9886
topology_update(void)9887 void topology_update(void)
9888 {
9889 topo.allowed_cpus = 0;
9890 topo.allowed_cores = 0;
9891 topo.allowed_packages = 0;
9892 for_all_cpus(update_topo, ODD_COUNTERS);
9893 if (debug)
9894 fprintf(stderr, "allowed_cpus %d allowed_cores %d allowed_packages %d\n", topo.allowed_cpus, topo.allowed_cores, topo.allowed_packages);
9895 }
9896
setup_all_buffers(bool startup)9897 void setup_all_buffers(bool startup)
9898 {
9899 topology_probe(startup);
9900 allocate_irq_buffers();
9901 allocate_fd_percpu();
9902 allocate_counters_1(&average);
9903 allocate_counters(&even);
9904 allocate_counters(&odd);
9905 allocate_output_buffer();
9906 for_all_proc_cpus(initialize_counters);
9907 topology_update();
9908 }
9909
set_master_cpu(void)9910 void set_master_cpu(void)
9911 {
9912 int i;
9913
9914 for (i = 0; i < topo.max_cpu_num + 1; ++i) {
9915 if (cpu_is_not_allowed(i))
9916 continue;
9917 master_cpu = i;
9918 if (debug > 1)
9919 fprintf(outf, "master_cpu = %d\n", master_cpu);
9920 return;
9921 }
9922 err(-ENODEV, "No valid cpus found");
9923 }
9924
has_added_counters(void)9925 bool has_added_counters(void)
9926 {
9927 /*
9928 * It only makes sense to call this after the command line is parsed,
9929 * otherwise sys structure is not populated.
9930 */
9931
9932 return sys.added_core_counters | sys.added_thread_counters | sys.added_package_counters;
9933 }
9934
check_msr_access(void)9935 void check_msr_access(void)
9936 {
9937 check_msr_driver();
9938 check_msr_permission();
9939
9940 if (no_msr)
9941 bic_disable_msr_access();
9942 }
9943
check_perf_access(void)9944 void check_perf_access(void)
9945 {
9946 if (BIC_IS_ENABLED(BIC_IPC))
9947 if (!has_perf_instr_count_access())
9948 no_perf = 1;
9949
9950 if (BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT))
9951 if (!has_perf_llc_access())
9952 no_perf = 1;
9953
9954 if (no_perf)
9955 bic_disable_perf_access();
9956 }
9957
perf_has_hybrid_devices(void)9958 bool perf_has_hybrid_devices(void)
9959 {
9960 /*
9961 * 0: unknown
9962 * 1: has separate perf device for p and e core
9963 * -1: doesn't have separate perf device for p and e core
9964 */
9965 static int cached;
9966
9967 if (cached > 0)
9968 return true;
9969
9970 if (cached < 0)
9971 return false;
9972
9973 if (access("/sys/bus/event_source/devices/cpu_core", F_OK)) {
9974 cached = -1;
9975 return false;
9976 }
9977
9978 if (access("/sys/bus/event_source/devices/cpu_atom", F_OK)) {
9979 cached = -1;
9980 return false;
9981 }
9982
9983 cached = 1;
9984 return true;
9985 }
9986
added_perf_counters_init_(struct perf_counter_info * pinfo)9987 int added_perf_counters_init_(struct perf_counter_info *pinfo)
9988 {
9989 size_t num_domains = 0;
9990 unsigned int next_domain;
9991 bool *domain_visited;
9992 unsigned int perf_type, perf_config;
9993 double perf_scale;
9994 int fd_perf;
9995
9996 if (!pinfo)
9997 return 0;
9998
9999 const size_t max_num_domains = MAX(topo.max_cpu_num + 1, MAX(topo.max_core_id + 1, topo.max_package_id + 1));
10000
10001 domain_visited = calloc(max_num_domains, sizeof(*domain_visited));
10002
10003 while (pinfo) {
10004 switch (pinfo->scope) {
10005 case SCOPE_CPU:
10006 num_domains = topo.max_cpu_num + 1;
10007 break;
10008
10009 case SCOPE_CORE:
10010 num_domains = topo.max_core_id + 1;
10011 break;
10012
10013 case SCOPE_PACKAGE:
10014 num_domains = topo.max_package_id + 1;
10015 break;
10016 }
10017
10018 /* Allocate buffer for file descriptor for each domain. */
10019 pinfo->fd_perf_per_domain = calloc(num_domains, sizeof(*pinfo->fd_perf_per_domain));
10020 if (!pinfo->fd_perf_per_domain)
10021 errx(1, "%s: alloc %s", __func__, "fd_perf_per_domain");
10022
10023 for (size_t i = 0; i < num_domains; ++i)
10024 pinfo->fd_perf_per_domain[i] = -1;
10025
10026 pinfo->num_domains = num_domains;
10027 pinfo->scale = 1.0;
10028
10029 memset(domain_visited, 0, max_num_domains * sizeof(*domain_visited));
10030
10031 for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
10032
10033 next_domain = cpu_to_domain(pinfo, cpu);
10034
10035 assert(next_domain < num_domains);
10036
10037 if (cpu_is_not_allowed(cpu))
10038 continue;
10039
10040 if (domain_visited[next_domain])
10041 continue;
10042
10043 /*
10044 * Intel hybrid platforms expose different perf devices for P and E cores.
10045 * Instead of one, "/sys/bus/event_source/devices/cpu" device, there are
10046 * "/sys/bus/event_source/devices/{cpu_core,cpu_atom}".
10047 *
10048 * This makes it more complicated to the user, because most of the counters
10049 * are available on both and have to be handled manually, otherwise.
10050 *
10051 * Code below, allow user to use the old "cpu" name, which is translated accordingly.
10052 */
10053 const char *perf_device = pinfo->device;
10054
10055 if (strcmp(perf_device, "cpu") == 0 && perf_has_hybrid_devices()) {
10056 switch (cpus[cpu].type) {
10057 case INTEL_PCORE_TYPE:
10058 perf_device = "cpu_core";
10059 break;
10060
10061 case INTEL_ECORE_TYPE:
10062 perf_device = "cpu_atom";
10063 break;
10064
10065 default: /* Don't change, we will probably fail and report a problem soon. */
10066 break;
10067 }
10068 }
10069
10070 perf_type = read_perf_type(perf_device);
10071 if (perf_type == (unsigned int)-1) {
10072 warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "type");
10073 continue;
10074 }
10075
10076 perf_config = read_perf_config(perf_device, pinfo->event);
10077 if (perf_config == (unsigned int)-1) {
10078 warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "config");
10079 continue;
10080 }
10081
10082 /* Scale is not required, some counters just don't have it. */
10083 perf_scale = read_perf_scale(perf_device, pinfo->event);
10084 if (perf_scale == 0.0)
10085 perf_scale = 1.0;
10086
10087 fd_perf = open_perf_counter(cpu, perf_type, perf_config, -1, 0);
10088 if (fd_perf == -1) {
10089 warnx("%s: perf/%s/%s: failed to open counter on cpu%d", __func__, perf_device, pinfo->event, cpu);
10090 continue;
10091 }
10092
10093 domain_visited[next_domain] = 1;
10094 pinfo->fd_perf_per_domain[next_domain] = fd_perf;
10095 pinfo->scale = perf_scale;
10096
10097 if (debug)
10098 fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n", perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]);
10099 }
10100
10101 pinfo = pinfo->next;
10102 }
10103
10104 free(domain_visited);
10105
10106 return 0;
10107 }
10108
added_perf_counters_init(void)10109 void added_perf_counters_init(void)
10110 {
10111 if (added_perf_counters_init_(sys.perf_tp))
10112 errx(1, "%s: %s", __func__, "thread");
10113
10114 if (added_perf_counters_init_(sys.perf_cp))
10115 errx(1, "%s: %s", __func__, "core");
10116
10117 if (added_perf_counters_init_(sys.perf_pp))
10118 errx(1, "%s: %s", __func__, "package");
10119 }
10120
parse_telem_info_file(int fd_dir,const char * info_filename,const char * format,unsigned long * output)10121 int parse_telem_info_file(int fd_dir, const char *info_filename, const char *format, unsigned long *output)
10122 {
10123 int fd_telem_info;
10124 FILE *file_telem_info;
10125 unsigned long value;
10126
10127 fd_telem_info = openat(fd_dir, info_filename, O_RDONLY);
10128 if (fd_telem_info == -1)
10129 return -1;
10130
10131 file_telem_info = fdopen(fd_telem_info, "r");
10132 if (file_telem_info == NULL) {
10133 close(fd_telem_info);
10134 return -1;
10135 }
10136
10137 if (fscanf(file_telem_info, format, &value) != 1) {
10138 fclose(file_telem_info);
10139 return -1;
10140 }
10141
10142 fclose(file_telem_info);
10143
10144 *output = value;
10145
10146 return 0;
10147 }
10148
pmt_mmio_open(unsigned int target_guid)10149 struct pmt_mmio *pmt_mmio_open(unsigned int target_guid)
10150 {
10151 struct pmt_diriter_t pmt_iter;
10152 const struct dirent *entry;
10153 struct stat st;
10154 int fd_telem_dir, fd_pmt;
10155 unsigned long guid, size, offset;
10156 size_t mmap_size;
10157 void *mmio;
10158 struct pmt_mmio *head = NULL, *last = NULL;
10159 struct pmt_mmio *new_pmt = NULL;
10160
10161 if (stat(SYSFS_TELEM_PATH, &st) == -1)
10162 return NULL;
10163
10164 pmt_diriter_init(&pmt_iter);
10165 entry = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH);
10166 if (!entry) {
10167 pmt_diriter_remove(&pmt_iter);
10168 return NULL;
10169 }
10170
10171 for (; entry != NULL; entry = pmt_diriter_next(&pmt_iter)) {
10172 if (fstatat(dirfd(pmt_iter.dir), entry->d_name, &st, 0) == -1)
10173 break;
10174
10175 if (!S_ISDIR(st.st_mode))
10176 continue;
10177
10178 fd_telem_dir = openat(dirfd(pmt_iter.dir), entry->d_name, O_RDONLY);
10179 if (fd_telem_dir == -1)
10180 break;
10181
10182 if (parse_telem_info_file(fd_telem_dir, "guid", "%lx", &guid)) {
10183 close(fd_telem_dir);
10184 break;
10185 }
10186
10187 if (parse_telem_info_file(fd_telem_dir, "size", "%lu", &size)) {
10188 close(fd_telem_dir);
10189 break;
10190 }
10191
10192 if (guid != target_guid) {
10193 close(fd_telem_dir);
10194 continue;
10195 }
10196
10197 if (parse_telem_info_file(fd_telem_dir, "offset", "%lu", &offset)) {
10198 close(fd_telem_dir);
10199 break;
10200 }
10201
10202 assert(offset == 0);
10203
10204 fd_pmt = openat(fd_telem_dir, "telem", O_RDONLY);
10205 if (fd_pmt == -1)
10206 goto loop_cleanup_and_break;
10207
10208 mmap_size = ROUND_UP_TO_PAGE_SIZE(size);
10209 mmio = mmap(0, mmap_size, PROT_READ, MAP_SHARED, fd_pmt, 0);
10210 if (mmio != MAP_FAILED) {
10211 if (debug)
10212 fprintf(stderr, "%s: 0x%lx mmaped at: %p\n", __func__, guid, mmio);
10213
10214 new_pmt = calloc(1, sizeof(*new_pmt));
10215
10216 if (!new_pmt) {
10217 fprintf(stderr, "%s: Failed to allocate pmt_mmio\n", __func__);
10218 exit(1);
10219 }
10220
10221 /*
10222 * Create linked list of mmaped regions,
10223 * but preserve the ordering from sysfs.
10224 * Ordering is important for the user to
10225 * use the seq=%u parameter when adding a counter.
10226 */
10227 new_pmt->guid = guid;
10228 new_pmt->mmio_base = mmio;
10229 new_pmt->pmt_offset = offset;
10230 new_pmt->size = size;
10231 new_pmt->next = pmt_mmios;
10232
10233 if (last)
10234 last->next = new_pmt;
10235 else
10236 head = new_pmt;
10237
10238 last = new_pmt;
10239 }
10240
10241 loop_cleanup_and_break:
10242 close(fd_pmt);
10243 close(fd_telem_dir);
10244 }
10245
10246 pmt_diriter_remove(&pmt_iter);
10247
10248 /*
10249 * If we found something, stick just
10250 * created linked list to the front.
10251 */
10252 if (head)
10253 pmt_mmios = head;
10254
10255 return head;
10256 }
10257
pmt_mmio_find(unsigned int guid)10258 struct pmt_mmio *pmt_mmio_find(unsigned int guid)
10259 {
10260 struct pmt_mmio *pmmio = pmt_mmios;
10261
10262 while (pmmio) {
10263 if (pmmio->guid == guid)
10264 return pmmio;
10265
10266 pmmio = pmmio->next;
10267 }
10268
10269 return NULL;
10270 }
10271
pmt_get_counter_pointer(struct pmt_mmio * pmmio,unsigned long counter_offset)10272 void *pmt_get_counter_pointer(struct pmt_mmio *pmmio, unsigned long counter_offset)
10273 {
10274 char *ret;
10275
10276 /* Get base of mmaped PMT file. */
10277 ret = (char *)pmmio->mmio_base;
10278
10279 /*
10280 * Apply PMT MMIO offset to obtain beginning of the mmaped telemetry data.
10281 * It's not guaranteed that the mmaped memory begins with the telemetry data
10282 * - we might have to apply the offset first.
10283 */
10284 ret += pmmio->pmt_offset;
10285
10286 /* Apply the counter offset to get the address to the mmaped counter. */
10287 ret += counter_offset;
10288
10289 return ret;
10290 }
10291
pmt_add_guid(unsigned int guid,unsigned int seq)10292 struct pmt_mmio *pmt_add_guid(unsigned int guid, unsigned int seq)
10293 {
10294 struct pmt_mmio *ret;
10295
10296 ret = pmt_mmio_find(guid);
10297 if (!ret)
10298 ret = pmt_mmio_open(guid);
10299
10300 while (ret && seq) {
10301 ret = ret->next;
10302 --seq;
10303 }
10304
10305 return ret;
10306 }
10307
10308 enum pmt_open_mode {
10309 PMT_OPEN_TRY, /* Open failure is not an error. */
10310 PMT_OPEN_REQUIRED, /* Open failure is a fatal error. */
10311 };
10312
pmt_find_counter(struct pmt_counter * pcounter,const char * name)10313 struct pmt_counter *pmt_find_counter(struct pmt_counter *pcounter, const char *name)
10314 {
10315 while (pcounter) {
10316 if (strcmp(pcounter->name, name) == 0)
10317 break;
10318
10319 pcounter = pcounter->next;
10320 }
10321
10322 return pcounter;
10323 }
10324
pmt_get_scope_root(enum counter_scope scope)10325 struct pmt_counter **pmt_get_scope_root(enum counter_scope scope)
10326 {
10327 switch (scope) {
10328 case SCOPE_CPU:
10329 return &sys.pmt_tp;
10330 case SCOPE_CORE:
10331 return &sys.pmt_cp;
10332 case SCOPE_PACKAGE:
10333 return &sys.pmt_pp;
10334 }
10335
10336 __builtin_unreachable();
10337 }
10338
pmt_counter_add_domain(struct pmt_counter * pcounter,unsigned long * pmmio,unsigned int domain_id)10339 void pmt_counter_add_domain(struct pmt_counter *pcounter, unsigned long *pmmio, unsigned int domain_id)
10340 {
10341 /* Make sure the new domain fits. */
10342 if (domain_id >= pcounter->num_domains)
10343 pmt_counter_resize(pcounter, domain_id + 1);
10344
10345 assert(pcounter->domains);
10346 assert(domain_id < pcounter->num_domains);
10347
10348 pcounter->domains[domain_id].pcounter = pmmio;
10349 }
10350
pmt_add_counter(unsigned int guid,unsigned int seq,const char * name,enum pmt_datatype type,unsigned int lsb,unsigned int msb,unsigned int offset,enum counter_scope scope,enum counter_format format,unsigned int domain_id,enum pmt_open_mode mode)10351 int pmt_add_counter(unsigned int guid, unsigned int seq, const char *name, enum pmt_datatype type,
10352 unsigned int lsb, unsigned int msb, unsigned int offset, enum counter_scope scope,
10353 enum counter_format format, unsigned int domain_id, enum pmt_open_mode mode)
10354 {
10355 struct pmt_mmio *mmio;
10356 struct pmt_counter *pcounter;
10357 struct pmt_counter **const pmt_root = pmt_get_scope_root(scope);
10358 bool new_counter = false;
10359 int conflict = 0;
10360
10361 if (lsb > msb) {
10362 fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "lsb <= msb", name);
10363 exit(1);
10364 }
10365
10366 if (msb >= 64) {
10367 fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "msb < 64", name);
10368 exit(1);
10369 }
10370
10371 mmio = pmt_add_guid(guid, seq);
10372 if (!mmio) {
10373 if (mode != PMT_OPEN_TRY) {
10374 fprintf(stderr, "%s: failed to map PMT MMIO for guid %x, seq %u\n", __func__, guid, seq);
10375 exit(1);
10376 }
10377
10378 return 1;
10379 }
10380
10381 if (offset >= mmio->size) {
10382 if (mode != PMT_OPEN_TRY) {
10383 fprintf(stderr, "%s: offset %u outside of PMT MMIO size %u\n", __func__, offset, mmio->size);
10384 exit(1);
10385 }
10386
10387 return 1;
10388 }
10389
10390 pcounter = pmt_find_counter(*pmt_root, name);
10391 if (!pcounter) {
10392 pcounter = calloc(1, sizeof(*pcounter));
10393 new_counter = true;
10394 }
10395
10396 if (new_counter) {
10397 strncpy(pcounter->name, name, ARRAY_SIZE(pcounter->name) - 1);
10398 pcounter->type = type;
10399 pcounter->scope = scope;
10400 pcounter->lsb = lsb;
10401 pcounter->msb = msb;
10402 pcounter->format = format;
10403 } else {
10404 conflict += pcounter->type != type;
10405 conflict += pcounter->scope != scope;
10406 conflict += pcounter->lsb != lsb;
10407 conflict += pcounter->msb != msb;
10408 conflict += pcounter->format != format;
10409 }
10410
10411 if (conflict) {
10412 fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n", __func__, name);
10413 exit(1);
10414 }
10415
10416 pmt_counter_add_domain(pcounter, pmt_get_counter_pointer(mmio, offset), domain_id);
10417
10418 if (new_counter) {
10419 pcounter->next = *pmt_root;
10420 *pmt_root = pcounter;
10421 }
10422
10423 return 0;
10424 }
10425
pmt_init(void)10426 void pmt_init(void)
10427 {
10428 int cpu_num;
10429 unsigned long seq, offset, mod_num;
10430
10431 if (BIC_IS_ENABLED(BIC_Diec6)) {
10432 pmt_add_counter(PMT_MTL_DC6_GUID, PMT_MTL_DC6_SEQ, "Die%c6", PMT_TYPE_XTAL_TIME,
10433 PMT_COUNTER_MTL_DC6_LSB, PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET, SCOPE_PACKAGE, FORMAT_DELTA, 0, PMT_OPEN_TRY);
10434 }
10435
10436 if (BIC_IS_ENABLED(BIC_CPU_c1e)) {
10437 seq = 0;
10438 offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE;
10439 mod_num = 0; /* Relative module number for current PMT file. */
10440
10441 /* Open the counter for each CPU. */
10442 for (cpu_num = 0; cpu_num < topo.max_cpu_num;) {
10443
10444 if (cpu_is_not_allowed(cpu_num))
10445 goto next_loop_iter;
10446
10447 /*
10448 * Set the scope to CPU, even though CWF report the counter per module.
10449 * CPUs inside the same module will read from the same location, instead of reporting zeros.
10450 *
10451 * CWF with newer firmware might require a PMT_TYPE_XTAL_TIME intead of PMT_TYPE_TCORE_CLOCK.
10452 */
10453 pmt_add_counter(PMT_CWF_MC1E_GUID, seq, "CPU%c1e", PMT_TYPE_TCORE_CLOCK,
10454 PMT_COUNTER_CWF_MC1E_LSB, PMT_COUNTER_CWF_MC1E_MSB, offset, SCOPE_CPU, FORMAT_DELTA, cpu_num, PMT_OPEN_TRY);
10455
10456 /*
10457 * Rather complex logic for each time we go to the next loop iteration,
10458 * so keep it as a label.
10459 */
10460 next_loop_iter:
10461 /*
10462 * Advance the cpu number and check if we should also advance offset to
10463 * the next counter inside the PMT file.
10464 *
10465 * On Clearwater Forest platform, the counter is reported per module,
10466 * so open the same counter for all of the CPUs inside the module.
10467 * That way, reported table show the correct value for all of the CPUs inside the module,
10468 * instead of zeros.
10469 */
10470 ++cpu_num;
10471 if (cpu_num % PMT_COUNTER_CWF_CPUS_PER_MODULE == 0) {
10472 offset += PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT;
10473 ++mod_num;
10474 }
10475
10476 /*
10477 * There are PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE in each PMT file.
10478 *
10479 * If that number is reached, seq must be incremented to advance to the next file in a sequence.
10480 * Offset inside that file and a module counter has to be reset.
10481 */
10482 if (mod_num == PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE) {
10483 ++seq;
10484 offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE;
10485 mod_num = 0;
10486 }
10487 }
10488 }
10489 }
10490
turbostat_init()10491 void turbostat_init()
10492 {
10493 setup_all_buffers(true);
10494 set_master_cpu();
10495 check_msr_access();
10496 check_perf_access();
10497 process_cpuid();
10498 counter_info_init();
10499 probe_pm_features();
10500 msr_perf_init();
10501 linux_perf_init();
10502 rapl_perf_init();
10503 cstate_perf_init();
10504 perf_llc_init();
10505 perf_l2_init();
10506 added_perf_counters_init();
10507 pmt_init();
10508
10509 for_all_cpus(get_cpu_type, ODD_COUNTERS);
10510 for_all_cpus(get_cpu_type, EVEN_COUNTERS);
10511
10512 if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(master_cpu) != -1)
10513 BIC_PRESENT(BIC_IPC);
10514
10515 /*
10516 * If TSC tweak is needed, but couldn't get it,
10517 * disable more BICs, since it can't be reported accurately.
10518 */
10519 if (platform->enable_tsc_tweak && !has_base_hz) {
10520 CLR_BIC(BIC_Busy, &bic_enabled);
10521 CLR_BIC(BIC_Bzy_MHz, &bic_enabled);
10522 }
10523 }
10524
affinitize_child(void)10525 void affinitize_child(void)
10526 {
10527 /* Prefer cpu_possible_set, if available */
10528 if (sched_setaffinity(0, cpu_possible_setsize, cpu_possible_set)) {
10529 warn("sched_setaffinity cpu_possible_set");
10530
10531 /* Otherwise, allow child to run on same cpu set as turbostat */
10532 if (sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set))
10533 warn("sched_setaffinity cpu_allowed_set");
10534 }
10535 }
10536
fork_it(char ** argv)10537 int fork_it(char **argv)
10538 {
10539 pid_t child_pid;
10540 int status;
10541
10542 snapshot_proc_sysfs_files();
10543 status = for_all_cpus(get_counters, EVEN_COUNTERS);
10544 first_counter_read = 0;
10545 if (status)
10546 exit(status);
10547 gettimeofday(&tv_even, (struct timezone *)NULL);
10548
10549 child_pid = fork();
10550 if (!child_pid) {
10551 /* child */
10552 affinitize_child();
10553 execvp(argv[0], argv);
10554 err(errno, "exec %s", argv[0]);
10555 } else {
10556
10557 /* parent */
10558 if (child_pid == -1)
10559 err(1, "fork");
10560
10561 signal(SIGINT, SIG_IGN);
10562 signal(SIGQUIT, SIG_IGN);
10563 if (waitpid(child_pid, &status, 0) == -1)
10564 err(status, "waitpid");
10565
10566 if (WIFEXITED(status))
10567 status = WEXITSTATUS(status);
10568 }
10569 /*
10570 * n.b. fork_it() does not check for errors from for_all_cpus()
10571 * because re-starting is problematic when forking
10572 */
10573 snapshot_proc_sysfs_files();
10574 for_all_cpus(get_counters, ODD_COUNTERS);
10575 gettimeofday(&tv_odd, (struct timezone *)NULL);
10576 timersub(&tv_odd, &tv_even, &tv_delta);
10577 if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS))
10578 fprintf(outf, "%s: Counter reset detected\n", progname);
10579 delta_platform(&platform_counters_odd, &platform_counters_even);
10580
10581 compute_average(EVEN_COUNTERS);
10582 format_all_counters(EVEN_COUNTERS);
10583
10584 fprintf(outf, "%.6f sec\n", tv_delta.tv_sec + tv_delta.tv_usec / 1000000.0);
10585
10586 flush_output_stderr();
10587
10588 return status;
10589 }
10590
get_and_dump_counters(void)10591 int get_and_dump_counters(void)
10592 {
10593 int status;
10594
10595 snapshot_proc_sysfs_files();
10596 status = for_all_cpus(get_counters, ODD_COUNTERS);
10597 if (status)
10598 return status;
10599
10600 status = for_all_cpus(dump_counters, ODD_COUNTERS);
10601 if (status)
10602 return status;
10603
10604 flush_output_stdout();
10605
10606 return status;
10607 }
10608
print_version()10609 void print_version()
10610 {
10611 fprintf(outf, "turbostat version 2026.04.21 - Len Brown <lenb@kernel.org>\n");
10612 }
10613
10614 #define COMMAND_LINE_SIZE 2048
10615
print_bootcmd(void)10616 void print_bootcmd(void)
10617 {
10618 char bootcmd[COMMAND_LINE_SIZE];
10619 FILE *fp;
10620 int ret;
10621
10622 memset(bootcmd, 0, COMMAND_LINE_SIZE);
10623 fp = fopen("/proc/cmdline", "r");
10624 if (!fp)
10625 return;
10626
10627 ret = fread(bootcmd, sizeof(char), COMMAND_LINE_SIZE - 1, fp);
10628 if (ret) {
10629 bootcmd[ret] = '\0';
10630 /* the last character is already '\n' */
10631 fprintf(outf, "Kernel command line: %s", bootcmd);
10632 }
10633
10634 fclose(fp);
10635 }
10636
find_msrp_by_name(struct msr_counter * head,char * name)10637 struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name)
10638 {
10639 struct msr_counter *mp;
10640
10641 for (mp = head; mp; mp = mp->next) {
10642 if (debug)
10643 fprintf(stderr, "%s: %s %s\n", __func__, name, mp->name);
10644 if (!strcmp(name, mp->name))
10645 return mp;
10646 }
10647 return NULL;
10648 }
10649
add_counter(unsigned int msr_num,char * path,char * name,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format,int flags,int id)10650 int add_counter(unsigned int msr_num, char *path, char *name,
10651 unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int id)
10652 {
10653 struct msr_counter *msrp;
10654
10655 if (no_msr && msr_num)
10656 errx(1, "Requested MSR counter 0x%x, but in --no-msr mode", msr_num);
10657
10658 if (debug)
10659 fprintf(stderr, "%s(msr%d, %s, %s, width%d, scope%d, type%d, format%d, flags%x, id%d)\n",
10660 __func__, msr_num, path, name, width, scope, type, format, flags, id);
10661
10662 switch (scope) {
10663
10664 case SCOPE_CPU:
10665 msrp = find_msrp_by_name(sys.tp, name);
10666 if (msrp) {
10667 if (debug)
10668 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10669 break;
10670 }
10671 if (sys.added_thread_counters++ >= MAX_ADDED_THREAD_COUNTERS) {
10672 warnx("ignoring thread counter %s", name);
10673 return -1;
10674 }
10675 break;
10676 case SCOPE_CORE:
10677 msrp = find_msrp_by_name(sys.cp, name);
10678 if (msrp) {
10679 if (debug)
10680 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10681 break;
10682 }
10683 if (sys.added_core_counters++ >= MAX_ADDED_CORE_COUNTERS) {
10684 warnx("ignoring core counter %s", name);
10685 return -1;
10686 }
10687 break;
10688 case SCOPE_PACKAGE:
10689 msrp = find_msrp_by_name(sys.pp, name);
10690 if (msrp) {
10691 if (debug)
10692 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10693 break;
10694 }
10695 if (sys.added_package_counters++ >= MAX_ADDED_PACKAGE_COUNTERS) {
10696 warnx("ignoring package counter %s", name);
10697 return -1;
10698 }
10699 break;
10700 default:
10701 warnx("ignoring counter %s with unknown scope", name);
10702 return -1;
10703 }
10704
10705 if (msrp == NULL) {
10706 msrp = calloc(1, sizeof(struct msr_counter));
10707 if (msrp == NULL)
10708 err(-1, "calloc msr_counter");
10709
10710 msrp->msr_num = msr_num;
10711 strncpy(msrp->name, name, NAME_BYTES - 1);
10712 msrp->width = width;
10713 msrp->type = type;
10714 msrp->format = format;
10715 msrp->flags = flags;
10716
10717 switch (scope) {
10718 case SCOPE_CPU:
10719 msrp->next = sys.tp;
10720 sys.tp = msrp;
10721 break;
10722 case SCOPE_CORE:
10723 msrp->next = sys.cp;
10724 sys.cp = msrp;
10725 break;
10726 case SCOPE_PACKAGE:
10727 msrp->next = sys.pp;
10728 sys.pp = msrp;
10729 break;
10730 }
10731 }
10732
10733 if (path) {
10734 struct sysfs_path *sp;
10735
10736 sp = calloc(1, sizeof(struct sysfs_path));
10737 if (sp == NULL) {
10738 perror("calloc");
10739 exit(1);
10740 }
10741 strncpy(sp->path, path, PATH_BYTES - 1);
10742 sp->id = id;
10743 sp->next = msrp->sp;
10744 msrp->sp = sp;
10745 }
10746
10747 return 0;
10748 }
10749
10750 /*
10751 * Initialize the fields used for identifying and opening the counter.
10752 *
10753 * Defer the initialization of any runtime buffers for actually reading
10754 * the counters for when we initialize all perf counters, so we can later
10755 * easily call re_initialize().
10756 */
make_perf_counter_info(const char * perf_device,const char * perf_event,const char * name,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format)10757 struct perf_counter_info *make_perf_counter_info(const char *perf_device,
10758 const char *perf_event,
10759 const char *name,
10760 unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format)
10761 {
10762 struct perf_counter_info *pinfo;
10763
10764 pinfo = calloc(1, sizeof(*pinfo));
10765 if (!pinfo)
10766 errx(1, "%s: Failed to allocate %s/%s\n", __func__, perf_device, perf_event);
10767
10768 strncpy(pinfo->device, perf_device, ARRAY_SIZE(pinfo->device) - 1);
10769 strncpy(pinfo->event, perf_event, ARRAY_SIZE(pinfo->event) - 1);
10770
10771 strncpy(pinfo->name, name, ARRAY_SIZE(pinfo->name) - 1);
10772 pinfo->width = width;
10773 pinfo->scope = scope;
10774 pinfo->type = type;
10775 pinfo->format = format;
10776
10777 return pinfo;
10778 }
10779
add_perf_counter(const char * perf_device,const char * perf_event,const char * name_buffer,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format)10780 int add_perf_counter(const char *perf_device, const char *perf_event, const char *name_buffer, unsigned int width,
10781 enum counter_scope scope, enum counter_type type, enum counter_format format)
10782 {
10783 struct perf_counter_info *pinfo;
10784
10785 switch (scope) {
10786 case SCOPE_CPU:
10787 if (sys.added_thread_perf_counters >= MAX_ADDED_THREAD_COUNTERS) {
10788 warnx("ignoring thread counter perf/%s/%s", perf_device, perf_event);
10789 return -1;
10790 }
10791 break;
10792
10793 case SCOPE_CORE:
10794 if (sys.added_core_perf_counters >= MAX_ADDED_CORE_COUNTERS) {
10795 warnx("ignoring core counter perf/%s/%s", perf_device, perf_event);
10796 return -1;
10797 }
10798 break;
10799
10800 case SCOPE_PACKAGE:
10801 if (sys.added_package_perf_counters >= MAX_ADDED_PACKAGE_COUNTERS) {
10802 warnx("ignoring package counter perf/%s/%s", perf_device, perf_event);
10803 return -1;
10804 }
10805 break;
10806 }
10807
10808 pinfo = make_perf_counter_info(perf_device, perf_event, name_buffer, width, scope, type, format);
10809
10810 if (!pinfo)
10811 return -1;
10812
10813 switch (scope) {
10814 case SCOPE_CPU:
10815 pinfo->next = sys.perf_tp;
10816 sys.perf_tp = pinfo;
10817 ++sys.added_thread_perf_counters;
10818 break;
10819
10820 case SCOPE_CORE:
10821 pinfo->next = sys.perf_cp;
10822 sys.perf_cp = pinfo;
10823 ++sys.added_core_perf_counters;
10824 break;
10825
10826 case SCOPE_PACKAGE:
10827 pinfo->next = sys.perf_pp;
10828 sys.perf_pp = pinfo;
10829 ++sys.added_package_perf_counters;
10830 break;
10831 }
10832
10833 // FIXME: we might not have debug here yet
10834 if (debug)
10835 fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n", __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope);
10836
10837 return 0;
10838 }
10839
parse_add_command_msr(char * add_command)10840 void parse_add_command_msr(char *add_command)
10841 {
10842 int msr_num = 0;
10843 char *path = NULL;
10844 char perf_device[PERF_DEV_NAME_BYTES] = "";
10845 char perf_event[PERF_EVT_NAME_BYTES] = "";
10846 char name_buffer[PERF_NAME_BYTES] = "";
10847 int width = 64;
10848 int fail = 0;
10849 enum counter_scope scope = SCOPE_CPU;
10850 enum counter_type type = COUNTER_CYCLES;
10851 enum counter_format format = FORMAT_DELTA;
10852
10853 while (add_command) {
10854
10855 if (sscanf(add_command, "msr0x%x", &msr_num) == 1)
10856 goto next;
10857
10858 if (sscanf(add_command, "msr%d", &msr_num) == 1)
10859 goto next;
10860
10861 BUILD_BUG_ON(ARRAY_SIZE(perf_device) <= 31);
10862 BUILD_BUG_ON(ARRAY_SIZE(perf_event) <= 31);
10863 if (sscanf(add_command, "perf/%31[^/]/%31[^,]", &perf_device[0], &perf_event[0]) == 2)
10864 goto next;
10865
10866 if (*add_command == '/') {
10867 path = add_command;
10868 goto next;
10869 }
10870
10871 if (sscanf(add_command, "u%d", &width) == 1) {
10872 if ((width == 32) || (width == 64))
10873 goto next;
10874 width = 64;
10875 }
10876 if (!strncmp(add_command, "cpu", strlen("cpu"))) {
10877 scope = SCOPE_CPU;
10878 goto next;
10879 }
10880 if (!strncmp(add_command, "core", strlen("core"))) {
10881 scope = SCOPE_CORE;
10882 goto next;
10883 }
10884 if (!strncmp(add_command, "package", strlen("package"))) {
10885 scope = SCOPE_PACKAGE;
10886 goto next;
10887 }
10888 if (!strncmp(add_command, "cycles", strlen("cycles"))) {
10889 type = COUNTER_CYCLES;
10890 goto next;
10891 }
10892 if (!strncmp(add_command, "seconds", strlen("seconds"))) {
10893 type = COUNTER_SECONDS;
10894 goto next;
10895 }
10896 if (!strncmp(add_command, "usec", strlen("usec"))) {
10897 type = COUNTER_USEC;
10898 goto next;
10899 }
10900 if (!strncmp(add_command, "raw", strlen("raw"))) {
10901 format = FORMAT_RAW;
10902 goto next;
10903 }
10904 if (!strncmp(add_command, "average", strlen("average"))) {
10905 format = FORMAT_AVERAGE;
10906 goto next;
10907 }
10908 if (!strncmp(add_command, "delta", strlen("delta"))) {
10909 format = FORMAT_DELTA;
10910 goto next;
10911 }
10912 if (!strncmp(add_command, "percent", strlen("percent"))) {
10913 format = FORMAT_PERCENT;
10914 goto next;
10915 }
10916
10917 BUILD_BUG_ON(ARRAY_SIZE(name_buffer) <= 18);
10918 if (sscanf(add_command, "%18s,%*s", name_buffer) == 1) {
10919 char *eos;
10920
10921 eos = strchr(name_buffer, ',');
10922 if (eos)
10923 *eos = '\0';
10924 goto next;
10925 }
10926
10927 next:
10928 add_command = strchr(add_command, ',');
10929 if (add_command) {
10930 *add_command = '\0';
10931 add_command++;
10932 }
10933
10934 }
10935 if ((msr_num == 0) && (path == NULL) && (perf_device[0] == '\0' || perf_event[0] == '\0')) {
10936 fprintf(stderr, "--add: (msrDDD | msr0xXXX | /path_to_counter | perf/device/event) required\n");
10937 fail++;
10938 }
10939
10940 /* Test for non-empty perf_device and perf_event */
10941 const bool is_perf_counter = perf_device[0] && perf_event[0];
10942
10943 /* generate default column header */
10944 if (*name_buffer == '\0') {
10945 if (is_perf_counter) {
10946 snprintf(name_buffer, ARRAY_SIZE(name_buffer), "perf/%s", perf_event);
10947 } else {
10948 if (width == 32)
10949 sprintf(name_buffer, "M0x%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
10950 else
10951 sprintf(name_buffer, "M0X%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
10952 }
10953 }
10954
10955 if (is_perf_counter) {
10956 if (add_perf_counter(perf_device, perf_event, name_buffer, width, scope, type, format))
10957 fail++;
10958 } else {
10959 if (add_counter(msr_num, path, name_buffer, width, scope, type, format, 0, 0))
10960 fail++;
10961 }
10962
10963 if (fail) {
10964 help();
10965 exit(1);
10966 }
10967 }
10968
starts_with(const char * str,const char * prefix)10969 bool starts_with(const char *str, const char *prefix)
10970 {
10971 return strncmp(prefix, str, strlen(prefix)) == 0;
10972 }
10973
pmt_parse_from_path(const char * target_path,unsigned int * out_guid,unsigned int * out_seq)10974 int pmt_parse_from_path(const char *target_path, unsigned int *out_guid, unsigned int *out_seq)
10975 {
10976 struct pmt_diriter_t pmt_iter;
10977 const struct dirent *dirname;
10978 struct stat stat, target_stat;
10979 int fd_telem_dir = -1;
10980 int fd_target_dir;
10981 unsigned int seq = 0;
10982 unsigned long guid, target_guid;
10983 int ret = -1;
10984
10985 fd_target_dir = open(target_path, O_RDONLY | O_DIRECTORY);
10986 if (fd_target_dir == -1) {
10987 return -1;
10988 }
10989
10990 if (fstat(fd_target_dir, &target_stat) == -1) {
10991 fprintf(stderr, "%s: Failed to stat the target: %s", __func__, strerror(errno));
10992 exit(1);
10993 }
10994
10995 if (parse_telem_info_file(fd_target_dir, "guid", "%lx", &target_guid)) {
10996 fprintf(stderr, "%s: Failed to parse the target guid file: %s", __func__, strerror(errno));
10997 exit(1);
10998 }
10999
11000 close(fd_target_dir);
11001
11002 pmt_diriter_init(&pmt_iter);
11003
11004 for (dirname = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH); dirname != NULL; dirname = pmt_diriter_next(&pmt_iter)) {
11005
11006 fd_telem_dir = openat(dirfd(pmt_iter.dir), dirname->d_name, O_RDONLY | O_DIRECTORY);
11007 if (fd_telem_dir == -1)
11008 continue;
11009
11010 if (parse_telem_info_file(fd_telem_dir, "guid", "%lx", &guid)) {
11011 fprintf(stderr, "%s: Failed to parse the guid file: %s", __func__, strerror(errno));
11012 continue;
11013 }
11014
11015 if (fstat(fd_telem_dir, &stat) == -1) {
11016 fprintf(stderr, "%s: Failed to stat %s directory: %s", __func__, dirname->d_name, strerror(errno));
11017 continue;
11018 }
11019
11020 /*
11021 * If reached the same directory as target, exit the loop.
11022 * Seq has the correct value now.
11023 */
11024 if (stat.st_dev == target_stat.st_dev && stat.st_ino == target_stat.st_ino) {
11025 ret = 0;
11026 break;
11027 }
11028
11029 /*
11030 * If reached directory with the same guid,
11031 * but it's not the target directory yet,
11032 * increment seq and continue the search.
11033 */
11034 if (guid == target_guid)
11035 ++seq;
11036
11037 close(fd_telem_dir);
11038 fd_telem_dir = -1;
11039 }
11040
11041 pmt_diriter_remove(&pmt_iter);
11042
11043 if (fd_telem_dir != -1)
11044 close(fd_telem_dir);
11045
11046 if (!ret) {
11047 *out_guid = target_guid;
11048 *out_seq = seq;
11049 }
11050
11051 return ret;
11052 }
11053
parse_add_command_pmt(char * add_command)11054 void parse_add_command_pmt(char *add_command)
11055 {
11056 char *name = NULL;
11057 char *type_name = NULL;
11058 char *format_name = NULL;
11059 char *direct_path = NULL;
11060 static const char direct_path_prefix[] = "path=";
11061 unsigned int offset;
11062 unsigned int lsb;
11063 unsigned int msb;
11064 unsigned int guid;
11065 unsigned int seq = 0; /* By default, pick first file in a sequence with a given GUID. */
11066 unsigned int domain_id;
11067 enum counter_scope scope = 0;
11068 enum pmt_datatype type = PMT_TYPE_RAW;
11069 enum counter_format format = FORMAT_RAW;
11070 bool has_offset = false;
11071 bool has_lsb = false;
11072 bool has_msb = false;
11073 bool has_format = true; /* Format has a default value. */
11074 bool has_guid = false;
11075 bool has_scope = false;
11076 bool has_type = true; /* Type has a default value. */
11077
11078 /* Consume the "pmt," prefix. */
11079 add_command = strchr(add_command, ',');
11080 if (!add_command) {
11081 help();
11082 exit(1);
11083 }
11084 ++add_command;
11085
11086 while (add_command) {
11087 if (starts_with(add_command, "name=")) {
11088 name = add_command + strlen("name=");
11089 goto next;
11090 }
11091
11092 if (starts_with(add_command, "type=")) {
11093 type_name = add_command + strlen("type=");
11094 goto next;
11095 }
11096
11097 if (starts_with(add_command, "domain=")) {
11098 const size_t prefix_len = strlen("domain=");
11099
11100 if (sscanf(add_command + prefix_len, "cpu%u", &domain_id) == 1) {
11101 scope = SCOPE_CPU;
11102 has_scope = true;
11103 } else if (sscanf(add_command + prefix_len, "core%u", &domain_id) == 1) {
11104 scope = SCOPE_CORE;
11105 has_scope = true;
11106 } else if (sscanf(add_command + prefix_len, "package%u", &domain_id) == 1) {
11107 scope = SCOPE_PACKAGE;
11108 has_scope = true;
11109 }
11110
11111 if (!has_scope) {
11112 printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n", __func__);
11113 exit(1);
11114 }
11115
11116 goto next;
11117 }
11118
11119 if (starts_with(add_command, "format=")) {
11120 format_name = add_command + strlen("format=");
11121 goto next;
11122 }
11123
11124 if (sscanf(add_command, "offset=%u", &offset) == 1) {
11125 has_offset = true;
11126 goto next;
11127 }
11128
11129 if (sscanf(add_command, "lsb=%u", &lsb) == 1) {
11130 has_lsb = true;
11131 goto next;
11132 }
11133
11134 if (sscanf(add_command, "msb=%u", &msb) == 1) {
11135 has_msb = true;
11136 goto next;
11137 }
11138
11139 if (sscanf(add_command, "guid=%x", &guid) == 1) {
11140 has_guid = true;
11141 goto next;
11142 }
11143
11144 if (sscanf(add_command, "seq=%x", &seq) == 1)
11145 goto next;
11146
11147 if (strncmp(add_command, direct_path_prefix, strlen(direct_path_prefix)) == 0) {
11148 direct_path = add_command + strlen(direct_path_prefix);
11149 goto next;
11150 }
11151 next:
11152 add_command = strchr(add_command, ',');
11153 if (add_command) {
11154 *add_command = '\0';
11155 add_command++;
11156 }
11157 }
11158
11159 if (!name) {
11160 printf("%s: missing %s\n", __func__, "name");
11161 exit(1);
11162 }
11163
11164 if (strlen(name) >= PMT_COUNTER_NAME_SIZE_BYTES) {
11165 printf("%s: name has to be at most %d characters long\n", __func__, PMT_COUNTER_NAME_SIZE_BYTES);
11166 exit(1);
11167 }
11168
11169 if (format_name) {
11170 has_format = false;
11171
11172 if (strcmp("raw", format_name) == 0) {
11173 format = FORMAT_RAW;
11174 has_format = true;
11175 }
11176
11177 if (strcmp("average", format_name) == 0) {
11178 format = FORMAT_AVERAGE;
11179 has_format = true;
11180 }
11181
11182 if (strcmp("delta", format_name) == 0) {
11183 format = FORMAT_DELTA;
11184 has_format = true;
11185 }
11186
11187 if (!has_format) {
11188 fprintf(stderr, "%s: Invalid format %s. Expected raw, average or delta\n", __func__, format_name);
11189 exit(1);
11190 }
11191 }
11192
11193 if (type_name) {
11194 has_type = false;
11195
11196 if (strcmp("raw", type_name) == 0) {
11197 type = PMT_TYPE_RAW;
11198 has_type = true;
11199 }
11200
11201 if (strcmp("txtal_time", type_name) == 0) {
11202 type = PMT_TYPE_XTAL_TIME;
11203 has_type = true;
11204 }
11205
11206 if (strcmp("tcore_clock", type_name) == 0) {
11207 type = PMT_TYPE_TCORE_CLOCK;
11208 has_type = true;
11209 }
11210
11211 if (!has_type) {
11212 printf("%s: invalid %s: %s\n", __func__, "type", type_name);
11213 exit(1);
11214 }
11215 }
11216
11217 if (!has_offset) {
11218 printf("%s : missing %s\n", __func__, "offset");
11219 exit(1);
11220 }
11221
11222 if (!has_lsb) {
11223 printf("%s: missing %s\n", __func__, "lsb");
11224 exit(1);
11225 }
11226
11227 if (!has_msb) {
11228 printf("%s: missing %s\n", __func__, "msb");
11229 exit(1);
11230 }
11231
11232 if (direct_path && has_guid) {
11233 printf("%s: path and guid+seq parameters are mutually exclusive\nnotice: passed guid=0x%x and path=%s\n", __func__, guid, direct_path);
11234 exit(1);
11235 }
11236
11237 if (direct_path) {
11238 if (pmt_parse_from_path(direct_path, &guid, &seq)) {
11239 printf("%s: failed to parse PMT file from %s\n", __func__, direct_path);
11240 exit(1);
11241 }
11242
11243 /* GUID was just infered from the direct path. */
11244 has_guid = true;
11245 }
11246
11247 if (!has_guid) {
11248 printf("%s: missing %s\n", __func__, "guid or path");
11249 exit(1);
11250 }
11251
11252 if (!has_scope) {
11253 printf("%s: missing %s\n", __func__, "scope");
11254 exit(1);
11255 }
11256
11257 if (lsb > msb) {
11258 printf("%s: lsb > msb doesn't make sense\n", __func__);
11259 exit(1);
11260 }
11261
11262 pmt_add_counter(guid, seq, name, type, lsb, msb, offset, scope, format, domain_id, PMT_OPEN_REQUIRED);
11263 }
11264
parse_add_command(char * add_command)11265 void parse_add_command(char *add_command)
11266 {
11267 if (strncmp(add_command, "pmt", strlen("pmt")) == 0)
11268 return parse_add_command_pmt(add_command);
11269 return parse_add_command_msr(add_command);
11270 }
11271
is_deferred_add(char * name)11272 int is_deferred_add(char *name)
11273 {
11274 int i;
11275
11276 for (i = 0; i < deferred_add_index; ++i)
11277 if (!strcmp(name, deferred_add_names[i])) {
11278 deferred_add_consumed |= (1 << i);
11279 return 1;
11280 }
11281 return 0;
11282 }
11283
is_deferred_skip(char * name)11284 int is_deferred_skip(char *name)
11285 {
11286 int i;
11287
11288 for (i = 0; i < deferred_skip_index; ++i)
11289 if (!strcmp(name, deferred_skip_names[i])) {
11290 deferred_skip_consumed |= (1 << i);
11291 return 1;
11292 }
11293 return 0;
11294 }
11295
verify_deferred_consumed(void)11296 void verify_deferred_consumed(void)
11297 {
11298 int i;
11299 int fail = 0;
11300
11301 for (i = 0; i < deferred_add_index; ++i) {
11302 if (!(deferred_add_consumed & (1 << i))) {
11303 warnx("Counter '%s' can not be added.", deferred_add_names[i]);
11304 fail++;
11305 }
11306 }
11307 for (i = 0; i < deferred_skip_index; ++i) {
11308 if (!(deferred_skip_consumed & (1 << i))) {
11309 warnx("Counter '%s' can not be skipped.", deferred_skip_names[i]);
11310 fail++;
11311 }
11312 }
11313 if (fail)
11314 exit(-EINVAL);
11315 }
11316
probe_cpuidle_residency(void)11317 void probe_cpuidle_residency(void)
11318 {
11319 char path[64];
11320 char name_buf[16];
11321 FILE *input;
11322 int state;
11323 int min_state = 1024, max_state = 0;
11324 char *sp;
11325
11326 for (state = 10; state >= 0; --state) {
11327
11328 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state);
11329 input = fopen(path, "r");
11330 if (input == NULL)
11331 continue;
11332 if (!fgets(name_buf, sizeof(name_buf), input))
11333 err(1, "%s: failed to read file", path);
11334
11335 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
11336 sp = strchr(name_buf, '-');
11337 if (!sp)
11338 sp = strchrnul(name_buf, '\n');
11339 *sp = '%';
11340 *(sp + 1) = '\0';
11341
11342 remove_underbar(name_buf);
11343
11344 fclose(input);
11345
11346 sprintf(path, "cpuidle/state%d/time", state);
11347
11348 if (!DO_BIC(BIC_pct_idle) && !is_deferred_add(name_buf))
11349 continue;
11350
11351 if (is_deferred_skip(name_buf))
11352 continue;
11353
11354 add_counter(0, path, name_buf, 32, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0);
11355
11356 if (state > max_state)
11357 max_state = state;
11358 if (state < min_state)
11359 min_state = state;
11360 }
11361 }
11362
cpuidle_counter_wanted(char * name)11363 static bool cpuidle_counter_wanted(char *name)
11364 {
11365 if (is_deferred_skip(name))
11366 return false;
11367
11368 return DO_BIC(BIC_cpuidle) || is_deferred_add(name);
11369 }
11370
probe_cpuidle_counts(void)11371 void probe_cpuidle_counts(void)
11372 {
11373 char path[64];
11374 char name_buf[16];
11375 FILE *input;
11376 int state;
11377 int min_state = 1024, max_state = 0;
11378 char *sp;
11379
11380 if (!DO_BIC(BIC_cpuidle) && !deferred_add_index)
11381 return;
11382
11383 for (state = 10; state >= 0; --state) {
11384
11385 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state);
11386 input = fopen(path, "r");
11387 if (input == NULL)
11388 continue;
11389 if (!fgets(name_buf, sizeof(name_buf), input))
11390 err(1, "%s: failed to read file", path);
11391 fclose(input);
11392
11393 remove_underbar(name_buf);
11394
11395 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
11396 sp = strchr(name_buf, '-');
11397 if (!sp)
11398 sp = strchrnul(name_buf, '\n');
11399
11400 /*
11401 * The 'below' sysfs file always contains 0 for the deepest state (largest index),
11402 * do not add it.
11403 */
11404 if (state != max_state) {
11405 /*
11406 * Add 'C1+' for C1, and so on. The 'below' sysfs file always contains 0 for
11407 * the last state, so do not add it.
11408 */
11409 *sp = '+';
11410 *(sp + 1) = '\0';
11411 if (cpuidle_counter_wanted(name_buf)) {
11412 sprintf(path, "cpuidle/state%d/below", state);
11413 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
11414 }
11415 }
11416
11417 *sp = '\0';
11418 if (cpuidle_counter_wanted(name_buf)) {
11419 sprintf(path, "cpuidle/state%d/usage", state);
11420 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
11421 }
11422
11423 /*
11424 * The 'above' sysfs file always contains 0 for the shallowest state (smallest
11425 * index), do not add it.
11426 */
11427 if (state != min_state) {
11428 *sp = '-';
11429 *(sp + 1) = '\0';
11430 if (cpuidle_counter_wanted(name_buf)) {
11431 sprintf(path, "cpuidle/state%d/above", state);
11432 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
11433 }
11434 }
11435 }
11436 }
11437
11438 /*
11439 * parse cpuset with following syntax
11440 * 1,2,4..6,8-10 and set bits in cpu_subset
11441 */
parse_cpu_command(char * optarg)11442 void parse_cpu_command(char *optarg)
11443 {
11444 if (!strcmp(optarg, "core")) {
11445 if (cpu_subset)
11446 goto error;
11447 show_core_only++;
11448 return;
11449 }
11450 if (!strcmp(optarg, "package")) {
11451 if (cpu_subset)
11452 goto error;
11453 show_pkg_only++;
11454 return;
11455 }
11456 if (show_core_only || show_pkg_only)
11457 goto error;
11458
11459 cpu_subset = CPU_ALLOC(CPU_SUBSET_MAXCPUS);
11460 if (cpu_subset == NULL)
11461 err(3, "CPU_ALLOC");
11462 cpu_subset_size = CPU_ALLOC_SIZE(CPU_SUBSET_MAXCPUS);
11463
11464 CPU_ZERO_S(cpu_subset_size, cpu_subset);
11465
11466 if (parse_cpu_str(optarg, cpu_subset, cpu_subset_size))
11467 goto error;
11468
11469 return;
11470
11471 error:
11472 fprintf(stderr, "\"--cpu %s\" malformed\n", optarg);
11473 help();
11474 exit(-1);
11475 }
11476
cmdline(int argc,char ** argv)11477 void cmdline(int argc, char **argv)
11478 {
11479 int opt;
11480 int option_index = 0;
11481 static struct option long_options[] = {
11482 { "add", required_argument, 0, 'a' },
11483 { "cpu", required_argument, 0, 'c' },
11484 { "Dump", no_argument, 0, 'D' },
11485 { "debug", no_argument, 0, 'd' }, /* internal, not documented */
11486 { "enable", required_argument, 0, 'e' },
11487 { "force", no_argument, 0, 'f' },
11488 { "interval", required_argument, 0, 'i' },
11489 { "IPC", no_argument, 0, 'I' },
11490 { "num_iterations", required_argument, 0, 'n' },
11491 { "header_iterations", required_argument, 0, 'N' },
11492 { "help", no_argument, 0, 'h' },
11493 { "hide", required_argument, 0, 'H' }, // meh, -h taken by --help
11494 { "Joules", no_argument, 0, 'J' },
11495 { "list", no_argument, 0, 'l' },
11496 { "out", required_argument, 0, 'o' },
11497 { "quiet", no_argument, 0, 'q' },
11498 { "no-msr", no_argument, 0, 'M' },
11499 { "no-perf", no_argument, 0, 'P' },
11500 { "show", required_argument, 0, 's' },
11501 { "Summary", no_argument, 0, 'S' },
11502 { "TCC", required_argument, 0, 'T' },
11503 { "version", no_argument, 0, 'v' },
11504 { 0, 0, 0, 0 }
11505 };
11506
11507 progname = argv[0];
11508
11509 /*
11510 * Parse some options early, because they may make other options invalid,
11511 * like adding the MSR counter with --add and at the same time using --no-msr.
11512 */
11513 while ((opt = getopt_long_only(argc, argv, "+:MP", long_options, &option_index)) != -1) {
11514 switch (opt) {
11515 case 'M':
11516 no_msr = 1;
11517 break;
11518 case 'P':
11519 no_perf = 1;
11520 break;
11521 default:
11522 break;
11523 }
11524 }
11525 optind = 0;
11526
11527 while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:N:o:qMPST:v", long_options, &option_index)) != -1) {
11528 switch (opt) {
11529 case 'a':
11530 parse_add_command(optarg);
11531 break;
11532 case 'c':
11533 parse_cpu_command(optarg);
11534 break;
11535 case 'D':
11536 dump_only++;
11537 /*
11538 * Force the no_perf early to prevent using it as a source.
11539 * User asks for raw values, but perf returns them relative
11540 * to the opening of the file descriptor.
11541 */
11542 no_perf = 1;
11543 break;
11544 case 'e':
11545 /* --enable specified counter, without clearning existing list */
11546 bic_lookup(&bic_enabled, optarg, SHOW_LIST);
11547 break;
11548 case 'f':
11549 force_load++;
11550 break;
11551 case 'd':
11552 debug++;
11553 bic_set_all(&bic_enabled);
11554 break;
11555 case 'H':
11556 /*
11557 * --hide: do not show those specified
11558 * multiple invocations simply clear more bits in enabled mask
11559 */
11560 {
11561 cpu_set_t bic_group_hide;
11562
11563 BIC_INIT(&bic_group_hide);
11564
11565 bic_lookup(&bic_group_hide, optarg, HIDE_LIST);
11566 bic_clear_bits(&bic_enabled, &bic_group_hide);
11567 }
11568 break;
11569 case 'h':
11570 help();
11571 exit(1);
11572 case 'i':
11573 {
11574 double interval = strtod(optarg, NULL);
11575
11576 if (interval < 0.001) {
11577 fprintf(outf, "interval %f seconds is too small\n", interval);
11578 exit(2);
11579 }
11580
11581 interval_tv.tv_sec = interval_ts.tv_sec = interval;
11582 interval_tv.tv_usec = (interval - interval_tv.tv_sec) * 1000000;
11583 interval_ts.tv_nsec = (interval - interval_ts.tv_sec) * 1000000000;
11584 }
11585 break;
11586 case 'J':
11587 rapl_joules++;
11588 break;
11589 case 'l':
11590 bic_set_all(&bic_enabled);
11591 list_header_only++;
11592 quiet++;
11593 break;
11594 case 'o':
11595 outf = fopen_or_die(optarg, "w");
11596 break;
11597 case 'q':
11598 quiet = 1;
11599 break;
11600 case 'M':
11601 case 'P':
11602 /* Parsed earlier */
11603 break;
11604 case 'n':
11605 num_iterations = strtoul(optarg, NULL, 0);
11606 errno = 0;
11607
11608 if (errno || num_iterations == 0)
11609 errx(-1, "invalid iteration count: %s", optarg);
11610 break;
11611 case 'N':
11612 header_iterations = strtoul(optarg, NULL, 0);
11613 errno = 0;
11614
11615 if (errno || header_iterations == 0)
11616 errx(-1, "invalid header iteration count: %s", optarg);
11617 break;
11618 case 's':
11619 /*
11620 * --show: show only those specified
11621 * The 1st invocation will clear and replace the enabled mask
11622 * subsequent invocations can add to it.
11623 */
11624 if (shown == 0)
11625 BIC_INIT(&bic_enabled);
11626 bic_lookup(&bic_enabled, optarg, SHOW_LIST);
11627 shown = 1;
11628 break;
11629 case 'S':
11630 summary_only++;
11631 break;
11632 case 'T':
11633 tj_max_override = atoi(optarg);
11634 break;
11635 case 'v':
11636 print_version();
11637 exit(0);
11638 break;
11639 default:
11640 help();
11641 exit(1);
11642 }
11643 }
11644 }
11645
set_rlimit(void)11646 void set_rlimit(void)
11647 {
11648 struct rlimit limit;
11649
11650 if (getrlimit(RLIMIT_NOFILE, &limit) < 0)
11651 err(1, "Failed to get rlimit");
11652
11653 if (limit.rlim_max < MAX_NOFILE)
11654 limit.rlim_max = MAX_NOFILE;
11655 if (limit.rlim_cur < MAX_NOFILE)
11656 limit.rlim_cur = MAX_NOFILE;
11657
11658 if (setrlimit(RLIMIT_NOFILE, &limit) < 0)
11659 err(1, "Failed to set rlimit");
11660 }
11661
main(int argc,char ** argv)11662 int main(int argc, char **argv)
11663 {
11664 int fd, ret;
11665
11666 bic_groups_init();
11667
11668 fd = open("/sys/fs/cgroup/cgroup.procs", O_WRONLY);
11669 if (fd < 0)
11670 goto skip_cgroup_setting;
11671
11672 ret = write(fd, "0\n", 2);
11673 if (ret == -1)
11674 perror("Can't update cgroup\n");
11675
11676 close(fd);
11677
11678 skip_cgroup_setting:
11679 outf = stderr;
11680 cmdline(argc, argv);
11681
11682 if (!quiet) {
11683 print_version();
11684 print_bootcmd();
11685 }
11686
11687 probe_cpuidle_residency();
11688 probe_cpuidle_counts();
11689
11690 verify_deferred_consumed();
11691
11692 if (!getuid())
11693 set_rlimit();
11694
11695 turbostat_init();
11696
11697 if (!no_msr)
11698 msr_sum_record();
11699
11700 /* dump counters and exit */
11701 if (dump_only)
11702 return get_and_dump_counters();
11703
11704 /* list header and exit */
11705 if (list_header_only) {
11706 print_header(",");
11707 flush_output_stdout();
11708 return 0;
11709 }
11710
11711 /*
11712 * if any params left, it must be a command to fork
11713 */
11714 if (argc - optind)
11715 return fork_it(argv + optind);
11716 else
11717 turbostat_loop();
11718
11719 return 0;
11720 }
11721