1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * turbostat -- show CPU frequency and C-state residency
4 * on modern Intel and AMD processors.
5 *
6 * Copyright (c) 2025 Intel Corporation.
7 * Len Brown <len.brown@intel.com>
8 */
9
10 #define _GNU_SOURCE
11 #include MSRHEADER
12
13 // copied from arch/x86/include/asm/cpu_device_id.h
14 #define VFM_MODEL_BIT 0
15 #define VFM_FAMILY_BIT 8
16 #define VFM_VENDOR_BIT 16
17 #define VFM_RSVD_BIT 24
18
19 #define VFM_MODEL_MASK GENMASK(VFM_FAMILY_BIT - 1, VFM_MODEL_BIT)
20 #define VFM_FAMILY_MASK GENMASK(VFM_VENDOR_BIT - 1, VFM_FAMILY_BIT)
21 #define VFM_VENDOR_MASK GENMASK(VFM_RSVD_BIT - 1, VFM_VENDOR_BIT)
22
23 #define VFM_MODEL(vfm) (((vfm) & VFM_MODEL_MASK) >> VFM_MODEL_BIT)
24 #define VFM_FAMILY(vfm) (((vfm) & VFM_FAMILY_MASK) >> VFM_FAMILY_BIT)
25 #define VFM_VENDOR(vfm) (((vfm) & VFM_VENDOR_MASK) >> VFM_VENDOR_BIT)
26
27 #define VFM_MAKE(_vendor, _family, _model) ( \
28 ((_model) << VFM_MODEL_BIT) | \
29 ((_family) << VFM_FAMILY_BIT) | \
30 ((_vendor) << VFM_VENDOR_BIT) \
31 )
32 // end copied section
33
34 #define CPUID_LEAF_MODEL_ID 0x1A
35 #define CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT 24
36
37 #define X86_VENDOR_INTEL 0
38
39 #include INTEL_FAMILY_HEADER
40 #include BUILD_BUG_HEADER
41 #include <stdarg.h>
42 #include <stdio.h>
43 #include <err.h>
44 #include <unistd.h>
45 #include <sys/types.h>
46 #include <sys/wait.h>
47 #include <sys/stat.h>
48 #include <sys/select.h>
49 #include <sys/resource.h>
50 #include <sys/mman.h>
51 #include <fcntl.h>
52 #include <signal.h>
53 #include <sys/time.h>
54 #include <stdlib.h>
55 #include <getopt.h>
56 #include <dirent.h>
57 #include <string.h>
58 #include <ctype.h>
59 #include <sched.h>
60 #include <time.h>
61 #include <cpuid.h>
62 #include <sys/capability.h>
63 #include <errno.h>
64 #include <math.h>
65 #include <linux/perf_event.h>
66 #include <asm/unistd.h>
67 #include <stdbool.h>
68 #include <assert.h>
69 #include <linux/kernel.h>
70 #include <limits.h>
71
72 #define UNUSED(x) (void)(x)
73
74 /*
75 * This list matches the column headers, except
76 * 1. built-in only, the sysfs counters are not here -- we learn of those at run-time
77 * 2. Core and CPU are moved to the end, we can't have strings that contain them
78 * matching on them for --show and --hide.
79 */
80
81 /*
82 * buffer size used by sscanf() for added column names
83 * Usually truncated to 7 characters, but also handles 18 columns for raw 64-bit counters
84 */
85 #define NAME_BYTES 20
86 #define PATH_BYTES 128
87 #define PERF_NAME_BYTES 128
88
89 #define MAX_NOFILE 0x8000
90
91 #define COUNTER_KIND_PERF_PREFIX "perf/"
92 #define COUNTER_KIND_PERF_PREFIX_LEN strlen(COUNTER_KIND_PERF_PREFIX)
93 #define PERF_DEV_NAME_BYTES 32
94 #define PERF_EVT_NAME_BYTES 32
95
96 #define INTEL_ECORE_TYPE 0x20
97 #define INTEL_PCORE_TYPE 0x40
98
99 #define ROUND_UP_TO_PAGE_SIZE(n) (((n) + 0x1000UL-1UL) & ~(0x1000UL-1UL))
100
101 enum counter_scope { SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE };
102 enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC, COUNTER_K2M };
103 enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE };
104 enum counter_source { COUNTER_SOURCE_NONE, COUNTER_SOURCE_PERF, COUNTER_SOURCE_MSR };
105
106 struct perf_counter_info {
107 struct perf_counter_info *next;
108
109 /* How to open the counter / What counter it is. */
110 char device[PERF_DEV_NAME_BYTES];
111 char event[PERF_EVT_NAME_BYTES];
112
113 /* How to show/format the counter. */
114 char name[PERF_NAME_BYTES];
115 unsigned int width;
116 enum counter_scope scope;
117 enum counter_type type;
118 enum counter_format format;
119 double scale;
120
121 /* For reading the counter. */
122 int *fd_perf_per_domain;
123 size_t num_domains;
124 };
125
126 struct sysfs_path {
127 char path[PATH_BYTES];
128 int id;
129 struct sysfs_path *next;
130 };
131
132 struct msr_counter {
133 unsigned int msr_num;
134 char name[NAME_BYTES];
135 struct sysfs_path *sp;
136 unsigned int width;
137 enum counter_type type;
138 enum counter_format format;
139 struct msr_counter *next;
140 unsigned int flags;
141 #define FLAGS_HIDE (1 << 0)
142 #define FLAGS_SHOW (1 << 1)
143 #define SYSFS_PERCPU (1 << 1)
144 };
145 static int use_android_msr_path;
146
147 struct msr_counter bic[] = {
148 { 0x0, "usec", NULL, 0, 0, 0, NULL, 0 },
149 { 0x0, "Time_Of_Day_Seconds", NULL, 0, 0, 0, NULL, 0 },
150 { 0x0, "Package", NULL, 0, 0, 0, NULL, 0 },
151 { 0x0, "Node", NULL, 0, 0, 0, NULL, 0 },
152 { 0x0, "Avg_MHz", NULL, 0, 0, 0, NULL, 0 },
153 { 0x0, "Busy%", NULL, 0, 0, 0, NULL, 0 },
154 { 0x0, "Bzy_MHz", NULL, 0, 0, 0, NULL, 0 },
155 { 0x0, "TSC_MHz", NULL, 0, 0, 0, NULL, 0 },
156 { 0x0, "IRQ", NULL, 0, 0, 0, NULL, 0 },
157 { 0x0, "SMI", NULL, 32, 0, FORMAT_DELTA, NULL, 0 },
158 { 0x0, "cpuidle", NULL, 0, 0, 0, NULL, 0 },
159 { 0x0, "CPU%c1", NULL, 0, 0, 0, NULL, 0 },
160 { 0x0, "CPU%c3", NULL, 0, 0, 0, NULL, 0 },
161 { 0x0, "CPU%c6", NULL, 0, 0, 0, NULL, 0 },
162 { 0x0, "CPU%c7", NULL, 0, 0, 0, NULL, 0 },
163 { 0x0, "ThreadC", NULL, 0, 0, 0, NULL, 0 },
164 { 0x0, "CoreTmp", NULL, 0, 0, 0, NULL, 0 },
165 { 0x0, "CoreCnt", NULL, 0, 0, 0, NULL, 0 },
166 { 0x0, "PkgTmp", NULL, 0, 0, 0, NULL, 0 },
167 { 0x0, "GFX%rc6", NULL, 0, 0, 0, NULL, 0 },
168 { 0x0, "GFXMHz", NULL, 0, 0, 0, NULL, 0 },
169 { 0x0, "Pkg%pc2", NULL, 0, 0, 0, NULL, 0 },
170 { 0x0, "Pkg%pc3", NULL, 0, 0, 0, NULL, 0 },
171 { 0x0, "Pkg%pc6", NULL, 0, 0, 0, NULL, 0 },
172 { 0x0, "Pkg%pc7", NULL, 0, 0, 0, NULL, 0 },
173 { 0x0, "Pkg%pc8", NULL, 0, 0, 0, NULL, 0 },
174 { 0x0, "Pkg%pc9", NULL, 0, 0, 0, NULL, 0 },
175 { 0x0, "Pk%pc10", NULL, 0, 0, 0, NULL, 0 },
176 { 0x0, "CPU%LPI", NULL, 0, 0, 0, NULL, 0 },
177 { 0x0, "SYS%LPI", NULL, 0, 0, 0, NULL, 0 },
178 { 0x0, "PkgWatt", NULL, 0, 0, 0, NULL, 0 },
179 { 0x0, "CorWatt", NULL, 0, 0, 0, NULL, 0 },
180 { 0x0, "GFXWatt", NULL, 0, 0, 0, NULL, 0 },
181 { 0x0, "PkgCnt", NULL, 0, 0, 0, NULL, 0 },
182 { 0x0, "RAMWatt", NULL, 0, 0, 0, NULL, 0 },
183 { 0x0, "PKG_%", NULL, 0, 0, 0, NULL, 0 },
184 { 0x0, "RAM_%", NULL, 0, 0, 0, NULL, 0 },
185 { 0x0, "Pkg_J", NULL, 0, 0, 0, NULL, 0 },
186 { 0x0, "Cor_J", NULL, 0, 0, 0, NULL, 0 },
187 { 0x0, "GFX_J", NULL, 0, 0, 0, NULL, 0 },
188 { 0x0, "RAM_J", NULL, 0, 0, 0, NULL, 0 },
189 { 0x0, "Mod%c6", NULL, 0, 0, 0, NULL, 0 },
190 { 0x0, "Totl%C0", NULL, 0, 0, 0, NULL, 0 },
191 { 0x0, "Any%C0", NULL, 0, 0, 0, NULL, 0 },
192 { 0x0, "GFX%C0", NULL, 0, 0, 0, NULL, 0 },
193 { 0x0, "CPUGFX%", NULL, 0, 0, 0, NULL, 0 },
194 { 0x0, "Core", NULL, 0, 0, 0, NULL, 0 },
195 { 0x0, "CPU", NULL, 0, 0, 0, NULL, 0 },
196 { 0x0, "APIC", NULL, 0, 0, 0, NULL, 0 },
197 { 0x0, "X2APIC", NULL, 0, 0, 0, NULL, 0 },
198 { 0x0, "Die", NULL, 0, 0, 0, NULL, 0 },
199 { 0x0, "L3", NULL, 0, 0, 0, NULL, 0 },
200 { 0x0, "GFXAMHz", NULL, 0, 0, 0, NULL, 0 },
201 { 0x0, "IPC", NULL, 0, 0, 0, NULL, 0 },
202 { 0x0, "CoreThr", NULL, 0, 0, 0, NULL, 0 },
203 { 0x0, "UncMHz", NULL, 0, 0, 0, NULL, 0 },
204 { 0x0, "SAM%mc6", NULL, 0, 0, 0, NULL, 0 },
205 { 0x0, "SAMMHz", NULL, 0, 0, 0, NULL, 0 },
206 { 0x0, "SAMAMHz", NULL, 0, 0, 0, NULL, 0 },
207 { 0x0, "Die%c6", NULL, 0, 0, 0, NULL, 0 },
208 { 0x0, "SysWatt", NULL, 0, 0, 0, NULL, 0 },
209 { 0x0, "Sys_J", NULL, 0, 0, 0, NULL, 0 },
210 { 0x0, "NMI", NULL, 0, 0, 0, NULL, 0 },
211 { 0x0, "CPU%c1e", NULL, 0, 0, 0, NULL, 0 },
212 { 0x0, "pct_idle", NULL, 0, 0, 0, NULL, 0 },
213 { 0x0, "LLCkRPS", NULL, 0, 0, 0, NULL, 0 },
214 { 0x0, "LLC%hit", NULL, 0, 0, 0, NULL, 0 },
215 };
216
217 /* n.b. bic_names must match the order in bic[], above */
218 enum bic_names {
219 BIC_USEC,
220 BIC_TOD,
221 BIC_Package,
222 BIC_Node,
223 BIC_Avg_MHz,
224 BIC_Busy,
225 BIC_Bzy_MHz,
226 BIC_TSC_MHz,
227 BIC_IRQ,
228 BIC_SMI,
229 BIC_cpuidle,
230 BIC_CPU_c1,
231 BIC_CPU_c3,
232 BIC_CPU_c6,
233 BIC_CPU_c7,
234 BIC_ThreadC,
235 BIC_CoreTmp,
236 BIC_CoreCnt,
237 BIC_PkgTmp,
238 BIC_GFX_rc6,
239 BIC_GFXMHz,
240 BIC_Pkgpc2,
241 BIC_Pkgpc3,
242 BIC_Pkgpc6,
243 BIC_Pkgpc7,
244 BIC_Pkgpc8,
245 BIC_Pkgpc9,
246 BIC_Pkgpc10,
247 BIC_CPU_LPI,
248 BIC_SYS_LPI,
249 BIC_PkgWatt,
250 BIC_CorWatt,
251 BIC_GFXWatt,
252 BIC_PkgCnt,
253 BIC_RAMWatt,
254 BIC_PKG__,
255 BIC_RAM__,
256 BIC_Pkg_J,
257 BIC_Cor_J,
258 BIC_GFX_J,
259 BIC_RAM_J,
260 BIC_Mod_c6,
261 BIC_Totl_c0,
262 BIC_Any_c0,
263 BIC_GFX_c0,
264 BIC_CPUGFX,
265 BIC_Core,
266 BIC_CPU,
267 BIC_APIC,
268 BIC_X2APIC,
269 BIC_Die,
270 BIC_L3,
271 BIC_GFXACTMHz,
272 BIC_IPC,
273 BIC_CORE_THROT_CNT,
274 BIC_UNCORE_MHZ,
275 BIC_SAM_mc6,
276 BIC_SAMMHz,
277 BIC_SAMACTMHz,
278 BIC_Diec6,
279 BIC_SysWatt,
280 BIC_Sys_J,
281 BIC_NMI,
282 BIC_CPU_c1e,
283 BIC_pct_idle,
284 BIC_LLC_RPS,
285 BIC_LLC_HIT,
286 MAX_BIC
287 };
288
print_bic_set(char * s,cpu_set_t * set)289 void print_bic_set(char *s, cpu_set_t *set)
290 {
291 int i;
292
293 assert(MAX_BIC < CPU_SETSIZE);
294
295 printf("%s:", s);
296
297 for (i = 0; i <= MAX_BIC; ++i) {
298
299 if (CPU_ISSET(i, set)) {
300 assert(i < MAX_BIC);
301 printf(" %s", bic[i].name);
302 }
303 }
304 putchar('\n');
305 }
306
307 static cpu_set_t bic_group_topology;
308 static cpu_set_t bic_group_thermal_pwr;
309 static cpu_set_t bic_group_frequency;
310 static cpu_set_t bic_group_hw_idle;
311 static cpu_set_t bic_group_sw_idle;
312 static cpu_set_t bic_group_idle;
313 static cpu_set_t bic_group_cache;
314 static cpu_set_t bic_group_other;
315 static cpu_set_t bic_group_disabled_by_default;
316 static cpu_set_t bic_enabled;
317 static cpu_set_t bic_present;
318
319 /* modify */
320 #define BIC_INIT(set) CPU_ZERO(set)
321
322 #define SET_BIC(COUNTER_NUMBER, set) CPU_SET(COUNTER_NUMBER, set)
323 #define CLR_BIC(COUNTER_NUMBER, set) CPU_CLR(COUNTER_NUMBER, set)
324
325 #define BIC_PRESENT(COUNTER_NUMBER) SET_BIC(COUNTER_NUMBER, &bic_present)
326 #define BIC_NOT_PRESENT(COUNTER_NUMBER) CPU_CLR(COUNTER_NUMBER, &bic_present)
327
328 /* test */
329 #define BIC_IS_ENABLED(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_enabled)
330 #define DO_BIC_READ(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_present)
331 #define DO_BIC(COUNTER_NUMBER) (CPU_ISSET(COUNTER_NUMBER, &bic_enabled) && CPU_ISSET(COUNTER_NUMBER, &bic_present))
332
bic_set_all(cpu_set_t * set)333 static void bic_set_all(cpu_set_t *set)
334 {
335 int i;
336
337 assert(MAX_BIC < CPU_SETSIZE);
338
339 for (i = 0; i < MAX_BIC; ++i)
340 SET_BIC(i, set);
341 }
342
343 /*
344 * bic_clear_bits()
345 * clear all the bits from "clr" in "dst"
346 */
bic_clear_bits(cpu_set_t * dst,cpu_set_t * clr)347 static void bic_clear_bits(cpu_set_t *dst, cpu_set_t *clr)
348 {
349 int i;
350
351 assert(MAX_BIC < CPU_SETSIZE);
352
353 for (i = 0; i < MAX_BIC; ++i)
354 if (CPU_ISSET(i, clr))
355 CLR_BIC(i, dst);
356 }
357
bic_groups_init(void)358 static void bic_groups_init(void)
359 {
360 BIC_INIT(&bic_group_topology);
361 SET_BIC(BIC_Package, &bic_group_topology);
362 SET_BIC(BIC_Node, &bic_group_topology);
363 SET_BIC(BIC_CoreCnt, &bic_group_topology);
364 SET_BIC(BIC_PkgCnt, &bic_group_topology);
365 SET_BIC(BIC_Core, &bic_group_topology);
366 SET_BIC(BIC_CPU, &bic_group_topology);
367 SET_BIC(BIC_Die, &bic_group_topology);
368 SET_BIC(BIC_L3, &bic_group_topology);
369
370 BIC_INIT(&bic_group_thermal_pwr);
371 SET_BIC(BIC_CoreTmp, &bic_group_thermal_pwr);
372 SET_BIC(BIC_PkgTmp, &bic_group_thermal_pwr);
373 SET_BIC(BIC_PkgWatt, &bic_group_thermal_pwr);
374 SET_BIC(BIC_CorWatt, &bic_group_thermal_pwr);
375 SET_BIC(BIC_GFXWatt, &bic_group_thermal_pwr);
376 SET_BIC(BIC_RAMWatt, &bic_group_thermal_pwr);
377 SET_BIC(BIC_PKG__, &bic_group_thermal_pwr);
378 SET_BIC(BIC_RAM__, &bic_group_thermal_pwr);
379 SET_BIC(BIC_SysWatt, &bic_group_thermal_pwr);
380
381 BIC_INIT(&bic_group_frequency);
382 SET_BIC(BIC_Avg_MHz, &bic_group_frequency);
383 SET_BIC(BIC_Busy, &bic_group_frequency);
384 SET_BIC(BIC_Bzy_MHz, &bic_group_frequency);
385 SET_BIC(BIC_TSC_MHz, &bic_group_frequency);
386 SET_BIC(BIC_GFXMHz, &bic_group_frequency);
387 SET_BIC(BIC_GFXACTMHz, &bic_group_frequency);
388 SET_BIC(BIC_SAMMHz, &bic_group_frequency);
389 SET_BIC(BIC_SAMACTMHz, &bic_group_frequency);
390 SET_BIC(BIC_UNCORE_MHZ, &bic_group_frequency);
391
392 BIC_INIT(&bic_group_hw_idle);
393 SET_BIC(BIC_Busy, &bic_group_hw_idle);
394 SET_BIC(BIC_CPU_c1, &bic_group_hw_idle);
395 SET_BIC(BIC_CPU_c3, &bic_group_hw_idle);
396 SET_BIC(BIC_CPU_c6, &bic_group_hw_idle);
397 SET_BIC(BIC_CPU_c7, &bic_group_hw_idle);
398 SET_BIC(BIC_GFX_rc6, &bic_group_hw_idle);
399 SET_BIC(BIC_Pkgpc2, &bic_group_hw_idle);
400 SET_BIC(BIC_Pkgpc3, &bic_group_hw_idle);
401 SET_BIC(BIC_Pkgpc6, &bic_group_hw_idle);
402 SET_BIC(BIC_Pkgpc7, &bic_group_hw_idle);
403 SET_BIC(BIC_Pkgpc8, &bic_group_hw_idle);
404 SET_BIC(BIC_Pkgpc9, &bic_group_hw_idle);
405 SET_BIC(BIC_Pkgpc10, &bic_group_hw_idle);
406 SET_BIC(BIC_CPU_LPI, &bic_group_hw_idle);
407 SET_BIC(BIC_SYS_LPI, &bic_group_hw_idle);
408 SET_BIC(BIC_Mod_c6, &bic_group_hw_idle);
409 SET_BIC(BIC_Totl_c0, &bic_group_hw_idle);
410 SET_BIC(BIC_Any_c0, &bic_group_hw_idle);
411 SET_BIC(BIC_GFX_c0, &bic_group_hw_idle);
412 SET_BIC(BIC_CPUGFX, &bic_group_hw_idle);
413 SET_BIC(BIC_SAM_mc6, &bic_group_hw_idle);
414 SET_BIC(BIC_Diec6, &bic_group_hw_idle);
415
416 BIC_INIT(&bic_group_sw_idle);
417 SET_BIC(BIC_Busy, &bic_group_sw_idle);
418 SET_BIC(BIC_cpuidle, &bic_group_sw_idle);
419 SET_BIC(BIC_pct_idle, &bic_group_sw_idle);
420
421 BIC_INIT(&bic_group_idle);
422
423 CPU_OR(&bic_group_idle, &bic_group_idle, &bic_group_hw_idle);
424 SET_BIC(BIC_pct_idle, &bic_group_idle);
425
426 BIC_INIT(&bic_group_cache);
427 SET_BIC(BIC_LLC_RPS, &bic_group_cache);
428 SET_BIC(BIC_LLC_HIT, &bic_group_cache);
429
430 BIC_INIT(&bic_group_other);
431 SET_BIC(BIC_IRQ, &bic_group_other);
432 SET_BIC(BIC_NMI, &bic_group_other);
433 SET_BIC(BIC_SMI, &bic_group_other);
434 SET_BIC(BIC_ThreadC, &bic_group_other);
435 SET_BIC(BIC_CoreTmp, &bic_group_other);
436 SET_BIC(BIC_IPC, &bic_group_other);
437
438 BIC_INIT(&bic_group_disabled_by_default);
439 SET_BIC(BIC_USEC, &bic_group_disabled_by_default);
440 SET_BIC(BIC_TOD, &bic_group_disabled_by_default);
441 SET_BIC(BIC_cpuidle, &bic_group_disabled_by_default);
442 SET_BIC(BIC_APIC, &bic_group_disabled_by_default);
443 SET_BIC(BIC_X2APIC, &bic_group_disabled_by_default);
444
445 BIC_INIT(&bic_enabled);
446 bic_set_all(&bic_enabled);
447 bic_clear_bits(&bic_enabled, &bic_group_disabled_by_default);
448
449 BIC_INIT(&bic_present);
450 SET_BIC(BIC_USEC, &bic_present);
451 SET_BIC(BIC_TOD, &bic_present);
452 SET_BIC(BIC_cpuidle, &bic_present);
453 SET_BIC(BIC_APIC, &bic_present);
454 SET_BIC(BIC_X2APIC, &bic_present);
455 SET_BIC(BIC_pct_idle, &bic_present);
456 }
457
458 /*
459 * MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit:
460 * If you change the values, note they are used both in comparisons
461 * (>= PCL__7) and to index pkg_cstate_limit_strings[].
462 */
463 #define PCLUKN 0 /* Unknown */
464 #define PCLRSV 1 /* Reserved */
465 #define PCL__0 2 /* PC0 */
466 #define PCL__1 3 /* PC1 */
467 #define PCL__2 4 /* PC2 */
468 #define PCL__3 5 /* PC3 */
469 #define PCL__4 6 /* PC4 */
470 #define PCL__6 7 /* PC6 */
471 #define PCL_6N 8 /* PC6 No Retention */
472 #define PCL_6R 9 /* PC6 Retention */
473 #define PCL__7 10 /* PC7 */
474 #define PCL_7S 11 /* PC7 Shrink */
475 #define PCL__8 12 /* PC8 */
476 #define PCL__9 13 /* PC9 */
477 #define PCL_10 14 /* PC10 */
478 #define PCLUNL 15 /* Unlimited */
479
480 char *proc_stat = "/proc/stat";
481 FILE *outf;
482 int *fd_percpu;
483 int *fd_instr_count_percpu;
484 int *fd_llc_percpu;
485 struct timeval interval_tv = { 5, 0 };
486 struct timespec interval_ts = { 5, 0 };
487
488 unsigned int num_iterations;
489 unsigned int header_iterations;
490 unsigned int debug;
491 unsigned int quiet;
492 unsigned int shown;
493 unsigned int sums_need_wide_columns;
494 unsigned int rapl_joules;
495 unsigned int valid_rapl_msrs;
496 unsigned int summary_only;
497 unsigned int list_header_only;
498 unsigned int dump_only;
499 unsigned int force_load;
500 unsigned int cpuid_has_aperf_mperf;
501 unsigned int has_aperf_access;
502 unsigned int has_epb;
503 unsigned int has_turbo;
504 unsigned int is_hybrid;
505 unsigned int units = 1000000; /* MHz etc */
506 unsigned int genuine_intel;
507 unsigned int authentic_amd;
508 unsigned int hygon_genuine;
509 unsigned int max_level, max_extended_level;
510 unsigned int has_invariant_tsc;
511 unsigned int aperf_mperf_multiplier = 1;
512 double bclk;
513 double base_hz;
514 unsigned int has_base_hz;
515 double tsc_tweak = 1.0;
516 unsigned int show_pkg_only;
517 unsigned int show_core_only;
518 char *output_buffer, *outp;
519 unsigned int do_dts;
520 unsigned int do_ptm;
521 unsigned int do_ipc;
522 unsigned long long cpuidle_cur_cpu_lpi_us;
523 unsigned long long cpuidle_cur_sys_lpi_us;
524 unsigned int tj_max;
525 unsigned int tj_max_override;
526 double rapl_power_units, rapl_time_units;
527 double rapl_dram_energy_units, rapl_energy_units, rapl_psys_energy_units;
528 double rapl_joule_counter_range;
529 unsigned int crystal_hz;
530 unsigned long long tsc_hz;
531 int base_cpu;
532 unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */
533 /* IA32_HWP_REQUEST, IA32_HWP_STATUS */
534 unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */
535 unsigned int has_hwp_activity_window; /* IA32_HWP_REQUEST[bits 41:32] */
536 unsigned int has_hwp_epp; /* IA32_HWP_REQUEST[bits 31:24] */
537 unsigned int has_hwp_pkg; /* IA32_HWP_REQUEST_PKG */
538 unsigned int first_counter_read = 1;
539
540 static struct timeval procsysfs_tv_begin;
541
542 int ignore_stdin;
543 bool no_msr;
544 bool no_perf;
545
546 enum gfx_sysfs_idx {
547 GFX_rc6,
548 GFX_MHz,
549 GFX_ACTMHz,
550 SAM_mc6,
551 SAM_MHz,
552 SAM_ACTMHz,
553 GFX_MAX
554 };
555
556 struct gfx_sysfs_info {
557 FILE *fp;
558 unsigned int val;
559 unsigned long long val_ull;
560 };
561
562 static struct gfx_sysfs_info gfx_info[GFX_MAX];
563
564 int get_msr(int cpu, off_t offset, unsigned long long *msr);
565 int add_counter(unsigned int msr_num, char *path, char *name,
566 unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int package_num);
567
568 /* Model specific support Start */
569
570 /* List of features that may diverge among different platforms */
571 struct platform_features {
572 bool has_msr_misc_feature_control; /* MSR_MISC_FEATURE_CONTROL */
573 bool has_msr_misc_pwr_mgmt; /* MSR_MISC_PWR_MGMT */
574 bool has_nhm_msrs; /* MSR_PLATFORM_INFO, MSR_IA32_TEMPERATURE_TARGET, MSR_SMI_COUNT, MSR_PKG_CST_CONFIG_CONTROL, MSR_IA32_POWER_CTL, TRL MSRs */
575 bool has_config_tdp; /* MSR_CONFIG_TDP_NOMINAL/LEVEL_1/LEVEL_2/CONTROL, MSR_TURBO_ACTIVATION_RATIO */
576 int bclk_freq; /* CPU base clock */
577 int crystal_freq; /* Crystal clock to use when not available from CPUID.15 */
578 int supported_cstates; /* Core cstates and Package cstates supported */
579 int cst_limit; /* MSR_PKG_CST_CONFIG_CONTROL */
580 bool has_cst_auto_convension; /* AUTOMATIC_CSTATE_CONVERSION bit in MSR_PKG_CST_CONFIG_CONTROL */
581 bool has_irtl_msrs; /* MSR_PKGC3/PKGC6/PKGC7/PKGC8/PKGC9/PKGC10_IRTL */
582 bool has_msr_core_c1_res; /* MSR_CORE_C1_RES */
583 bool has_msr_module_c6_res_ms; /* MSR_MODULE_C6_RES_MS */
584 bool has_msr_c6_demotion_policy_config; /* MSR_CC6_DEMOTION_POLICY_CONFIG/MSR_MC6_DEMOTION_POLICY_CONFIG */
585 bool has_msr_atom_pkg_c6_residency; /* MSR_ATOM_PKG_C6_RESIDENCY */
586 bool has_msr_knl_core_c6_residency; /* MSR_KNL_CORE_C6_RESIDENCY */
587 bool has_ext_cst_msrs; /* MSR_PKG_WEIGHTED_CORE_C0_RES/MSR_PKG_ANY_CORE_C0_RES/MSR_PKG_ANY_GFXE_C0_RES/MSR_PKG_BOTH_CORE_GFXE_C0_RES */
588 bool has_cst_prewake_bit; /* Cstate prewake bit in MSR_IA32_POWER_CTL */
589 int trl_msrs; /* MSR_TURBO_RATIO_LIMIT/LIMIT1/LIMIT2/SECONDARY, Atom TRL MSRs */
590 int plr_msrs; /* MSR_CORE/GFX/RING_PERF_LIMIT_REASONS */
591 int plat_rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */
592 bool has_per_core_rapl; /* Indicates cores energy collection is per-core, not per-package. AMD specific for now */
593 bool has_rapl_divisor; /* Divisor for Energy unit raw value from MSR_RAPL_POWER_UNIT */
594 bool has_fixed_rapl_unit; /* Fixed Energy Unit used for DRAM RAPL Domain */
595 bool has_fixed_rapl_psys_unit; /* Fixed Energy Unit used for PSYS RAPL Domain */
596 int rapl_quirk_tdp; /* Hardcoded TDP value when cannot be retrieved from hardware */
597 int tcc_offset_bits; /* TCC Offset bits in MSR_IA32_TEMPERATURE_TARGET */
598 bool enable_tsc_tweak; /* Use CPU Base freq instead of TSC freq for aperf/mperf counter */
599 bool need_perf_multiplier; /* mperf/aperf multiplier */
600 };
601
602 struct platform_data {
603 unsigned int vfm;
604 const struct platform_features *features;
605 };
606
607 /* For BCLK */
608 enum bclk_freq {
609 BCLK_100MHZ = 1,
610 BCLK_133MHZ,
611 BCLK_SLV,
612 };
613
614 #define SLM_BCLK_FREQS 5
615 double slm_freq_table[SLM_BCLK_FREQS] = { 83.3, 100.0, 133.3, 116.7, 80.0 };
616
slm_bclk(void)617 double slm_bclk(void)
618 {
619 unsigned long long msr = 3;
620 unsigned int i;
621 double freq;
622
623 if (get_msr(base_cpu, MSR_FSB_FREQ, &msr))
624 fprintf(outf, "SLM BCLK: unknown\n");
625
626 i = msr & 0xf;
627 if (i >= SLM_BCLK_FREQS) {
628 fprintf(outf, "SLM BCLK[%d] invalid\n", i);
629 i = 3;
630 }
631 freq = slm_freq_table[i];
632
633 if (!quiet)
634 fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq);
635
636 return freq;
637 }
638
639 /* For Package cstate limit */
640 enum package_cstate_limit {
641 CST_LIMIT_NHM = 1,
642 CST_LIMIT_SNB,
643 CST_LIMIT_HSW,
644 CST_LIMIT_SKX,
645 CST_LIMIT_ICX,
646 CST_LIMIT_SLV,
647 CST_LIMIT_AMT,
648 CST_LIMIT_KNL,
649 CST_LIMIT_GMT,
650 };
651
652 /* For Turbo Ratio Limit MSRs */
653 enum turbo_ratio_limit_msrs {
654 TRL_BASE = BIT(0),
655 TRL_LIMIT1 = BIT(1),
656 TRL_LIMIT2 = BIT(2),
657 TRL_ATOM = BIT(3),
658 TRL_KNL = BIT(4),
659 TRL_CORECOUNT = BIT(5),
660 };
661
662 /* For Perf Limit Reason MSRs */
663 enum perf_limit_reason_msrs {
664 PLR_CORE = BIT(0),
665 PLR_GFX = BIT(1),
666 PLR_RING = BIT(2),
667 };
668
669 /* For RAPL MSRs */
670 enum rapl_msrs {
671 RAPL_PKG_POWER_LIMIT = BIT(0), /* 0x610 MSR_PKG_POWER_LIMIT */
672 RAPL_PKG_ENERGY_STATUS = BIT(1), /* 0x611 MSR_PKG_ENERGY_STATUS */
673 RAPL_PKG_PERF_STATUS = BIT(2), /* 0x613 MSR_PKG_PERF_STATUS */
674 RAPL_PKG_POWER_INFO = BIT(3), /* 0x614 MSR_PKG_POWER_INFO */
675 RAPL_DRAM_POWER_LIMIT = BIT(4), /* 0x618 MSR_DRAM_POWER_LIMIT */
676 RAPL_DRAM_ENERGY_STATUS = BIT(5), /* 0x619 MSR_DRAM_ENERGY_STATUS */
677 RAPL_DRAM_PERF_STATUS = BIT(6), /* 0x61b MSR_DRAM_PERF_STATUS */
678 RAPL_DRAM_POWER_INFO = BIT(7), /* 0x61c MSR_DRAM_POWER_INFO */
679 RAPL_CORE_POWER_LIMIT = BIT(8), /* 0x638 MSR_PP0_POWER_LIMIT */
680 RAPL_CORE_ENERGY_STATUS = BIT(9), /* 0x639 MSR_PP0_ENERGY_STATUS */
681 RAPL_CORE_POLICY = BIT(10), /* 0x63a MSR_PP0_POLICY */
682 RAPL_GFX_POWER_LIMIT = BIT(11), /* 0x640 MSR_PP1_POWER_LIMIT */
683 RAPL_GFX_ENERGY_STATUS = BIT(12), /* 0x641 MSR_PP1_ENERGY_STATUS */
684 RAPL_GFX_POLICY = BIT(13), /* 0x642 MSR_PP1_POLICY */
685 RAPL_AMD_PWR_UNIT = BIT(14), /* 0xc0010299 MSR_AMD_RAPL_POWER_UNIT */
686 RAPL_AMD_CORE_ENERGY_STAT = BIT(15), /* 0xc001029a MSR_AMD_CORE_ENERGY_STATUS */
687 RAPL_AMD_PKG_ENERGY_STAT = BIT(16), /* 0xc001029b MSR_AMD_PKG_ENERGY_STATUS */
688 RAPL_PLATFORM_ENERGY_LIMIT = BIT(17), /* 0x64c MSR_PLATFORM_ENERGY_LIMIT */
689 RAPL_PLATFORM_ENERGY_STATUS = BIT(18), /* 0x64d MSR_PLATFORM_ENERGY_STATUS */
690 };
691
692 #define RAPL_PKG (RAPL_PKG_ENERGY_STATUS | RAPL_PKG_POWER_LIMIT)
693 #define RAPL_DRAM (RAPL_DRAM_ENERGY_STATUS | RAPL_DRAM_POWER_LIMIT)
694 #define RAPL_CORE (RAPL_CORE_ENERGY_STATUS | RAPL_CORE_POWER_LIMIT)
695 #define RAPL_GFX (RAPL_GFX_POWER_LIMIT | RAPL_GFX_ENERGY_STATUS)
696 #define RAPL_PSYS (RAPL_PLATFORM_ENERGY_STATUS | RAPL_PLATFORM_ENERGY_LIMIT)
697
698 #define RAPL_PKG_ALL (RAPL_PKG | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO)
699 #define RAPL_DRAM_ALL (RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_DRAM_POWER_INFO)
700 #define RAPL_CORE_ALL (RAPL_CORE | RAPL_CORE_POLICY)
701 #define RAPL_GFX_ALL (RAPL_GFX | RAPL_GFX_POLICY)
702
703 #define RAPL_AMD_F17H (RAPL_AMD_PWR_UNIT | RAPL_AMD_CORE_ENERGY_STAT | RAPL_AMD_PKG_ENERGY_STAT)
704
705 /* For Cstates */
706 enum cstates {
707 CC1 = BIT(0),
708 CC3 = BIT(1),
709 CC6 = BIT(2),
710 CC7 = BIT(3),
711 PC2 = BIT(4),
712 PC3 = BIT(5),
713 PC6 = BIT(6),
714 PC7 = BIT(7),
715 PC8 = BIT(8),
716 PC9 = BIT(9),
717 PC10 = BIT(10),
718 };
719
720 static const struct platform_features nhm_features = {
721 .has_msr_misc_pwr_mgmt = 1,
722 .has_nhm_msrs = 1,
723 .bclk_freq = BCLK_133MHZ,
724 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
725 .cst_limit = CST_LIMIT_NHM,
726 .trl_msrs = TRL_BASE,
727 };
728
729 static const struct platform_features nhx_features = {
730 .has_msr_misc_pwr_mgmt = 1,
731 .has_nhm_msrs = 1,
732 .bclk_freq = BCLK_133MHZ,
733 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
734 .cst_limit = CST_LIMIT_NHM,
735 };
736
737 static const struct platform_features snb_features = {
738 .has_msr_misc_feature_control = 1,
739 .has_msr_misc_pwr_mgmt = 1,
740 .has_nhm_msrs = 1,
741 .bclk_freq = BCLK_100MHZ,
742 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
743 .cst_limit = CST_LIMIT_SNB,
744 .has_irtl_msrs = 1,
745 .trl_msrs = TRL_BASE,
746 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
747 };
748
749 static const struct platform_features snx_features = {
750 .has_msr_misc_feature_control = 1,
751 .has_msr_misc_pwr_mgmt = 1,
752 .has_nhm_msrs = 1,
753 .bclk_freq = BCLK_100MHZ,
754 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
755 .cst_limit = CST_LIMIT_SNB,
756 .has_irtl_msrs = 1,
757 .trl_msrs = TRL_BASE,
758 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
759 };
760
761 static const struct platform_features ivb_features = {
762 .has_msr_misc_feature_control = 1,
763 .has_msr_misc_pwr_mgmt = 1,
764 .has_nhm_msrs = 1,
765 .has_config_tdp = 1,
766 .bclk_freq = BCLK_100MHZ,
767 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
768 .cst_limit = CST_LIMIT_SNB,
769 .has_irtl_msrs = 1,
770 .trl_msrs = TRL_BASE,
771 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
772 };
773
774 static const struct platform_features ivx_features = {
775 .has_msr_misc_feature_control = 1,
776 .has_msr_misc_pwr_mgmt = 1,
777 .has_nhm_msrs = 1,
778 .bclk_freq = BCLK_100MHZ,
779 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
780 .cst_limit = CST_LIMIT_SNB,
781 .has_irtl_msrs = 1,
782 .trl_msrs = TRL_BASE | TRL_LIMIT1,
783 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
784 };
785
786 static const struct platform_features hsw_features = {
787 .has_msr_misc_feature_control = 1,
788 .has_msr_misc_pwr_mgmt = 1,
789 .has_nhm_msrs = 1,
790 .has_config_tdp = 1,
791 .bclk_freq = BCLK_100MHZ,
792 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
793 .cst_limit = CST_LIMIT_HSW,
794 .has_irtl_msrs = 1,
795 .trl_msrs = TRL_BASE,
796 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
797 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
798 };
799
800 static const struct platform_features hsx_features = {
801 .has_msr_misc_feature_control = 1,
802 .has_msr_misc_pwr_mgmt = 1,
803 .has_nhm_msrs = 1,
804 .has_config_tdp = 1,
805 .bclk_freq = BCLK_100MHZ,
806 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
807 .cst_limit = CST_LIMIT_HSW,
808 .has_irtl_msrs = 1,
809 .trl_msrs = TRL_BASE | TRL_LIMIT1 | TRL_LIMIT2,
810 .plr_msrs = PLR_CORE | PLR_RING,
811 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
812 .has_fixed_rapl_unit = 1,
813 };
814
815 static const struct platform_features hswl_features = {
816 .has_msr_misc_feature_control = 1,
817 .has_msr_misc_pwr_mgmt = 1,
818 .has_nhm_msrs = 1,
819 .has_config_tdp = 1,
820 .bclk_freq = BCLK_100MHZ,
821 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
822 .cst_limit = CST_LIMIT_HSW,
823 .has_irtl_msrs = 1,
824 .trl_msrs = TRL_BASE,
825 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
826 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
827 };
828
829 static const struct platform_features hswg_features = {
830 .has_msr_misc_feature_control = 1,
831 .has_msr_misc_pwr_mgmt = 1,
832 .has_nhm_msrs = 1,
833 .has_config_tdp = 1,
834 .bclk_freq = BCLK_100MHZ,
835 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
836 .cst_limit = CST_LIMIT_HSW,
837 .has_irtl_msrs = 1,
838 .trl_msrs = TRL_BASE,
839 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
840 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
841 };
842
843 static const struct platform_features bdw_features = {
844 .has_msr_misc_feature_control = 1,
845 .has_msr_misc_pwr_mgmt = 1,
846 .has_nhm_msrs = 1,
847 .has_config_tdp = 1,
848 .bclk_freq = BCLK_100MHZ,
849 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
850 .cst_limit = CST_LIMIT_HSW,
851 .has_irtl_msrs = 1,
852 .trl_msrs = TRL_BASE,
853 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
854 };
855
856 static const struct platform_features bdwg_features = {
857 .has_msr_misc_feature_control = 1,
858 .has_msr_misc_pwr_mgmt = 1,
859 .has_nhm_msrs = 1,
860 .has_config_tdp = 1,
861 .bclk_freq = BCLK_100MHZ,
862 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
863 .cst_limit = CST_LIMIT_HSW,
864 .has_irtl_msrs = 1,
865 .trl_msrs = TRL_BASE,
866 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
867 };
868
869 static const struct platform_features bdx_features = {
870 .has_msr_misc_feature_control = 1,
871 .has_msr_misc_pwr_mgmt = 1,
872 .has_nhm_msrs = 1,
873 .has_config_tdp = 1,
874 .bclk_freq = BCLK_100MHZ,
875 .supported_cstates = CC1 | CC3 | CC6 | PC2 | PC3 | PC6,
876 .cst_limit = CST_LIMIT_HSW,
877 .has_irtl_msrs = 1,
878 .has_cst_auto_convension = 1,
879 .trl_msrs = TRL_BASE,
880 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
881 .has_fixed_rapl_unit = 1,
882 };
883
884 static const struct platform_features skl_features = {
885 .has_msr_misc_feature_control = 1,
886 .has_msr_misc_pwr_mgmt = 1,
887 .has_nhm_msrs = 1,
888 .has_config_tdp = 1,
889 .bclk_freq = BCLK_100MHZ,
890 .crystal_freq = 24000000,
891 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
892 .cst_limit = CST_LIMIT_HSW,
893 .has_irtl_msrs = 1,
894 .has_ext_cst_msrs = 1,
895 .trl_msrs = TRL_BASE,
896 .tcc_offset_bits = 6,
897 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS,
898 .enable_tsc_tweak = 1,
899 };
900
901 static const struct platform_features cnl_features = {
902 .has_msr_misc_feature_control = 1,
903 .has_msr_misc_pwr_mgmt = 1,
904 .has_nhm_msrs = 1,
905 .has_config_tdp = 1,
906 .bclk_freq = BCLK_100MHZ,
907 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
908 .cst_limit = CST_LIMIT_HSW,
909 .has_irtl_msrs = 1,
910 .has_msr_core_c1_res = 1,
911 .has_ext_cst_msrs = 1,
912 .trl_msrs = TRL_BASE,
913 .tcc_offset_bits = 6,
914 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS,
915 .enable_tsc_tweak = 1,
916 };
917
918 /* Copied from cnl_features, with PC7/PC9 removed */
919 static const struct platform_features adl_features = {
920 .has_msr_misc_feature_control = cnl_features.has_msr_misc_feature_control,
921 .has_msr_misc_pwr_mgmt = cnl_features.has_msr_misc_pwr_mgmt,
922 .has_nhm_msrs = cnl_features.has_nhm_msrs,
923 .has_config_tdp = cnl_features.has_config_tdp,
924 .bclk_freq = cnl_features.bclk_freq,
925 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC8 | PC10,
926 .cst_limit = cnl_features.cst_limit,
927 .has_irtl_msrs = cnl_features.has_irtl_msrs,
928 .has_msr_core_c1_res = cnl_features.has_msr_core_c1_res,
929 .has_ext_cst_msrs = cnl_features.has_ext_cst_msrs,
930 .trl_msrs = cnl_features.trl_msrs,
931 .tcc_offset_bits = cnl_features.tcc_offset_bits,
932 .plat_rapl_msrs = cnl_features.plat_rapl_msrs,
933 .enable_tsc_tweak = cnl_features.enable_tsc_tweak,
934 };
935
936 /* Copied from adl_features, with PC3/PC8 removed */
937 static const struct platform_features lnl_features = {
938 .has_msr_misc_feature_control = adl_features.has_msr_misc_feature_control,
939 .has_msr_misc_pwr_mgmt = adl_features.has_msr_misc_pwr_mgmt,
940 .has_nhm_msrs = adl_features.has_nhm_msrs,
941 .has_config_tdp = adl_features.has_config_tdp,
942 .bclk_freq = adl_features.bclk_freq,
943 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC6 | PC10,
944 .cst_limit = adl_features.cst_limit,
945 .has_irtl_msrs = adl_features.has_irtl_msrs,
946 .has_msr_core_c1_res = adl_features.has_msr_core_c1_res,
947 .has_ext_cst_msrs = adl_features.has_ext_cst_msrs,
948 .trl_msrs = adl_features.trl_msrs,
949 .tcc_offset_bits = adl_features.tcc_offset_bits,
950 .plat_rapl_msrs = adl_features.plat_rapl_msrs,
951 .enable_tsc_tweak = adl_features.enable_tsc_tweak,
952 };
953
954 static const struct platform_features skx_features = {
955 .has_msr_misc_feature_control = 1,
956 .has_msr_misc_pwr_mgmt = 1,
957 .has_nhm_msrs = 1,
958 .has_config_tdp = 1,
959 .bclk_freq = BCLK_100MHZ,
960 .supported_cstates = CC1 | CC6 | PC2 | PC6,
961 .cst_limit = CST_LIMIT_SKX,
962 .has_irtl_msrs = 1,
963 .has_cst_auto_convension = 1,
964 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
965 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
966 .has_fixed_rapl_unit = 1,
967 };
968
969 static const struct platform_features icx_features = {
970 .has_msr_misc_feature_control = 1,
971 .has_msr_misc_pwr_mgmt = 1,
972 .has_nhm_msrs = 1,
973 .has_config_tdp = 1,
974 .bclk_freq = BCLK_100MHZ,
975 .supported_cstates = CC1 | CC6 | PC2 | PC6,
976 .cst_limit = CST_LIMIT_ICX,
977 .has_msr_core_c1_res = 1,
978 .has_irtl_msrs = 1,
979 .has_cst_prewake_bit = 1,
980 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
981 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
982 .has_fixed_rapl_unit = 1,
983 };
984
985 static const struct platform_features spr_features = {
986 .has_msr_misc_feature_control = 1,
987 .has_msr_misc_pwr_mgmt = 1,
988 .has_nhm_msrs = 1,
989 .has_config_tdp = 1,
990 .bclk_freq = BCLK_100MHZ,
991 .supported_cstates = CC1 | CC6 | PC2 | PC6,
992 .cst_limit = CST_LIMIT_SKX,
993 .has_msr_core_c1_res = 1,
994 .has_irtl_msrs = 1,
995 .has_cst_prewake_bit = 1,
996 .has_fixed_rapl_psys_unit = 1,
997 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
998 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
999 };
1000
1001 static const struct platform_features dmr_features = {
1002 .has_msr_misc_feature_control = spr_features.has_msr_misc_feature_control,
1003 .has_msr_misc_pwr_mgmt = spr_features.has_msr_misc_pwr_mgmt,
1004 .has_nhm_msrs = spr_features.has_nhm_msrs,
1005 .bclk_freq = spr_features.bclk_freq,
1006 .supported_cstates = spr_features.supported_cstates,
1007 .cst_limit = spr_features.cst_limit,
1008 .has_msr_core_c1_res = spr_features.has_msr_core_c1_res,
1009 .has_cst_prewake_bit = spr_features.has_cst_prewake_bit,
1010 .has_fixed_rapl_psys_unit = spr_features.has_fixed_rapl_psys_unit,
1011 .trl_msrs = spr_features.trl_msrs,
1012 .has_msr_module_c6_res_ms = 1, /* DMR has Dual-Core-Module and MC6 MSR */
1013 .plat_rapl_msrs = 0, /* DMR does not have RAPL MSRs */
1014 .plr_msrs = 0, /* DMR does not have PLR MSRs */
1015 .has_irtl_msrs = 0, /* DMR does not have IRTL MSRs */
1016 .has_config_tdp = 0, /* DMR does not have CTDP MSRs */
1017 };
1018
1019 static const struct platform_features srf_features = {
1020 .has_msr_misc_feature_control = 1,
1021 .has_msr_misc_pwr_mgmt = 1,
1022 .has_nhm_msrs = 1,
1023 .has_config_tdp = 1,
1024 .bclk_freq = BCLK_100MHZ,
1025 .supported_cstates = CC1 | CC6 | PC2 | PC6,
1026 .cst_limit = CST_LIMIT_SKX,
1027 .has_msr_core_c1_res = 1,
1028 .has_msr_module_c6_res_ms = 1,
1029 .has_irtl_msrs = 1,
1030 .has_cst_prewake_bit = 1,
1031 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1032 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1033 };
1034
1035 static const struct platform_features grr_features = {
1036 .has_msr_misc_feature_control = 1,
1037 .has_msr_misc_pwr_mgmt = 1,
1038 .has_nhm_msrs = 1,
1039 .has_config_tdp = 1,
1040 .bclk_freq = BCLK_100MHZ,
1041 .supported_cstates = CC1 | CC6,
1042 .cst_limit = CST_LIMIT_SKX,
1043 .has_msr_core_c1_res = 1,
1044 .has_msr_module_c6_res_ms = 1,
1045 .has_irtl_msrs = 1,
1046 .has_cst_prewake_bit = 1,
1047 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1048 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1049 };
1050
1051 static const struct platform_features slv_features = {
1052 .has_nhm_msrs = 1,
1053 .bclk_freq = BCLK_SLV,
1054 .supported_cstates = CC1 | CC6 | PC6,
1055 .cst_limit = CST_LIMIT_SLV,
1056 .has_msr_core_c1_res = 1,
1057 .has_msr_module_c6_res_ms = 1,
1058 .has_msr_c6_demotion_policy_config = 1,
1059 .has_msr_atom_pkg_c6_residency = 1,
1060 .trl_msrs = TRL_ATOM,
1061 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE,
1062 .has_rapl_divisor = 1,
1063 .rapl_quirk_tdp = 30,
1064 };
1065
1066 static const struct platform_features slvd_features = {
1067 .has_msr_misc_pwr_mgmt = 1,
1068 .has_nhm_msrs = 1,
1069 .bclk_freq = BCLK_SLV,
1070 .supported_cstates = CC1 | CC6 | PC3 | PC6,
1071 .cst_limit = CST_LIMIT_SLV,
1072 .has_msr_atom_pkg_c6_residency = 1,
1073 .trl_msrs = TRL_BASE,
1074 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE,
1075 .rapl_quirk_tdp = 30,
1076 };
1077
1078 static const struct platform_features amt_features = {
1079 .has_nhm_msrs = 1,
1080 .bclk_freq = BCLK_133MHZ,
1081 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
1082 .cst_limit = CST_LIMIT_AMT,
1083 .trl_msrs = TRL_BASE,
1084 };
1085
1086 static const struct platform_features gmt_features = {
1087 .has_msr_misc_pwr_mgmt = 1,
1088 .has_nhm_msrs = 1,
1089 .bclk_freq = BCLK_100MHZ,
1090 .crystal_freq = 19200000,
1091 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1092 .cst_limit = CST_LIMIT_GMT,
1093 .has_irtl_msrs = 1,
1094 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1095 .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
1096 };
1097
1098 static const struct platform_features gmtd_features = {
1099 .has_msr_misc_pwr_mgmt = 1,
1100 .has_nhm_msrs = 1,
1101 .bclk_freq = BCLK_100MHZ,
1102 .crystal_freq = 25000000,
1103 .supported_cstates = CC1 | CC6 | PC2 | PC6,
1104 .cst_limit = CST_LIMIT_GMT,
1105 .has_irtl_msrs = 1,
1106 .has_msr_core_c1_res = 1,
1107 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1108 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS,
1109 };
1110
1111 static const struct platform_features gmtp_features = {
1112 .has_msr_misc_pwr_mgmt = 1,
1113 .has_nhm_msrs = 1,
1114 .bclk_freq = BCLK_100MHZ,
1115 .crystal_freq = 19200000,
1116 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1117 .cst_limit = CST_LIMIT_GMT,
1118 .has_irtl_msrs = 1,
1119 .trl_msrs = TRL_BASE,
1120 .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
1121 };
1122
1123 static const struct platform_features tmt_features = {
1124 .has_msr_misc_pwr_mgmt = 1,
1125 .has_nhm_msrs = 1,
1126 .bclk_freq = BCLK_100MHZ,
1127 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1128 .cst_limit = CST_LIMIT_GMT,
1129 .has_irtl_msrs = 1,
1130 .trl_msrs = TRL_BASE,
1131 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX,
1132 .enable_tsc_tweak = 1,
1133 };
1134
1135 static const struct platform_features tmtd_features = {
1136 .has_msr_misc_pwr_mgmt = 1,
1137 .has_nhm_msrs = 1,
1138 .bclk_freq = BCLK_100MHZ,
1139 .supported_cstates = CC1 | CC6,
1140 .cst_limit = CST_LIMIT_GMT,
1141 .has_irtl_msrs = 1,
1142 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1143 .plat_rapl_msrs = RAPL_PKG_ALL,
1144 };
1145
1146 static const struct platform_features knl_features = {
1147 .has_msr_misc_pwr_mgmt = 1,
1148 .has_nhm_msrs = 1,
1149 .has_config_tdp = 1,
1150 .bclk_freq = BCLK_100MHZ,
1151 .supported_cstates = CC1 | CC6 | PC3 | PC6,
1152 .cst_limit = CST_LIMIT_KNL,
1153 .has_msr_knl_core_c6_residency = 1,
1154 .trl_msrs = TRL_KNL,
1155 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
1156 .has_fixed_rapl_unit = 1,
1157 .need_perf_multiplier = 1,
1158 };
1159
1160 static const struct platform_features default_features = {
1161 };
1162
1163 static const struct platform_features amd_features_with_rapl = {
1164 .plat_rapl_msrs = RAPL_AMD_F17H,
1165 .has_per_core_rapl = 1,
1166 .rapl_quirk_tdp = 280, /* This is the max stock TDP of HEDT/Server Fam17h+ chips */
1167 };
1168
1169 static const struct platform_data turbostat_pdata[] = {
1170 { INTEL_NEHALEM, &nhm_features },
1171 { INTEL_NEHALEM_G, &nhm_features },
1172 { INTEL_NEHALEM_EP, &nhm_features },
1173 { INTEL_NEHALEM_EX, &nhx_features },
1174 { INTEL_WESTMERE, &nhm_features },
1175 { INTEL_WESTMERE_EP, &nhm_features },
1176 { INTEL_WESTMERE_EX, &nhx_features },
1177 { INTEL_SANDYBRIDGE, &snb_features },
1178 { INTEL_SANDYBRIDGE_X, &snx_features },
1179 { INTEL_IVYBRIDGE, &ivb_features },
1180 { INTEL_IVYBRIDGE_X, &ivx_features },
1181 { INTEL_HASWELL, &hsw_features },
1182 { INTEL_HASWELL_X, &hsx_features },
1183 { INTEL_HASWELL_L, &hswl_features },
1184 { INTEL_HASWELL_G, &hswg_features },
1185 { INTEL_BROADWELL, &bdw_features },
1186 { INTEL_BROADWELL_G, &bdwg_features },
1187 { INTEL_BROADWELL_X, &bdx_features },
1188 { INTEL_BROADWELL_D, &bdx_features },
1189 { INTEL_SKYLAKE_L, &skl_features },
1190 { INTEL_SKYLAKE, &skl_features },
1191 { INTEL_SKYLAKE_X, &skx_features },
1192 { INTEL_KABYLAKE_L, &skl_features },
1193 { INTEL_KABYLAKE, &skl_features },
1194 { INTEL_COMETLAKE, &skl_features },
1195 { INTEL_COMETLAKE_L, &skl_features },
1196 { INTEL_CANNONLAKE_L, &cnl_features },
1197 { INTEL_ICELAKE_X, &icx_features },
1198 { INTEL_ICELAKE_D, &icx_features },
1199 { INTEL_ICELAKE_L, &cnl_features },
1200 { INTEL_ICELAKE_NNPI, &cnl_features },
1201 { INTEL_ROCKETLAKE, &cnl_features },
1202 { INTEL_TIGERLAKE_L, &cnl_features },
1203 { INTEL_TIGERLAKE, &cnl_features },
1204 { INTEL_SAPPHIRERAPIDS_X, &spr_features },
1205 { INTEL_EMERALDRAPIDS_X, &spr_features },
1206 { INTEL_GRANITERAPIDS_X, &spr_features },
1207 { INTEL_GRANITERAPIDS_D, &spr_features },
1208 { INTEL_DIAMONDRAPIDS_X, &dmr_features },
1209 { INTEL_LAKEFIELD, &cnl_features },
1210 { INTEL_ALDERLAKE, &adl_features },
1211 { INTEL_ALDERLAKE_L, &adl_features },
1212 { INTEL_RAPTORLAKE, &adl_features },
1213 { INTEL_RAPTORLAKE_P, &adl_features },
1214 { INTEL_RAPTORLAKE_S, &adl_features },
1215 { INTEL_BARTLETTLAKE, &adl_features },
1216 { INTEL_METEORLAKE, &adl_features },
1217 { INTEL_METEORLAKE_L, &adl_features },
1218 { INTEL_ARROWLAKE_H, &adl_features },
1219 { INTEL_ARROWLAKE_U, &adl_features },
1220 { INTEL_ARROWLAKE, &adl_features },
1221 { INTEL_LUNARLAKE_M, &lnl_features },
1222 { INTEL_PANTHERLAKE_L, &lnl_features },
1223 { INTEL_NOVALAKE, &lnl_features },
1224 { INTEL_NOVALAKE_L, &lnl_features },
1225 { INTEL_WILDCATLAKE_L, &lnl_features },
1226 { INTEL_ATOM_SILVERMONT, &slv_features },
1227 { INTEL_ATOM_SILVERMONT_D, &slvd_features },
1228 { INTEL_ATOM_AIRMONT, &amt_features },
1229 { INTEL_ATOM_GOLDMONT, &gmt_features },
1230 { INTEL_ATOM_GOLDMONT_D, &gmtd_features },
1231 { INTEL_ATOM_GOLDMONT_PLUS, &gmtp_features },
1232 { INTEL_ATOM_TREMONT_D, &tmtd_features },
1233 { INTEL_ATOM_TREMONT, &tmt_features },
1234 { INTEL_ATOM_TREMONT_L, &tmt_features },
1235 { INTEL_ATOM_GRACEMONT, &adl_features },
1236 { INTEL_ATOM_CRESTMONT_X, &srf_features },
1237 { INTEL_ATOM_CRESTMONT, &grr_features },
1238 { INTEL_ATOM_DARKMONT_X, &srf_features },
1239 { INTEL_XEON_PHI_KNL, &knl_features },
1240 { INTEL_XEON_PHI_KNM, &knl_features },
1241 /*
1242 * Missing support for
1243 * INTEL_ICELAKE
1244 * INTEL_ATOM_SILVERMONT_MID
1245 * INTEL_ATOM_SILVERMONT_MID2
1246 * INTEL_ATOM_AIRMONT_NP
1247 */
1248 { 0, NULL },
1249 };
1250
1251 static const struct platform_features *platform;
1252
probe_platform_features(unsigned int family,unsigned int model)1253 void probe_platform_features(unsigned int family, unsigned int model)
1254 {
1255 int i;
1256
1257 if (authentic_amd || hygon_genuine) {
1258 /* fallback to default features on unsupported models */
1259 force_load++;
1260 if (max_extended_level >= 0x80000007) {
1261 unsigned int eax, ebx, ecx, edx;
1262
1263 __cpuid(0x80000007, eax, ebx, ecx, edx);
1264 /* RAPL (Fam 17h+) */
1265 if ((edx & (1 << 14)) && family >= 0x17)
1266 platform = &amd_features_with_rapl;
1267 }
1268 goto end;
1269 }
1270
1271 if (!genuine_intel)
1272 goto end;
1273
1274 for (i = 0; turbostat_pdata[i].features; i++) {
1275 if (VFM_FAMILY(turbostat_pdata[i].vfm) == family && VFM_MODEL(turbostat_pdata[i].vfm) == model) {
1276 platform = turbostat_pdata[i].features;
1277 return;
1278 }
1279 }
1280
1281 end:
1282 if (force_load && !platform) {
1283 fprintf(outf, "Forced to run on unsupported platform!\n");
1284 platform = &default_features;
1285 }
1286
1287 if (platform)
1288 return;
1289
1290 fprintf(stderr, "Unsupported platform detected.\n\tSee RUN THE LATEST VERSION on turbostat(8)\n");
1291 exit(1);
1292 }
1293
1294 /* Model specific support End */
1295
1296 #define TJMAX_DEFAULT 100
1297
1298 /* MSRs that are not yet in the kernel-provided header. */
1299 #define MSR_RAPL_PWR_UNIT 0xc0010299
1300 #define MSR_CORE_ENERGY_STAT 0xc001029a
1301 #define MSR_PKG_ENERGY_STAT 0xc001029b
1302
1303 #define MAX(a, b) ((a) > (b) ? (a) : (b))
1304
1305 int backwards_count;
1306 char *progname;
1307
1308 #define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */
1309 cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset;
1310 size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size;
1311 #define MAX_ADDED_THREAD_COUNTERS 24
1312 #define MAX_ADDED_CORE_COUNTERS 8
1313 #define MAX_ADDED_PACKAGE_COUNTERS 16
1314 #define PMT_MAX_ADDED_THREAD_COUNTERS 24
1315 #define PMT_MAX_ADDED_CORE_COUNTERS 8
1316 #define PMT_MAX_ADDED_PACKAGE_COUNTERS 16
1317 #define BITMASK_SIZE 32
1318
1319 #define ZERO_ARRAY(arr) (memset(arr, 0, sizeof(arr)) + __must_be_array(arr))
1320
1321 /* Indexes used to map data read from perf and MSRs into global variables */
1322 enum rapl_rci_index {
1323 RAPL_RCI_INDEX_ENERGY_PKG = 0,
1324 RAPL_RCI_INDEX_ENERGY_CORES = 1,
1325 RAPL_RCI_INDEX_DRAM = 2,
1326 RAPL_RCI_INDEX_GFX = 3,
1327 RAPL_RCI_INDEX_PKG_PERF_STATUS = 4,
1328 RAPL_RCI_INDEX_DRAM_PERF_STATUS = 5,
1329 RAPL_RCI_INDEX_CORE_ENERGY = 6,
1330 RAPL_RCI_INDEX_ENERGY_PLATFORM = 7,
1331 NUM_RAPL_COUNTERS,
1332 };
1333
1334 enum rapl_unit {
1335 RAPL_UNIT_INVALID,
1336 RAPL_UNIT_JOULES,
1337 RAPL_UNIT_WATTS,
1338 };
1339
1340 struct rapl_counter_info_t {
1341 unsigned long long data[NUM_RAPL_COUNTERS];
1342 enum counter_source source[NUM_RAPL_COUNTERS];
1343 unsigned long long flags[NUM_RAPL_COUNTERS];
1344 double scale[NUM_RAPL_COUNTERS];
1345 enum rapl_unit unit[NUM_RAPL_COUNTERS];
1346 unsigned long long msr[NUM_RAPL_COUNTERS];
1347 unsigned long long msr_mask[NUM_RAPL_COUNTERS];
1348 int msr_shift[NUM_RAPL_COUNTERS];
1349
1350 int fd_perf;
1351 };
1352
1353 /* struct rapl_counter_info_t for each RAPL domain */
1354 struct rapl_counter_info_t *rapl_counter_info_perdomain;
1355 unsigned int rapl_counter_info_perdomain_size;
1356
1357 #define RAPL_COUNTER_FLAG_PLATFORM_COUNTER (1u << 0)
1358 #define RAPL_COUNTER_FLAG_USE_MSR_SUM (1u << 1)
1359
1360 struct rapl_counter_arch_info {
1361 int feature_mask; /* Mask for testing if the counter is supported on host */
1362 const char *perf_subsys;
1363 const char *perf_name;
1364 unsigned long long msr;
1365 unsigned long long msr_mask;
1366 int msr_shift; /* Positive mean shift right, negative mean shift left */
1367 double *platform_rapl_msr_scale; /* Scale applied to values read by MSR (platform dependent, filled at runtime) */
1368 unsigned int rci_index; /* Maps data from perf counters to global variables */
1369 unsigned int bic_number;
1370 double compat_scale; /* Some counters require constant scaling to be in the same range as other, similar ones */
1371 unsigned long long flags;
1372 };
1373
1374 static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = {
1375 {
1376 .feature_mask = RAPL_PKG,
1377 .perf_subsys = "power",
1378 .perf_name = "energy-pkg",
1379 .msr = MSR_PKG_ENERGY_STATUS,
1380 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1381 .msr_shift = 0,
1382 .platform_rapl_msr_scale = &rapl_energy_units,
1383 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1384 .bic_number = BIC_PkgWatt,
1385 .compat_scale = 1.0,
1386 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1387 },
1388 {
1389 .feature_mask = RAPL_PKG,
1390 .perf_subsys = "power",
1391 .perf_name = "energy-pkg",
1392 .msr = MSR_PKG_ENERGY_STATUS,
1393 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1394 .msr_shift = 0,
1395 .platform_rapl_msr_scale = &rapl_energy_units,
1396 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1397 .bic_number = BIC_Pkg_J,
1398 .compat_scale = 1.0,
1399 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1400 },
1401 {
1402 .feature_mask = RAPL_AMD_F17H,
1403 .perf_subsys = "power",
1404 .perf_name = "energy-pkg",
1405 .msr = MSR_PKG_ENERGY_STAT,
1406 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1407 .msr_shift = 0,
1408 .platform_rapl_msr_scale = &rapl_energy_units,
1409 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1410 .bic_number = BIC_PkgWatt,
1411 .compat_scale = 1.0,
1412 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1413 },
1414 {
1415 .feature_mask = RAPL_AMD_F17H,
1416 .perf_subsys = "power",
1417 .perf_name = "energy-pkg",
1418 .msr = MSR_PKG_ENERGY_STAT,
1419 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1420 .msr_shift = 0,
1421 .platform_rapl_msr_scale = &rapl_energy_units,
1422 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1423 .bic_number = BIC_Pkg_J,
1424 .compat_scale = 1.0,
1425 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1426 },
1427 {
1428 .feature_mask = RAPL_CORE_ENERGY_STATUS,
1429 .perf_subsys = "power",
1430 .perf_name = "energy-cores",
1431 .msr = MSR_PP0_ENERGY_STATUS,
1432 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1433 .msr_shift = 0,
1434 .platform_rapl_msr_scale = &rapl_energy_units,
1435 .rci_index = RAPL_RCI_INDEX_ENERGY_CORES,
1436 .bic_number = BIC_CorWatt,
1437 .compat_scale = 1.0,
1438 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1439 },
1440 {
1441 .feature_mask = RAPL_CORE_ENERGY_STATUS,
1442 .perf_subsys = "power",
1443 .perf_name = "energy-cores",
1444 .msr = MSR_PP0_ENERGY_STATUS,
1445 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1446 .msr_shift = 0,
1447 .platform_rapl_msr_scale = &rapl_energy_units,
1448 .rci_index = RAPL_RCI_INDEX_ENERGY_CORES,
1449 .bic_number = BIC_Cor_J,
1450 .compat_scale = 1.0,
1451 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1452 },
1453 {
1454 .feature_mask = RAPL_DRAM,
1455 .perf_subsys = "power",
1456 .perf_name = "energy-ram",
1457 .msr = MSR_DRAM_ENERGY_STATUS,
1458 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1459 .msr_shift = 0,
1460 .platform_rapl_msr_scale = &rapl_dram_energy_units,
1461 .rci_index = RAPL_RCI_INDEX_DRAM,
1462 .bic_number = BIC_RAMWatt,
1463 .compat_scale = 1.0,
1464 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1465 },
1466 {
1467 .feature_mask = RAPL_DRAM,
1468 .perf_subsys = "power",
1469 .perf_name = "energy-ram",
1470 .msr = MSR_DRAM_ENERGY_STATUS,
1471 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1472 .msr_shift = 0,
1473 .platform_rapl_msr_scale = &rapl_dram_energy_units,
1474 .rci_index = RAPL_RCI_INDEX_DRAM,
1475 .bic_number = BIC_RAM_J,
1476 .compat_scale = 1.0,
1477 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1478 },
1479 {
1480 .feature_mask = RAPL_GFX,
1481 .perf_subsys = "power",
1482 .perf_name = "energy-gpu",
1483 .msr = MSR_PP1_ENERGY_STATUS,
1484 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1485 .msr_shift = 0,
1486 .platform_rapl_msr_scale = &rapl_energy_units,
1487 .rci_index = RAPL_RCI_INDEX_GFX,
1488 .bic_number = BIC_GFXWatt,
1489 .compat_scale = 1.0,
1490 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1491 },
1492 {
1493 .feature_mask = RAPL_GFX,
1494 .perf_subsys = "power",
1495 .perf_name = "energy-gpu",
1496 .msr = MSR_PP1_ENERGY_STATUS,
1497 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1498 .msr_shift = 0,
1499 .platform_rapl_msr_scale = &rapl_energy_units,
1500 .rci_index = RAPL_RCI_INDEX_GFX,
1501 .bic_number = BIC_GFX_J,
1502 .compat_scale = 1.0,
1503 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1504 },
1505 {
1506 .feature_mask = RAPL_PKG_PERF_STATUS,
1507 .perf_subsys = NULL,
1508 .perf_name = NULL,
1509 .msr = MSR_PKG_PERF_STATUS,
1510 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1511 .msr_shift = 0,
1512 .platform_rapl_msr_scale = &rapl_time_units,
1513 .rci_index = RAPL_RCI_INDEX_PKG_PERF_STATUS,
1514 .bic_number = BIC_PKG__,
1515 .compat_scale = 100.0,
1516 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1517 },
1518 {
1519 .feature_mask = RAPL_DRAM_PERF_STATUS,
1520 .perf_subsys = NULL,
1521 .perf_name = NULL,
1522 .msr = MSR_DRAM_PERF_STATUS,
1523 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1524 .msr_shift = 0,
1525 .platform_rapl_msr_scale = &rapl_time_units,
1526 .rci_index = RAPL_RCI_INDEX_DRAM_PERF_STATUS,
1527 .bic_number = BIC_RAM__,
1528 .compat_scale = 100.0,
1529 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1530 },
1531 {
1532 .feature_mask = RAPL_AMD_F17H,
1533 .perf_subsys = NULL,
1534 .perf_name = NULL,
1535 .msr = MSR_CORE_ENERGY_STAT,
1536 .msr_mask = 0xFFFFFFFF,
1537 .msr_shift = 0,
1538 .platform_rapl_msr_scale = &rapl_energy_units,
1539 .rci_index = RAPL_RCI_INDEX_CORE_ENERGY,
1540 .bic_number = BIC_CorWatt,
1541 .compat_scale = 1.0,
1542 .flags = 0,
1543 },
1544 {
1545 .feature_mask = RAPL_AMD_F17H,
1546 .perf_subsys = NULL,
1547 .perf_name = NULL,
1548 .msr = MSR_CORE_ENERGY_STAT,
1549 .msr_mask = 0xFFFFFFFF,
1550 .msr_shift = 0,
1551 .platform_rapl_msr_scale = &rapl_energy_units,
1552 .rci_index = RAPL_RCI_INDEX_CORE_ENERGY,
1553 .bic_number = BIC_Cor_J,
1554 .compat_scale = 1.0,
1555 .flags = 0,
1556 },
1557 {
1558 .feature_mask = RAPL_PSYS,
1559 .perf_subsys = "power",
1560 .perf_name = "energy-psys",
1561 .msr = MSR_PLATFORM_ENERGY_STATUS,
1562 .msr_mask = 0x00000000FFFFFFFF,
1563 .msr_shift = 0,
1564 .platform_rapl_msr_scale = &rapl_psys_energy_units,
1565 .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM,
1566 .bic_number = BIC_SysWatt,
1567 .compat_scale = 1.0,
1568 .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM,
1569 },
1570 {
1571 .feature_mask = RAPL_PSYS,
1572 .perf_subsys = "power",
1573 .perf_name = "energy-psys",
1574 .msr = MSR_PLATFORM_ENERGY_STATUS,
1575 .msr_mask = 0x00000000FFFFFFFF,
1576 .msr_shift = 0,
1577 .platform_rapl_msr_scale = &rapl_psys_energy_units,
1578 .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM,
1579 .bic_number = BIC_Sys_J,
1580 .compat_scale = 1.0,
1581 .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM,
1582 },
1583 };
1584
1585 struct rapl_counter {
1586 unsigned long long raw_value;
1587 enum rapl_unit unit;
1588 double scale;
1589 };
1590
1591 /* Indexes used to map data read from perf and MSRs into global variables */
1592 enum ccstate_rci_index {
1593 CCSTATE_RCI_INDEX_C1_RESIDENCY = 0,
1594 CCSTATE_RCI_INDEX_C3_RESIDENCY = 1,
1595 CCSTATE_RCI_INDEX_C6_RESIDENCY = 2,
1596 CCSTATE_RCI_INDEX_C7_RESIDENCY = 3,
1597 PCSTATE_RCI_INDEX_C2_RESIDENCY = 4,
1598 PCSTATE_RCI_INDEX_C3_RESIDENCY = 5,
1599 PCSTATE_RCI_INDEX_C6_RESIDENCY = 6,
1600 PCSTATE_RCI_INDEX_C7_RESIDENCY = 7,
1601 PCSTATE_RCI_INDEX_C8_RESIDENCY = 8,
1602 PCSTATE_RCI_INDEX_C9_RESIDENCY = 9,
1603 PCSTATE_RCI_INDEX_C10_RESIDENCY = 10,
1604 NUM_CSTATE_COUNTERS,
1605 };
1606
1607 struct cstate_counter_info_t {
1608 unsigned long long data[NUM_CSTATE_COUNTERS];
1609 enum counter_source source[NUM_CSTATE_COUNTERS];
1610 unsigned long long msr[NUM_CSTATE_COUNTERS];
1611 int fd_perf_core;
1612 int fd_perf_pkg;
1613 };
1614
1615 struct cstate_counter_info_t *ccstate_counter_info;
1616 unsigned int ccstate_counter_info_size;
1617
1618 #define CSTATE_COUNTER_FLAG_COLLECT_PER_CORE (1u << 0)
1619 #define CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD ((1u << 1) | CSTATE_COUNTER_FLAG_COLLECT_PER_CORE)
1620 #define CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY (1u << 2)
1621
1622 struct cstate_counter_arch_info {
1623 int feature_mask; /* Mask for testing if the counter is supported on host */
1624 const char *perf_subsys;
1625 const char *perf_name;
1626 unsigned long long msr;
1627 unsigned int rci_index; /* Maps data from perf counters to global variables */
1628 unsigned int bic_number;
1629 unsigned long long flags;
1630 int pkg_cstate_limit;
1631 };
1632
1633 static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = {
1634 {
1635 .feature_mask = CC1,
1636 .perf_subsys = "cstate_core",
1637 .perf_name = "c1-residency",
1638 .msr = MSR_CORE_C1_RES,
1639 .rci_index = CCSTATE_RCI_INDEX_C1_RESIDENCY,
1640 .bic_number = BIC_CPU_c1,
1641 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD,
1642 .pkg_cstate_limit = 0,
1643 },
1644 {
1645 .feature_mask = CC3,
1646 .perf_subsys = "cstate_core",
1647 .perf_name = "c3-residency",
1648 .msr = MSR_CORE_C3_RESIDENCY,
1649 .rci_index = CCSTATE_RCI_INDEX_C3_RESIDENCY,
1650 .bic_number = BIC_CPU_c3,
1651 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1652 .pkg_cstate_limit = 0,
1653 },
1654 {
1655 .feature_mask = CC6,
1656 .perf_subsys = "cstate_core",
1657 .perf_name = "c6-residency",
1658 .msr = MSR_CORE_C6_RESIDENCY,
1659 .rci_index = CCSTATE_RCI_INDEX_C6_RESIDENCY,
1660 .bic_number = BIC_CPU_c6,
1661 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1662 .pkg_cstate_limit = 0,
1663 },
1664 {
1665 .feature_mask = CC7,
1666 .perf_subsys = "cstate_core",
1667 .perf_name = "c7-residency",
1668 .msr = MSR_CORE_C7_RESIDENCY,
1669 .rci_index = CCSTATE_RCI_INDEX_C7_RESIDENCY,
1670 .bic_number = BIC_CPU_c7,
1671 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1672 .pkg_cstate_limit = 0,
1673 },
1674 {
1675 .feature_mask = PC2,
1676 .perf_subsys = "cstate_pkg",
1677 .perf_name = "c2-residency",
1678 .msr = MSR_PKG_C2_RESIDENCY,
1679 .rci_index = PCSTATE_RCI_INDEX_C2_RESIDENCY,
1680 .bic_number = BIC_Pkgpc2,
1681 .flags = 0,
1682 .pkg_cstate_limit = PCL__2,
1683 },
1684 {
1685 .feature_mask = PC3,
1686 .perf_subsys = "cstate_pkg",
1687 .perf_name = "c3-residency",
1688 .msr = MSR_PKG_C3_RESIDENCY,
1689 .rci_index = PCSTATE_RCI_INDEX_C3_RESIDENCY,
1690 .bic_number = BIC_Pkgpc3,
1691 .flags = 0,
1692 .pkg_cstate_limit = PCL__3,
1693 },
1694 {
1695 .feature_mask = PC6,
1696 .perf_subsys = "cstate_pkg",
1697 .perf_name = "c6-residency",
1698 .msr = MSR_PKG_C6_RESIDENCY,
1699 .rci_index = PCSTATE_RCI_INDEX_C6_RESIDENCY,
1700 .bic_number = BIC_Pkgpc6,
1701 .flags = 0,
1702 .pkg_cstate_limit = PCL__6,
1703 },
1704 {
1705 .feature_mask = PC7,
1706 .perf_subsys = "cstate_pkg",
1707 .perf_name = "c7-residency",
1708 .msr = MSR_PKG_C7_RESIDENCY,
1709 .rci_index = PCSTATE_RCI_INDEX_C7_RESIDENCY,
1710 .bic_number = BIC_Pkgpc7,
1711 .flags = 0,
1712 .pkg_cstate_limit = PCL__7,
1713 },
1714 {
1715 .feature_mask = PC8,
1716 .perf_subsys = "cstate_pkg",
1717 .perf_name = "c8-residency",
1718 .msr = MSR_PKG_C8_RESIDENCY,
1719 .rci_index = PCSTATE_RCI_INDEX_C8_RESIDENCY,
1720 .bic_number = BIC_Pkgpc8,
1721 .flags = 0,
1722 .pkg_cstate_limit = PCL__8,
1723 },
1724 {
1725 .feature_mask = PC9,
1726 .perf_subsys = "cstate_pkg",
1727 .perf_name = "c9-residency",
1728 .msr = MSR_PKG_C9_RESIDENCY,
1729 .rci_index = PCSTATE_RCI_INDEX_C9_RESIDENCY,
1730 .bic_number = BIC_Pkgpc9,
1731 .flags = 0,
1732 .pkg_cstate_limit = PCL__9,
1733 },
1734 {
1735 .feature_mask = PC10,
1736 .perf_subsys = "cstate_pkg",
1737 .perf_name = "c10-residency",
1738 .msr = MSR_PKG_C10_RESIDENCY,
1739 .rci_index = PCSTATE_RCI_INDEX_C10_RESIDENCY,
1740 .bic_number = BIC_Pkgpc10,
1741 .flags = 0,
1742 .pkg_cstate_limit = PCL_10,
1743 },
1744 };
1745
1746 /* Indexes used to map data read from perf and MSRs into global variables */
1747 enum msr_rci_index {
1748 MSR_RCI_INDEX_APERF = 0,
1749 MSR_RCI_INDEX_MPERF = 1,
1750 MSR_RCI_INDEX_SMI = 2,
1751 NUM_MSR_COUNTERS,
1752 };
1753
1754 struct msr_counter_info_t {
1755 unsigned long long data[NUM_MSR_COUNTERS];
1756 enum counter_source source[NUM_MSR_COUNTERS];
1757 unsigned long long msr[NUM_MSR_COUNTERS];
1758 unsigned long long msr_mask[NUM_MSR_COUNTERS];
1759 int fd_perf;
1760 };
1761
1762 struct msr_counter_info_t *msr_counter_info;
1763 unsigned int msr_counter_info_size;
1764
1765 struct msr_counter_arch_info {
1766 const char *perf_subsys;
1767 const char *perf_name;
1768 unsigned long long msr;
1769 unsigned long long msr_mask;
1770 unsigned int rci_index; /* Maps data from perf counters to global variables */
1771 bool needed;
1772 bool present;
1773 };
1774
1775 enum msr_arch_info_index {
1776 MSR_ARCH_INFO_APERF_INDEX = 0,
1777 MSR_ARCH_INFO_MPERF_INDEX = 1,
1778 MSR_ARCH_INFO_SMI_INDEX = 2,
1779 };
1780
1781 static struct msr_counter_arch_info msr_counter_arch_infos[] = {
1782 [MSR_ARCH_INFO_APERF_INDEX] = {
1783 .perf_subsys = "msr",
1784 .perf_name = "aperf",
1785 .msr = MSR_IA32_APERF,
1786 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1787 .rci_index = MSR_RCI_INDEX_APERF,
1788 },
1789
1790 [MSR_ARCH_INFO_MPERF_INDEX] = {
1791 .perf_subsys = "msr",
1792 .perf_name = "mperf",
1793 .msr = MSR_IA32_MPERF,
1794 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1795 .rci_index = MSR_RCI_INDEX_MPERF,
1796 },
1797
1798 [MSR_ARCH_INFO_SMI_INDEX] = {
1799 .perf_subsys = "msr",
1800 .perf_name = "smi",
1801 .msr = MSR_SMI_COUNT,
1802 .msr_mask = 0xFFFFFFFF,
1803 .rci_index = MSR_RCI_INDEX_SMI,
1804 },
1805 };
1806
1807 /* Can be redefined when compiling, useful for testing. */
1808 #ifndef SYSFS_TELEM_PATH
1809 #define SYSFS_TELEM_PATH "/sys/class/intel_pmt"
1810 #endif
1811
1812 #define PMT_COUNTER_MTL_DC6_OFFSET 120
1813 #define PMT_COUNTER_MTL_DC6_LSB 0
1814 #define PMT_COUNTER_MTL_DC6_MSB 63
1815 #define PMT_MTL_DC6_GUID 0x1a067102
1816 #define PMT_MTL_DC6_SEQ 0
1817
1818 #define PMT_COUNTER_CWF_MC1E_OFFSET_BASE 20936
1819 #define PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT 24
1820 #define PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE 12
1821 #define PMT_COUNTER_CWF_CPUS_PER_MODULE 4
1822 #define PMT_COUNTER_CWF_MC1E_LSB 0
1823 #define PMT_COUNTER_CWF_MC1E_MSB 63
1824 #define PMT_CWF_MC1E_GUID 0x14421519
1825
1826 unsigned long long tcore_clock_freq_hz = 800000000;
1827
1828 #define PMT_COUNTER_NAME_SIZE_BYTES 16
1829 #define PMT_COUNTER_TYPE_NAME_SIZE_BYTES 32
1830
1831 struct pmt_mmio {
1832 struct pmt_mmio *next;
1833
1834 unsigned int guid;
1835 unsigned int size;
1836
1837 /* Base pointer to the mmaped memory. */
1838 void *mmio_base;
1839
1840 /*
1841 * Offset to be applied to the mmio_base
1842 * to get the beginning of the PMT counters for given GUID.
1843 */
1844 unsigned long pmt_offset;
1845 } *pmt_mmios;
1846
1847 enum pmt_datatype {
1848 PMT_TYPE_RAW,
1849 PMT_TYPE_XTAL_TIME,
1850 PMT_TYPE_TCORE_CLOCK,
1851 };
1852
1853 struct pmt_domain_info {
1854 /*
1855 * Pointer to the MMIO obtained by applying a counter offset
1856 * to the mmio_base of the mmaped region for the given GUID.
1857 *
1858 * This is where to read the raw value of the counter from.
1859 */
1860 unsigned long *pcounter;
1861 };
1862
1863 struct pmt_counter {
1864 struct pmt_counter *next;
1865
1866 /* PMT metadata */
1867 char name[PMT_COUNTER_NAME_SIZE_BYTES];
1868 enum pmt_datatype type;
1869 enum counter_scope scope;
1870 unsigned int lsb;
1871 unsigned int msb;
1872
1873 /* BIC-like metadata */
1874 enum counter_format format;
1875
1876 unsigned int num_domains;
1877 struct pmt_domain_info *domains;
1878 };
1879
1880 /*
1881 * PMT telemetry directory iterator.
1882 * Used to iterate telemetry files in sysfs in correct order.
1883 */
1884 struct pmt_diriter_t {
1885 DIR *dir;
1886 struct dirent **namelist;
1887 unsigned int num_names;
1888 unsigned int current_name_idx;
1889 };
1890
pmt_telemdir_filter(const struct dirent * e)1891 int pmt_telemdir_filter(const struct dirent *e)
1892 {
1893 unsigned int dummy;
1894
1895 return sscanf(e->d_name, "telem%u", &dummy);
1896 }
1897
pmt_telemdir_sort(const struct dirent ** a,const struct dirent ** b)1898 int pmt_telemdir_sort(const struct dirent **a, const struct dirent **b)
1899 {
1900 unsigned int aidx = 0, bidx = 0;
1901
1902 sscanf((*a)->d_name, "telem%u", &aidx);
1903 sscanf((*b)->d_name, "telem%u", &bidx);
1904
1905 return (aidx > bidx) ? 1 : (aidx < bidx) ? -1 : 0;
1906 }
1907
pmt_diriter_next(struct pmt_diriter_t * iter)1908 const struct dirent *pmt_diriter_next(struct pmt_diriter_t *iter)
1909 {
1910 const struct dirent *ret = NULL;
1911
1912 if (!iter->dir)
1913 return NULL;
1914
1915 if (iter->current_name_idx >= iter->num_names)
1916 return NULL;
1917
1918 ret = iter->namelist[iter->current_name_idx];
1919 ++iter->current_name_idx;
1920
1921 return ret;
1922 }
1923
pmt_diriter_begin(struct pmt_diriter_t * iter,const char * pmt_root_path)1924 const struct dirent *pmt_diriter_begin(struct pmt_diriter_t *iter, const char *pmt_root_path)
1925 {
1926 int num_names = iter->num_names;
1927
1928 if (!iter->dir) {
1929 iter->dir = opendir(pmt_root_path);
1930 if (iter->dir == NULL)
1931 return NULL;
1932
1933 num_names = scandir(pmt_root_path, &iter->namelist, pmt_telemdir_filter, pmt_telemdir_sort);
1934 if (num_names == -1)
1935 return NULL;
1936 }
1937
1938 iter->current_name_idx = 0;
1939 iter->num_names = num_names;
1940
1941 return pmt_diriter_next(iter);
1942 }
1943
pmt_diriter_init(struct pmt_diriter_t * iter)1944 void pmt_diriter_init(struct pmt_diriter_t *iter)
1945 {
1946 memset(iter, 0, sizeof(*iter));
1947 }
1948
pmt_diriter_remove(struct pmt_diriter_t * iter)1949 void pmt_diriter_remove(struct pmt_diriter_t *iter)
1950 {
1951 if (iter->namelist) {
1952 for (unsigned int i = 0; i < iter->num_names; i++) {
1953 free(iter->namelist[i]);
1954 iter->namelist[i] = NULL;
1955 }
1956 }
1957
1958 free(iter->namelist);
1959 iter->namelist = NULL;
1960 iter->num_names = 0;
1961 iter->current_name_idx = 0;
1962
1963 closedir(iter->dir);
1964 iter->dir = NULL;
1965 }
1966
pmt_counter_get_width(const struct pmt_counter * p)1967 unsigned int pmt_counter_get_width(const struct pmt_counter *p)
1968 {
1969 return (p->msb - p->lsb) + 1;
1970 }
1971
pmt_counter_resize_(struct pmt_counter * pcounter,unsigned int new_size)1972 void pmt_counter_resize_(struct pmt_counter *pcounter, unsigned int new_size)
1973 {
1974 struct pmt_domain_info *new_mem;
1975
1976 new_mem = (struct pmt_domain_info *)reallocarray(pcounter->domains, new_size, sizeof(*pcounter->domains));
1977 if (!new_mem) {
1978 fprintf(stderr, "%s: failed to allocate memory for PMT counters\n", __func__);
1979 exit(1);
1980 }
1981
1982 /* Zero initialize just allocated memory. */
1983 const size_t num_new_domains = new_size - pcounter->num_domains;
1984
1985 memset(&new_mem[pcounter->num_domains], 0, num_new_domains * sizeof(*pcounter->domains));
1986
1987 pcounter->num_domains = new_size;
1988 pcounter->domains = new_mem;
1989 }
1990
pmt_counter_resize(struct pmt_counter * pcounter,unsigned int new_size)1991 void pmt_counter_resize(struct pmt_counter *pcounter, unsigned int new_size)
1992 {
1993 /*
1994 * Allocate more memory ahead of time.
1995 *
1996 * Always allocate space for at least 8 elements
1997 * and double the size when growing.
1998 */
1999 if (new_size < 8)
2000 new_size = 8;
2001 new_size = MAX(new_size, pcounter->num_domains * 2);
2002
2003 pmt_counter_resize_(pcounter, new_size);
2004 }
2005
2006 struct llc_stats {
2007 unsigned long long references;
2008 unsigned long long misses;
2009 };
2010 struct thread_data {
2011 struct timeval tv_begin;
2012 struct timeval tv_end;
2013 struct timeval tv_delta;
2014 unsigned long long tsc;
2015 unsigned long long aperf;
2016 unsigned long long mperf;
2017 unsigned long long c1;
2018 unsigned long long instr_count;
2019 unsigned long long irq_count;
2020 unsigned long long nmi_count;
2021 unsigned int smi_count;
2022 struct llc_stats llc;
2023 unsigned int cpu_id;
2024 unsigned int apic_id;
2025 unsigned int x2apic_id;
2026 unsigned int flags;
2027 bool is_atom;
2028 unsigned long long counter[MAX_ADDED_THREAD_COUNTERS];
2029 unsigned long long perf_counter[MAX_ADDED_THREAD_COUNTERS];
2030 unsigned long long pmt_counter[PMT_MAX_ADDED_THREAD_COUNTERS];
2031 } *thread_even, *thread_odd;
2032
2033 struct core_data {
2034 int base_cpu;
2035 unsigned long long c3;
2036 unsigned long long c6;
2037 unsigned long long c7;
2038 unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */
2039 unsigned int core_temp_c;
2040 struct rapl_counter core_energy; /* MSR_CORE_ENERGY_STAT */
2041 unsigned int core_id;
2042 unsigned long long core_throt_cnt;
2043 unsigned long long counter[MAX_ADDED_CORE_COUNTERS];
2044 unsigned long long perf_counter[MAX_ADDED_CORE_COUNTERS];
2045 unsigned long long pmt_counter[PMT_MAX_ADDED_CORE_COUNTERS];
2046 } *core_even, *core_odd;
2047
2048 struct pkg_data {
2049 int base_cpu;
2050 unsigned long long pc2;
2051 unsigned long long pc3;
2052 unsigned long long pc6;
2053 unsigned long long pc7;
2054 unsigned long long pc8;
2055 unsigned long long pc9;
2056 unsigned long long pc10;
2057 long long cpu_lpi;
2058 long long sys_lpi;
2059 unsigned long long pkg_wtd_core_c0;
2060 unsigned long long pkg_any_core_c0;
2061 unsigned long long pkg_any_gfxe_c0;
2062 unsigned long long pkg_both_core_gfxe_c0;
2063 long long gfx_rc6_ms;
2064 unsigned int gfx_mhz;
2065 unsigned int gfx_act_mhz;
2066 long long sam_mc6_ms;
2067 unsigned int sam_mhz;
2068 unsigned int sam_act_mhz;
2069 unsigned int package_id;
2070 struct rapl_counter energy_pkg; /* MSR_PKG_ENERGY_STATUS */
2071 struct rapl_counter energy_dram; /* MSR_DRAM_ENERGY_STATUS */
2072 struct rapl_counter energy_cores; /* MSR_PP0_ENERGY_STATUS */
2073 struct rapl_counter energy_gfx; /* MSR_PP1_ENERGY_STATUS */
2074 struct rapl_counter rapl_pkg_perf_status; /* MSR_PKG_PERF_STATUS */
2075 struct rapl_counter rapl_dram_perf_status; /* MSR_DRAM_PERF_STATUS */
2076 unsigned int pkg_temp_c;
2077 unsigned int uncore_mhz;
2078 unsigned long long die_c6;
2079 unsigned long long counter[MAX_ADDED_PACKAGE_COUNTERS];
2080 unsigned long long perf_counter[MAX_ADDED_PACKAGE_COUNTERS];
2081 unsigned long long pmt_counter[PMT_MAX_ADDED_PACKAGE_COUNTERS];
2082 } *package_even, *package_odd;
2083
2084 #define ODD_COUNTERS thread_odd, core_odd, package_odd
2085 #define EVEN_COUNTERS thread_even, core_even, package_even
2086
2087 #define GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no) \
2088 ((thread_base) + \
2089 ((pkg_no) * \
2090 topo.nodes_per_pkg * topo.cores_per_node * topo.threads_per_core) + \
2091 ((node_no) * topo.cores_per_node * topo.threads_per_core) + \
2092 ((core_no) * topo.threads_per_core) + \
2093 (thread_no))
2094
2095 #define GET_CORE(core_base, core_no, node_no, pkg_no) \
2096 ((core_base) + \
2097 ((pkg_no) * topo.nodes_per_pkg * topo.cores_per_node) + \
2098 ((node_no) * topo.cores_per_node) + \
2099 (core_no))
2100
2101 /*
2102 * The accumulated sum of MSR is defined as a monotonic
2103 * increasing MSR, it will be accumulated periodically,
2104 * despite its register's bit width.
2105 */
2106 enum {
2107 IDX_PKG_ENERGY,
2108 IDX_DRAM_ENERGY,
2109 IDX_PP0_ENERGY,
2110 IDX_PP1_ENERGY,
2111 IDX_PKG_PERF,
2112 IDX_DRAM_PERF,
2113 IDX_PSYS_ENERGY,
2114 IDX_COUNT,
2115 };
2116
2117 int get_msr_sum(int cpu, off_t offset, unsigned long long *msr);
2118
2119 struct msr_sum_array {
2120 /* get_msr_sum() = sum + (get_msr() - last) */
2121 struct {
2122 /*The accumulated MSR value is updated by the timer */
2123 unsigned long long sum;
2124 /*The MSR footprint recorded in last timer */
2125 unsigned long long last;
2126 } entries[IDX_COUNT];
2127 };
2128
2129 /* The percpu MSR sum array.*/
2130 struct msr_sum_array *per_cpu_msr_sum;
2131
idx_to_offset(int idx)2132 off_t idx_to_offset(int idx)
2133 {
2134 off_t offset;
2135
2136 switch (idx) {
2137 case IDX_PKG_ENERGY:
2138 if (valid_rapl_msrs & RAPL_AMD_F17H)
2139 offset = MSR_PKG_ENERGY_STAT;
2140 else
2141 offset = MSR_PKG_ENERGY_STATUS;
2142 break;
2143 case IDX_DRAM_ENERGY:
2144 offset = MSR_DRAM_ENERGY_STATUS;
2145 break;
2146 case IDX_PP0_ENERGY:
2147 offset = MSR_PP0_ENERGY_STATUS;
2148 break;
2149 case IDX_PP1_ENERGY:
2150 offset = MSR_PP1_ENERGY_STATUS;
2151 break;
2152 case IDX_PKG_PERF:
2153 offset = MSR_PKG_PERF_STATUS;
2154 break;
2155 case IDX_DRAM_PERF:
2156 offset = MSR_DRAM_PERF_STATUS;
2157 break;
2158 case IDX_PSYS_ENERGY:
2159 offset = MSR_PLATFORM_ENERGY_STATUS;
2160 break;
2161 default:
2162 offset = -1;
2163 }
2164 return offset;
2165 }
2166
offset_to_idx(off_t offset)2167 int offset_to_idx(off_t offset)
2168 {
2169 int idx;
2170
2171 switch (offset) {
2172 case MSR_PKG_ENERGY_STATUS:
2173 case MSR_PKG_ENERGY_STAT:
2174 idx = IDX_PKG_ENERGY;
2175 break;
2176 case MSR_DRAM_ENERGY_STATUS:
2177 idx = IDX_DRAM_ENERGY;
2178 break;
2179 case MSR_PP0_ENERGY_STATUS:
2180 idx = IDX_PP0_ENERGY;
2181 break;
2182 case MSR_PP1_ENERGY_STATUS:
2183 idx = IDX_PP1_ENERGY;
2184 break;
2185 case MSR_PKG_PERF_STATUS:
2186 idx = IDX_PKG_PERF;
2187 break;
2188 case MSR_DRAM_PERF_STATUS:
2189 idx = IDX_DRAM_PERF;
2190 break;
2191 case MSR_PLATFORM_ENERGY_STATUS:
2192 idx = IDX_PSYS_ENERGY;
2193 break;
2194 default:
2195 idx = -1;
2196 }
2197 return idx;
2198 }
2199
idx_valid(int idx)2200 int idx_valid(int idx)
2201 {
2202 switch (idx) {
2203 case IDX_PKG_ENERGY:
2204 return valid_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H);
2205 case IDX_DRAM_ENERGY:
2206 return valid_rapl_msrs & RAPL_DRAM;
2207 case IDX_PP0_ENERGY:
2208 return valid_rapl_msrs & RAPL_CORE_ENERGY_STATUS;
2209 case IDX_PP1_ENERGY:
2210 return valid_rapl_msrs & RAPL_GFX;
2211 case IDX_PKG_PERF:
2212 return valid_rapl_msrs & RAPL_PKG_PERF_STATUS;
2213 case IDX_DRAM_PERF:
2214 return valid_rapl_msrs & RAPL_DRAM_PERF_STATUS;
2215 case IDX_PSYS_ENERGY:
2216 return valid_rapl_msrs & RAPL_PSYS;
2217 default:
2218 return 0;
2219 }
2220 }
2221
2222 struct sys_counters {
2223 /* MSR added counters */
2224 unsigned int added_thread_counters;
2225 unsigned int added_core_counters;
2226 unsigned int added_package_counters;
2227 struct msr_counter *tp;
2228 struct msr_counter *cp;
2229 struct msr_counter *pp;
2230
2231 /* perf added counters */
2232 unsigned int added_thread_perf_counters;
2233 unsigned int added_core_perf_counters;
2234 unsigned int added_package_perf_counters;
2235 struct perf_counter_info *perf_tp;
2236 struct perf_counter_info *perf_cp;
2237 struct perf_counter_info *perf_pp;
2238
2239 struct pmt_counter *pmt_tp;
2240 struct pmt_counter *pmt_cp;
2241 struct pmt_counter *pmt_pp;
2242 } sys;
2243
free_msr_counters_(struct msr_counter ** pp)2244 static size_t free_msr_counters_(struct msr_counter **pp)
2245 {
2246 struct msr_counter *p = NULL;
2247 size_t num_freed = 0;
2248
2249 while (*pp) {
2250 p = *pp;
2251
2252 if (p->msr_num != 0) {
2253 *pp = p->next;
2254
2255 free(p);
2256 ++num_freed;
2257
2258 continue;
2259 }
2260
2261 pp = &p->next;
2262 }
2263
2264 return num_freed;
2265 }
2266
2267 /*
2268 * Free all added counters accessed via msr.
2269 */
free_sys_msr_counters(void)2270 static void free_sys_msr_counters(void)
2271 {
2272 /* Thread counters */
2273 sys.added_thread_counters -= free_msr_counters_(&sys.tp);
2274
2275 /* Core counters */
2276 sys.added_core_counters -= free_msr_counters_(&sys.cp);
2277
2278 /* Package counters */
2279 sys.added_package_counters -= free_msr_counters_(&sys.pp);
2280 }
2281
2282 struct system_summary {
2283 struct thread_data threads;
2284 struct core_data cores;
2285 struct pkg_data packages;
2286 } average;
2287
2288 struct platform_counters {
2289 struct rapl_counter energy_psys; /* MSR_PLATFORM_ENERGY_STATUS */
2290 } platform_counters_odd, platform_counters_even;
2291
2292 struct cpu_topology {
2293 int physical_package_id;
2294 int die_id;
2295 int l3_id;
2296 int logical_cpu_id;
2297 int physical_node_id;
2298 int logical_node_id; /* 0-based count within the package */
2299 int physical_core_id;
2300 int thread_id;
2301 int type;
2302 cpu_set_t *put_ids; /* Processing Unit/Thread IDs */
2303 } *cpus;
2304
2305 struct topo_params {
2306 int num_packages;
2307 int num_die;
2308 int num_cpus;
2309 int num_cores;
2310 int allowed_packages;
2311 int allowed_cpus;
2312 int allowed_cores;
2313 int max_cpu_num;
2314 int max_core_id;
2315 int max_package_id;
2316 int max_die_id;
2317 int max_l3_id;
2318 int max_node_num;
2319 int nodes_per_pkg;
2320 int cores_per_node;
2321 int threads_per_core;
2322 } topo;
2323
2324 struct timeval tv_even, tv_odd, tv_delta;
2325
2326 int *irq_column_2_cpu; /* /proc/interrupts column numbers */
2327 int *irqs_per_cpu; /* indexed by cpu_num */
2328 int *nmi_per_cpu; /* indexed by cpu_num */
2329
2330 void setup_all_buffers(bool startup);
2331
2332 char *sys_lpi_file;
2333 char *sys_lpi_file_sysfs = "/sys/devices/system/cpu/cpuidle/low_power_idle_system_residency_us";
2334 char *sys_lpi_file_debugfs = "/sys/kernel/debug/pmc_core/slp_s0_residency_usec";
2335
cpu_is_not_present(int cpu)2336 int cpu_is_not_present(int cpu)
2337 {
2338 return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set);
2339 }
2340
cpu_is_not_allowed(int cpu)2341 int cpu_is_not_allowed(int cpu)
2342 {
2343 return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set);
2344 }
2345
2346 /*
2347 * run func(thread, core, package) in topology order
2348 * skip non-present cpus
2349 */
2350
2351 #define PER_THREAD_PARAMS struct thread_data *t, struct core_data *c, struct pkg_data *p
2352
for_all_cpus(int (func)(struct thread_data *,struct core_data *,struct pkg_data *),struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base)2353 int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *),
2354 struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base)
2355 {
2356 int retval, pkg_no, core_no, thread_no, node_no;
2357
2358 retval = 0;
2359
2360 for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
2361 for (node_no = 0; node_no < topo.nodes_per_pkg; node_no++) {
2362 for (core_no = 0; core_no < topo.cores_per_node; ++core_no) {
2363 for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) {
2364 struct thread_data *t;
2365 struct core_data *c;
2366
2367 t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no);
2368
2369 if (cpu_is_not_allowed(t->cpu_id))
2370 continue;
2371
2372 c = GET_CORE(core_base, core_no, node_no, pkg_no);
2373
2374 retval |= func(t, c, &pkg_base[pkg_no]);
2375 }
2376 }
2377 }
2378 }
2379 return retval;
2380 }
2381
is_cpu_first_thread_in_core(struct thread_data * t,struct core_data * c)2382 int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c)
2383 {
2384 return ((int)t->cpu_id == c->base_cpu || c->base_cpu < 0);
2385 }
2386
is_cpu_first_core_in_package(struct thread_data * t,struct pkg_data * p)2387 int is_cpu_first_core_in_package(struct thread_data *t, struct pkg_data *p)
2388 {
2389 return ((int)t->cpu_id == p->base_cpu || p->base_cpu < 0);
2390 }
2391
is_cpu_first_thread_in_package(struct thread_data * t,struct core_data * c,struct pkg_data * p)2392 int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p)
2393 {
2394 return is_cpu_first_thread_in_core(t, c) && is_cpu_first_core_in_package(t, p);
2395 }
2396
cpu_migrate(int cpu)2397 int cpu_migrate(int cpu)
2398 {
2399 CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
2400 CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set);
2401 if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1)
2402 return -1;
2403 else
2404 return 0;
2405 }
2406
get_msr_fd(int cpu)2407 int get_msr_fd(int cpu)
2408 {
2409 char pathname[32];
2410 int fd;
2411
2412 fd = fd_percpu[cpu];
2413
2414 if (fd)
2415 return fd;
2416 sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu);
2417 fd = open(pathname, O_RDONLY);
2418 if (fd < 0)
2419 err(-1, "%s open failed, try chown or chmod +r %s, "
2420 "or run with --no-msr, or run as root", pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr");
2421 fd_percpu[cpu] = fd;
2422
2423 return fd;
2424 }
2425
bic_disable_msr_access(void)2426 static void bic_disable_msr_access(void)
2427 {
2428 CLR_BIC(BIC_Mod_c6, &bic_enabled);
2429 CLR_BIC(BIC_CoreTmp, &bic_enabled);
2430 CLR_BIC(BIC_Totl_c0, &bic_enabled);
2431 CLR_BIC(BIC_Any_c0, &bic_enabled);
2432 CLR_BIC(BIC_GFX_c0, &bic_enabled);
2433 CLR_BIC(BIC_CPUGFX, &bic_enabled);
2434 CLR_BIC(BIC_PkgTmp, &bic_enabled);
2435
2436 free_sys_msr_counters();
2437 }
2438
bic_disable_perf_access(void)2439 static void bic_disable_perf_access(void)
2440 {
2441 CLR_BIC(BIC_IPC, &bic_enabled);
2442 CLR_BIC(BIC_LLC_RPS, &bic_enabled);
2443 CLR_BIC(BIC_LLC_HIT, &bic_enabled);
2444 }
2445
perf_event_open(struct perf_event_attr * hw_event,pid_t pid,int cpu,int group_fd,unsigned long flags)2446 static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags)
2447 {
2448 assert(!no_perf);
2449
2450 return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags);
2451 }
2452
open_perf_counter(int cpu,unsigned int type,unsigned int config,int group_fd,__u64 read_format)2453 static long open_perf_counter(int cpu, unsigned int type, unsigned int config, int group_fd, __u64 read_format)
2454 {
2455 struct perf_event_attr attr;
2456 const pid_t pid = -1;
2457 const unsigned long flags = 0;
2458
2459 assert(!no_perf);
2460
2461 memset(&attr, 0, sizeof(struct perf_event_attr));
2462
2463 attr.type = type;
2464 attr.size = sizeof(struct perf_event_attr);
2465 attr.config = config;
2466 attr.disabled = 0;
2467 attr.sample_type = PERF_SAMPLE_IDENTIFIER;
2468 attr.read_format = read_format;
2469
2470 const int fd = perf_event_open(&attr, pid, cpu, group_fd, flags);
2471
2472 return fd;
2473 }
2474
get_instr_count_fd(int cpu)2475 int get_instr_count_fd(int cpu)
2476 {
2477 if (fd_instr_count_percpu[cpu])
2478 return fd_instr_count_percpu[cpu];
2479
2480 fd_instr_count_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0);
2481
2482 return fd_instr_count_percpu[cpu];
2483 }
2484
get_msr(int cpu,off_t offset,unsigned long long * msr)2485 int get_msr(int cpu, off_t offset, unsigned long long *msr)
2486 {
2487 ssize_t retval;
2488
2489 assert(!no_msr);
2490
2491 retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset);
2492
2493 if (retval != sizeof *msr)
2494 err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset);
2495
2496 return 0;
2497 }
2498
add_msr_counter(int cpu,off_t offset)2499 int add_msr_counter(int cpu, off_t offset)
2500 {
2501 ssize_t retval;
2502 unsigned long long value;
2503
2504 if (no_msr)
2505 return -1;
2506
2507 if (!offset)
2508 return -1;
2509
2510 retval = pread(get_msr_fd(cpu), &value, sizeof(value), offset);
2511
2512 /* if the read failed, the probe fails */
2513 if (retval != sizeof(value))
2514 return -1;
2515
2516 if (value == 0)
2517 return 0;
2518
2519 return 1;
2520 }
2521
add_rapl_msr_counter(int cpu,const struct rapl_counter_arch_info * cai)2522 int add_rapl_msr_counter(int cpu, const struct rapl_counter_arch_info *cai)
2523 {
2524 int ret;
2525
2526 if (!(valid_rapl_msrs & cai->feature_mask))
2527 return -1;
2528
2529 ret = add_msr_counter(cpu, cai->msr);
2530 if (ret < 0)
2531 return -1;
2532
2533 switch (cai->rci_index) {
2534 case RAPL_RCI_INDEX_ENERGY_PKG:
2535 case RAPL_RCI_INDEX_ENERGY_CORES:
2536 case RAPL_RCI_INDEX_DRAM:
2537 case RAPL_RCI_INDEX_GFX:
2538 case RAPL_RCI_INDEX_ENERGY_PLATFORM:
2539 if (ret == 0)
2540 return 1;
2541 }
2542
2543 /* PKG,DRAM_PERF_STATUS MSRs, can return any value */
2544 return 1;
2545 }
2546
2547 /* Convert CPU ID to domain ID for given added perf counter. */
cpu_to_domain(const struct perf_counter_info * pc,int cpu)2548 unsigned int cpu_to_domain(const struct perf_counter_info *pc, int cpu)
2549 {
2550 switch (pc->scope) {
2551 case SCOPE_CPU:
2552 return cpu;
2553
2554 case SCOPE_CORE:
2555 return cpus[cpu].physical_core_id;
2556
2557 case SCOPE_PACKAGE:
2558 return cpus[cpu].physical_package_id;
2559 }
2560
2561 __builtin_unreachable();
2562 }
2563
2564 #define MAX_DEFERRED 16
2565 char *deferred_add_names[MAX_DEFERRED];
2566 char *deferred_skip_names[MAX_DEFERRED];
2567 int deferred_add_index;
2568 int deferred_skip_index;
2569 unsigned int deferred_add_consumed;
2570 unsigned int deferred_skip_consumed;
2571
2572 /*
2573 * HIDE_LIST - hide this list of counters, show the rest [default]
2574 * SHOW_LIST - show this list of counters, hide the rest
2575 */
2576 enum show_hide_mode { SHOW_LIST, HIDE_LIST } global_show_hide_mode = HIDE_LIST;
2577
help(void)2578 void help(void)
2579 {
2580 fprintf(outf,
2581 "Usage: turbostat [OPTIONS][(--interval seconds) | COMMAND ...]\n"
2582 "\n"
2583 "Turbostat forks the specified COMMAND and prints statistics\n"
2584 "when COMMAND completes.\n"
2585 "If no COMMAND is specified, turbostat wakes every 5-seconds\n"
2586 "to print statistics, until interrupted.\n"
2587 " -a, --add counter\n"
2588 " add a counter\n"
2589 " eg. --add msr0x10,u64,cpu,delta,MY_TSC\n"
2590 " eg. --add perf/cstate_pkg/c2-residency,package,delta,percent,perfPC2\n"
2591 " eg. --add pmt,name=XTAL,type=raw,domain=package0,offset=0,lsb=0,msb=63,guid=0x1a067102\n"
2592 " -c, --cpu cpu-set\n"
2593 " limit output to summary plus cpu-set:\n"
2594 " {core | package | j,k,l..m,n-p }\n"
2595 " -d, --debug\n"
2596 " displays usec, Time_Of_Day_Seconds and more debugging\n"
2597 " debug messages are printed to stderr\n"
2598 " -D, --Dump\n"
2599 " displays the raw counter values\n"
2600 " -e, --enable [all | column]\n"
2601 " shows all or the specified disabled column\n"
2602 " -f, --force\n"
2603 " force load turbostat with minimum default features on unsupported platforms.\n"
2604 " -H, --hide [column | column,column,...]\n"
2605 " hide the specified column(s)\n"
2606 " -i, --interval sec.subsec\n"
2607 " override default 5-second measurement interval\n"
2608 " -J, --Joules\n"
2609 " displays energy in Joules instead of Watts\n"
2610 " -l, --list\n"
2611 " list column headers only\n"
2612 " -M, --no-msr\n"
2613 " disable all uses of the MSR driver\n"
2614 " -P, --no-perf\n"
2615 " disable all uses of the perf API\n"
2616 " -n, --num_iterations num\n"
2617 " number of the measurement iterations\n"
2618 " -N, --header_iterations num\n"
2619 " print header every num iterations\n"
2620 " -o, --out file\n"
2621 " create or truncate \"file\" for all output\n"
2622 " -q, --quiet\n"
2623 " skip decoding system configuration header\n"
2624 " -s, --show [column | column,column,...]\n"
2625 " show only the specified column(s)\n"
2626 " -S, --Summary\n"
2627 " limits output to 1-line system summary per interval\n"
2628 " -T, --TCC temperature\n"
2629 " sets the Thermal Control Circuit temperature in\n"
2630 " degrees Celsius\n"
2631 " -h, --help\n"
2632 " print this help message\n"
2633 " -v, --version\n\t\tprint version information\n\nFor more help, run \"man turbostat\"\n");
2634 }
2635
2636 /*
2637 * bic_lookup
2638 * for all the strings in comma separate name_list,
2639 * set the approprate bit in return value.
2640 */
bic_lookup(cpu_set_t * ret_set,char * name_list,enum show_hide_mode mode)2641 void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode)
2642 {
2643 unsigned int i;
2644
2645 while (name_list) {
2646 char *comma;
2647
2648 comma = strchr(name_list, ',');
2649
2650 if (comma)
2651 *comma = '\0';
2652
2653 for (i = 0; i < MAX_BIC; ++i) {
2654 if (!strcmp(name_list, bic[i].name)) {
2655 SET_BIC(i, ret_set);
2656 break;
2657 }
2658 if (!strcmp(name_list, "all")) {
2659 bic_set_all(ret_set);
2660 break;
2661 } else if (!strcmp(name_list, "topology")) {
2662 CPU_OR(ret_set, ret_set, &bic_group_topology);
2663 break;
2664 } else if (!strcmp(name_list, "power")) {
2665 CPU_OR(ret_set, ret_set, &bic_group_thermal_pwr);
2666 break;
2667 } else if (!strcmp(name_list, "idle")) {
2668 CPU_OR(ret_set, ret_set, &bic_group_idle);
2669 break;
2670 } else if (!strcmp(name_list, "cache")) {
2671 CPU_OR(ret_set, ret_set, &bic_group_cache);
2672 break;
2673 } else if (!strcmp(name_list, "llc")) {
2674 CPU_OR(ret_set, ret_set, &bic_group_cache);
2675 break;
2676 } else if (!strcmp(name_list, "swidle")) {
2677 CPU_OR(ret_set, ret_set, &bic_group_sw_idle);
2678 break;
2679 } else if (!strcmp(name_list, "sysfs")) { /* legacy compatibility */
2680 CPU_OR(ret_set, ret_set, &bic_group_sw_idle);
2681 break;
2682 } else if (!strcmp(name_list, "hwidle")) {
2683 CPU_OR(ret_set, ret_set, &bic_group_hw_idle);
2684 break;
2685 } else if (!strcmp(name_list, "frequency")) {
2686 CPU_OR(ret_set, ret_set, &bic_group_frequency);
2687 break;
2688 } else if (!strcmp(name_list, "other")) {
2689 CPU_OR(ret_set, ret_set, &bic_group_other);
2690 break;
2691 }
2692 }
2693 if (i == MAX_BIC) {
2694 if (mode == SHOW_LIST) {
2695 deferred_add_names[deferred_add_index++] = name_list;
2696 if (deferred_add_index >= MAX_DEFERRED) {
2697 fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", MAX_DEFERRED, name_list);
2698 help();
2699 exit(1);
2700 }
2701 } else {
2702 deferred_skip_names[deferred_skip_index++] = name_list;
2703 if (debug)
2704 fprintf(stderr, "deferred \"%s\"\n", name_list);
2705 if (deferred_skip_index >= MAX_DEFERRED) {
2706 fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", MAX_DEFERRED, name_list);
2707 help();
2708 exit(1);
2709 }
2710 }
2711 }
2712
2713 name_list = comma;
2714 if (name_list)
2715 name_list++;
2716
2717 }
2718 }
2719
2720 /*
2721 * print_name()
2722 * Print column header name for raw 64-bit counter in 16 columns (at least 8-char plus a tab)
2723 * Otherwise, allow the name + tab to fit within 8-coumn tab-stop.
2724 * In both cases, left justififed, just like other turbostat columns,
2725 * to allow the column values to consume the tab.
2726 *
2727 * Yes, 32-bit counters can overflow 8-columns, and
2728 * 64-bit counters can overflow 16-columns, but that is uncommon.
2729 */
print_name(int width,int * printed,char * delim,char * name,enum counter_type type,enum counter_format format)2730 static inline int print_name(int width, int *printed, char *delim, char *name, enum counter_type type, enum counter_format format)
2731 {
2732 UNUSED(type);
2733
2734 if (format == FORMAT_RAW && width >= 64)
2735 return (sprintf(outp, "%s%-8s", (*printed++ ? delim : ""), name));
2736 else
2737 return (sprintf(outp, "%s%s", (*printed++ ? delim : ""), name));
2738 }
2739
print_hex_value(int width,int * printed,char * delim,unsigned long long value)2740 static inline int print_hex_value(int width, int *printed, char *delim, unsigned long long value)
2741 {
2742 if (width <= 32)
2743 return (sprintf(outp, "%s%08x", (*printed++ ? delim : ""), (unsigned int)value));
2744 else
2745 return (sprintf(outp, "%s%016llx", (*printed++ ? delim : ""), value));
2746 }
2747
print_decimal_value(int width,int * printed,char * delim,unsigned long long value)2748 static inline int print_decimal_value(int width, int *printed, char *delim, unsigned long long value)
2749 {
2750 if (width <= 32)
2751 return (sprintf(outp, "%s%d", (*printed++ ? delim : ""), (unsigned int)value));
2752 else
2753 return (sprintf(outp, "%s%-8lld", (*printed++ ? delim : ""), value));
2754 }
2755
print_float_value(int * printed,char * delim,double value)2756 static inline int print_float_value(int *printed, char *delim, double value)
2757 {
2758 return (sprintf(outp, "%s%0.2f", (*printed++ ? delim : ""), value));
2759 }
2760
print_header(char * delim)2761 void print_header(char *delim)
2762 {
2763 struct msr_counter *mp;
2764 struct perf_counter_info *pp;
2765 struct pmt_counter *ppmt;
2766 int printed = 0;
2767
2768 if (DO_BIC(BIC_USEC))
2769 outp += sprintf(outp, "%susec", (printed++ ? delim : ""));
2770 if (DO_BIC(BIC_TOD))
2771 outp += sprintf(outp, "%sTime_Of_Day_Seconds", (printed++ ? delim : ""));
2772 if (DO_BIC(BIC_Package))
2773 outp += sprintf(outp, "%sPackage", (printed++ ? delim : ""));
2774 if (DO_BIC(BIC_Die))
2775 outp += sprintf(outp, "%sDie", (printed++ ? delim : ""));
2776 if (DO_BIC(BIC_L3))
2777 outp += sprintf(outp, "%sL3", (printed++ ? delim : ""));
2778 if (DO_BIC(BIC_Node))
2779 outp += sprintf(outp, "%sNode", (printed++ ? delim : ""));
2780 if (DO_BIC(BIC_Core))
2781 outp += sprintf(outp, "%sCore", (printed++ ? delim : ""));
2782 if (DO_BIC(BIC_CPU))
2783 outp += sprintf(outp, "%sCPU", (printed++ ? delim : ""));
2784 if (DO_BIC(BIC_APIC))
2785 outp += sprintf(outp, "%sAPIC", (printed++ ? delim : ""));
2786 if (DO_BIC(BIC_X2APIC))
2787 outp += sprintf(outp, "%sX2APIC", (printed++ ? delim : ""));
2788 if (DO_BIC(BIC_Avg_MHz))
2789 outp += sprintf(outp, "%sAvg_MHz", (printed++ ? delim : ""));
2790 if (DO_BIC(BIC_Busy))
2791 outp += sprintf(outp, "%sBusy%%", (printed++ ? delim : ""));
2792 if (DO_BIC(BIC_Bzy_MHz))
2793 outp += sprintf(outp, "%sBzy_MHz", (printed++ ? delim : ""));
2794 if (DO_BIC(BIC_TSC_MHz))
2795 outp += sprintf(outp, "%sTSC_MHz", (printed++ ? delim : ""));
2796
2797 if (DO_BIC(BIC_IPC))
2798 outp += sprintf(outp, "%sIPC", (printed++ ? delim : ""));
2799
2800 if (DO_BIC(BIC_IRQ)) {
2801 if (sums_need_wide_columns)
2802 outp += sprintf(outp, "%s IRQ", (printed++ ? delim : ""));
2803 else
2804 outp += sprintf(outp, "%sIRQ", (printed++ ? delim : ""));
2805 }
2806 if (DO_BIC(BIC_NMI)) {
2807 if (sums_need_wide_columns)
2808 outp += sprintf(outp, "%s NMI", (printed++ ? delim : ""));
2809 else
2810 outp += sprintf(outp, "%sNMI", (printed++ ? delim : ""));
2811 }
2812
2813 if (DO_BIC(BIC_SMI))
2814 outp += sprintf(outp, "%sSMI", (printed++ ? delim : ""));
2815
2816 if (DO_BIC(BIC_LLC_RPS))
2817 outp += sprintf(outp, "%sLLCkRPS", (printed++ ? delim : ""));
2818
2819 if (DO_BIC(BIC_LLC_HIT))
2820 outp += sprintf(outp, "%sLLC%%hit", (printed++ ? delim : ""));
2821
2822 for (mp = sys.tp; mp; mp = mp->next)
2823 outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format);
2824
2825 for (pp = sys.perf_tp; pp; pp = pp->next)
2826 outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format);
2827
2828 ppmt = sys.pmt_tp;
2829 while (ppmt) {
2830 switch (ppmt->type) {
2831 case PMT_TYPE_RAW:
2832 outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2833 break;
2834
2835 case PMT_TYPE_XTAL_TIME:
2836 case PMT_TYPE_TCORE_CLOCK:
2837 outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2838 break;
2839 }
2840
2841 ppmt = ppmt->next;
2842 }
2843
2844 if (DO_BIC(BIC_CPU_c1))
2845 outp += sprintf(outp, "%sCPU%%c1", (printed++ ? delim : ""));
2846 if (DO_BIC(BIC_CPU_c3))
2847 outp += sprintf(outp, "%sCPU%%c3", (printed++ ? delim : ""));
2848 if (DO_BIC(BIC_CPU_c6))
2849 outp += sprintf(outp, "%sCPU%%c6", (printed++ ? delim : ""));
2850 if (DO_BIC(BIC_CPU_c7))
2851 outp += sprintf(outp, "%sCPU%%c7", (printed++ ? delim : ""));
2852
2853 if (DO_BIC(BIC_Mod_c6))
2854 outp += sprintf(outp, "%sMod%%c6", (printed++ ? delim : ""));
2855
2856 if (DO_BIC(BIC_CoreTmp))
2857 outp += sprintf(outp, "%sCoreTmp", (printed++ ? delim : ""));
2858
2859 if (DO_BIC(BIC_CORE_THROT_CNT))
2860 outp += sprintf(outp, "%sCoreThr", (printed++ ? delim : ""));
2861
2862 if (valid_rapl_msrs && !rapl_joules) {
2863 if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
2864 outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : ""));
2865 } else if (valid_rapl_msrs && rapl_joules) {
2866 if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
2867 outp += sprintf(outp, "%sCor_J", (printed++ ? delim : ""));
2868 }
2869
2870 for (mp = sys.cp; mp; mp = mp->next)
2871 outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format);
2872
2873 for (pp = sys.perf_cp; pp; pp = pp->next)
2874 outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format);
2875
2876 ppmt = sys.pmt_cp;
2877 while (ppmt) {
2878 switch (ppmt->type) {
2879 case PMT_TYPE_RAW:
2880 outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2881
2882 break;
2883
2884 case PMT_TYPE_XTAL_TIME:
2885 case PMT_TYPE_TCORE_CLOCK:
2886 outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2887 break;
2888 }
2889
2890 ppmt = ppmt->next;
2891 }
2892 if (DO_BIC(BIC_PkgTmp))
2893 outp += sprintf(outp, "%sPkgTmp", (printed++ ? delim : ""));
2894
2895 if (DO_BIC(BIC_GFX_rc6))
2896 outp += sprintf(outp, "%sGFX%%rc6", (printed++ ? delim : ""));
2897
2898 if (DO_BIC(BIC_GFXMHz))
2899 outp += sprintf(outp, "%sGFXMHz", (printed++ ? delim : ""));
2900
2901 if (DO_BIC(BIC_GFXACTMHz))
2902 outp += sprintf(outp, "%sGFXAMHz", (printed++ ? delim : ""));
2903
2904 if (DO_BIC(BIC_SAM_mc6))
2905 outp += sprintf(outp, "%sSAM%%mc6", (printed++ ? delim : ""));
2906
2907 if (DO_BIC(BIC_SAMMHz))
2908 outp += sprintf(outp, "%sSAMMHz", (printed++ ? delim : ""));
2909
2910 if (DO_BIC(BIC_SAMACTMHz))
2911 outp += sprintf(outp, "%sSAMAMHz", (printed++ ? delim : ""));
2912
2913 if (DO_BIC(BIC_Totl_c0))
2914 outp += sprintf(outp, "%sTotl%%C0", (printed++ ? delim : ""));
2915 if (DO_BIC(BIC_Any_c0))
2916 outp += sprintf(outp, "%sAny%%C0", (printed++ ? delim : ""));
2917 if (DO_BIC(BIC_GFX_c0))
2918 outp += sprintf(outp, "%sGFX%%C0", (printed++ ? delim : ""));
2919 if (DO_BIC(BIC_CPUGFX))
2920 outp += sprintf(outp, "%sCPUGFX%%", (printed++ ? delim : ""));
2921
2922 if (DO_BIC(BIC_Pkgpc2))
2923 outp += sprintf(outp, "%sPkg%%pc2", (printed++ ? delim : ""));
2924 if (DO_BIC(BIC_Pkgpc3))
2925 outp += sprintf(outp, "%sPkg%%pc3", (printed++ ? delim : ""));
2926 if (DO_BIC(BIC_Pkgpc6))
2927 outp += sprintf(outp, "%sPkg%%pc6", (printed++ ? delim : ""));
2928 if (DO_BIC(BIC_Pkgpc7))
2929 outp += sprintf(outp, "%sPkg%%pc7", (printed++ ? delim : ""));
2930 if (DO_BIC(BIC_Pkgpc8))
2931 outp += sprintf(outp, "%sPkg%%pc8", (printed++ ? delim : ""));
2932 if (DO_BIC(BIC_Pkgpc9))
2933 outp += sprintf(outp, "%sPkg%%pc9", (printed++ ? delim : ""));
2934 if (DO_BIC(BIC_Pkgpc10))
2935 outp += sprintf(outp, "%sPk%%pc10", (printed++ ? delim : ""));
2936 if (DO_BIC(BIC_Diec6))
2937 outp += sprintf(outp, "%sDie%%c6", (printed++ ? delim : ""));
2938 if (DO_BIC(BIC_CPU_LPI))
2939 outp += sprintf(outp, "%sCPU%%LPI", (printed++ ? delim : ""));
2940 if (DO_BIC(BIC_SYS_LPI))
2941 outp += sprintf(outp, "%sSYS%%LPI", (printed++ ? delim : ""));
2942
2943 if (!rapl_joules) {
2944 if (DO_BIC(BIC_PkgWatt))
2945 outp += sprintf(outp, "%sPkgWatt", (printed++ ? delim : ""));
2946 if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
2947 outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : ""));
2948 if (DO_BIC(BIC_GFXWatt))
2949 outp += sprintf(outp, "%sGFXWatt", (printed++ ? delim : ""));
2950 if (DO_BIC(BIC_RAMWatt))
2951 outp += sprintf(outp, "%sRAMWatt", (printed++ ? delim : ""));
2952 if (DO_BIC(BIC_PKG__))
2953 outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
2954 if (DO_BIC(BIC_RAM__))
2955 outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
2956 } else {
2957 if (DO_BIC(BIC_Pkg_J))
2958 outp += sprintf(outp, "%sPkg_J", (printed++ ? delim : ""));
2959 if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
2960 outp += sprintf(outp, "%sCor_J", (printed++ ? delim : ""));
2961 if (DO_BIC(BIC_GFX_J))
2962 outp += sprintf(outp, "%sGFX_J", (printed++ ? delim : ""));
2963 if (DO_BIC(BIC_RAM_J))
2964 outp += sprintf(outp, "%sRAM_J", (printed++ ? delim : ""));
2965 if (DO_BIC(BIC_PKG__))
2966 outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
2967 if (DO_BIC(BIC_RAM__))
2968 outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
2969 }
2970 if (DO_BIC(BIC_UNCORE_MHZ))
2971 outp += sprintf(outp, "%sUncMHz", (printed++ ? delim : ""));
2972
2973 for (mp = sys.pp; mp; mp = mp->next)
2974 outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format);
2975
2976 for (pp = sys.perf_pp; pp; pp = pp->next)
2977 outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format);
2978
2979 ppmt = sys.pmt_pp;
2980 while (ppmt) {
2981 switch (ppmt->type) {
2982 case PMT_TYPE_RAW:
2983 outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2984 break;
2985
2986 case PMT_TYPE_XTAL_TIME:
2987 case PMT_TYPE_TCORE_CLOCK:
2988 outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2989 break;
2990 }
2991
2992 ppmt = ppmt->next;
2993 }
2994
2995 if (DO_BIC(BIC_SysWatt))
2996 outp += sprintf(outp, "%sSysWatt", (printed++ ? delim : ""));
2997 if (DO_BIC(BIC_Sys_J))
2998 outp += sprintf(outp, "%sSys_J", (printed++ ? delim : ""));
2999
3000 outp += sprintf(outp, "\n");
3001 }
3002
3003 /*
3004 * pct()
3005 *
3006 * If absolute value is < 1.1, return percentage
3007 * otherwise, return nan
3008 *
3009 * return value is appropriate for printing percentages with %f
3010 * while flagging some obvious erroneous values.
3011 */
pct(double d)3012 double pct(double d)
3013 {
3014
3015 double abs = fabs(d);
3016
3017 if (abs < 1.10)
3018 return (100.0 * d);
3019 return nan("");
3020 }
3021
dump_counters(PER_THREAD_PARAMS)3022 int dump_counters(PER_THREAD_PARAMS)
3023 {
3024 int i;
3025 struct msr_counter *mp;
3026 struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even;
3027
3028 outp += sprintf(outp, "t %p, c %p, p %p\n", t, c, p);
3029
3030 if (t) {
3031 outp += sprintf(outp, "CPU: %d flags 0x%x\n", t->cpu_id, t->flags);
3032 outp += sprintf(outp, "TSC: %016llX\n", t->tsc);
3033 outp += sprintf(outp, "aperf: %016llX\n", t->aperf);
3034 outp += sprintf(outp, "mperf: %016llX\n", t->mperf);
3035 outp += sprintf(outp, "c1: %016llX\n", t->c1);
3036
3037 if (DO_BIC(BIC_IPC))
3038 outp += sprintf(outp, "IPC: %lld\n", t->instr_count);
3039
3040 if (DO_BIC(BIC_IRQ))
3041 outp += sprintf(outp, "IRQ: %lld\n", t->irq_count);
3042 if (DO_BIC(BIC_NMI))
3043 outp += sprintf(outp, "IRQ: %lld\n", t->nmi_count);
3044 if (DO_BIC(BIC_SMI))
3045 outp += sprintf(outp, "SMI: %d\n", t->smi_count);
3046
3047 outp += sprintf(outp, "LLC refs: %lld", t->llc.references);
3048 outp += sprintf(outp, "LLC miss: %lld", t->llc.misses);
3049 outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses) / t->llc.references));
3050
3051 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3052 outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path);
3053 }
3054 }
3055
3056 if (c && is_cpu_first_thread_in_core(t, c)) {
3057 outp += sprintf(outp, "core: %d\n", c->core_id);
3058 outp += sprintf(outp, "c3: %016llX\n", c->c3);
3059 outp += sprintf(outp, "c6: %016llX\n", c->c6);
3060 outp += sprintf(outp, "c7: %016llX\n", c->c7);
3061 outp += sprintf(outp, "DTS: %dC\n", c->core_temp_c);
3062 outp += sprintf(outp, "cpu_throt_count: %016llX\n", c->core_throt_cnt);
3063
3064 const unsigned long long energy_value = c->core_energy.raw_value * c->core_energy.scale;
3065 const double energy_scale = c->core_energy.scale;
3066
3067 if (c->core_energy.unit == RAPL_UNIT_JOULES)
3068 outp += sprintf(outp, "Joules: %0llX (scale: %lf)\n", energy_value, energy_scale);
3069
3070 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3071 outp += sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, c->counter[i], mp->sp->path);
3072 }
3073 outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us);
3074 }
3075
3076 if (p && is_cpu_first_core_in_package(t, p)) {
3077 outp += sprintf(outp, "package: %d\n", p->package_id);
3078
3079 outp += sprintf(outp, "Weighted cores: %016llX\n", p->pkg_wtd_core_c0);
3080 outp += sprintf(outp, "Any cores: %016llX\n", p->pkg_any_core_c0);
3081 outp += sprintf(outp, "Any GFX: %016llX\n", p->pkg_any_gfxe_c0);
3082 outp += sprintf(outp, "CPU + GFX: %016llX\n", p->pkg_both_core_gfxe_c0);
3083
3084 outp += sprintf(outp, "pc2: %016llX\n", p->pc2);
3085 if (DO_BIC(BIC_Pkgpc3))
3086 outp += sprintf(outp, "pc3: %016llX\n", p->pc3);
3087 if (DO_BIC(BIC_Pkgpc6))
3088 outp += sprintf(outp, "pc6: %016llX\n", p->pc6);
3089 if (DO_BIC(BIC_Pkgpc7))
3090 outp += sprintf(outp, "pc7: %016llX\n", p->pc7);
3091 outp += sprintf(outp, "pc8: %016llX\n", p->pc8);
3092 outp += sprintf(outp, "pc9: %016llX\n", p->pc9);
3093 outp += sprintf(outp, "pc10: %016llX\n", p->pc10);
3094 outp += sprintf(outp, "cpu_lpi: %016llX\n", p->cpu_lpi);
3095 outp += sprintf(outp, "sys_lpi: %016llX\n", p->sys_lpi);
3096 outp += sprintf(outp, "Joules PKG: %0llX\n", p->energy_pkg.raw_value);
3097 outp += sprintf(outp, "Joules COR: %0llX\n", p->energy_cores.raw_value);
3098 outp += sprintf(outp, "Joules GFX: %0llX\n", p->energy_gfx.raw_value);
3099 outp += sprintf(outp, "Joules RAM: %0llX\n", p->energy_dram.raw_value);
3100 outp += sprintf(outp, "Joules PSYS: %0llX\n", pplat_cnt->energy_psys.raw_value);
3101 outp += sprintf(outp, "Throttle PKG: %0llX\n", p->rapl_pkg_perf_status.raw_value);
3102 outp += sprintf(outp, "Throttle RAM: %0llX\n", p->rapl_dram_perf_status.raw_value);
3103 outp += sprintf(outp, "PTM: %dC\n", p->pkg_temp_c);
3104
3105 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3106 outp += sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, p->counter[i], mp->sp->path);
3107 }
3108 }
3109
3110 outp += sprintf(outp, "\n");
3111
3112 return 0;
3113 }
3114
rapl_counter_get_value(const struct rapl_counter * c,enum rapl_unit desired_unit,double interval)3115 double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desired_unit, double interval)
3116 {
3117 assert(desired_unit != RAPL_UNIT_INVALID);
3118
3119 /*
3120 * For now we don't expect anything other than joules,
3121 * so just simplify the logic.
3122 */
3123 assert(c->unit == RAPL_UNIT_JOULES);
3124
3125 const double scaled = c->raw_value * c->scale;
3126
3127 if (desired_unit == RAPL_UNIT_WATTS)
3128 return scaled / interval;
3129 return scaled;
3130 }
3131
get_perf_llc_stats(int cpu,struct llc_stats * llc)3132 void get_perf_llc_stats(int cpu, struct llc_stats *llc)
3133 {
3134 struct read_format {
3135 unsigned long long num_read;
3136 struct llc_stats llc;
3137 } r;
3138 const ssize_t expected_read_size = sizeof(r);
3139 ssize_t actual_read_size;
3140
3141 actual_read_size = read(fd_llc_percpu[cpu], &r, expected_read_size);
3142
3143 if (actual_read_size == -1)
3144 err(-1, "%s(cpu%d,) %d,,%ld\n", __func__, cpu, fd_llc_percpu[cpu], expected_read_size);
3145
3146 llc->references = r.llc.references;
3147 llc->misses = r.llc.misses;
3148 if (actual_read_size != expected_read_size)
3149 warn("%s: failed to read perf_data (req %zu act %zu)", __func__, expected_read_size, actual_read_size);
3150 }
3151
3152 /*
3153 * column formatting convention & formats
3154 */
format_counters(PER_THREAD_PARAMS)3155 int format_counters(PER_THREAD_PARAMS)
3156 {
3157 static int count;
3158
3159 struct platform_counters *pplat_cnt = NULL;
3160 double interval_float, tsc;
3161 char *fmt8 = "%s%.2f";
3162
3163 int i;
3164 struct msr_counter *mp;
3165 struct perf_counter_info *pp;
3166 struct pmt_counter *ppmt;
3167 char *delim = "\t";
3168 int printed = 0;
3169
3170 if (t == &average.threads) {
3171 pplat_cnt = count & 1 ? &platform_counters_odd : &platform_counters_even;
3172 ++count;
3173 }
3174
3175 /* if showing only 1st thread in core and this isn't one, bail out */
3176 if (show_core_only && !is_cpu_first_thread_in_core(t, c))
3177 return 0;
3178
3179 /* if showing only 1st thread in pkg and this isn't one, bail out */
3180 if (show_pkg_only && !is_cpu_first_core_in_package(t, p))
3181 return 0;
3182
3183 /*if not summary line and --cpu is used */
3184 if ((t != &average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset)))
3185 return 0;
3186
3187 if (DO_BIC(BIC_USEC)) {
3188 /* on each row, print how many usec each timestamp took to gather */
3189 struct timeval tv;
3190
3191 timersub(&t->tv_end, &t->tv_begin, &tv);
3192 outp += sprintf(outp, "%5ld\t", tv.tv_sec * 1000000 + tv.tv_usec);
3193 }
3194
3195 /* Time_Of_Day_Seconds: on each row, print sec.usec last timestamp taken */
3196 if (DO_BIC(BIC_TOD))
3197 outp += sprintf(outp, "%10ld.%06ld\t", t->tv_end.tv_sec, t->tv_end.tv_usec);
3198
3199 interval_float = t->tv_delta.tv_sec + t->tv_delta.tv_usec / 1000000.0;
3200
3201 tsc = t->tsc * tsc_tweak;
3202
3203 /* topo columns, print blanks on 1st (average) line */
3204 if (t == &average.threads) {
3205 if (DO_BIC(BIC_Package))
3206 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3207 if (DO_BIC(BIC_Die))
3208 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3209 if (DO_BIC(BIC_L3))
3210 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3211 if (DO_BIC(BIC_Node))
3212 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3213 if (DO_BIC(BIC_Core))
3214 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3215 if (DO_BIC(BIC_CPU))
3216 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3217 if (DO_BIC(BIC_APIC))
3218 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3219 if (DO_BIC(BIC_X2APIC))
3220 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3221 } else {
3222 if (DO_BIC(BIC_Package)) {
3223 if (p)
3224 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->package_id);
3225 else
3226 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3227 }
3228 if (DO_BIC(BIC_Die)) {
3229 if (c)
3230 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].die_id);
3231 else
3232 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3233 }
3234 if (DO_BIC(BIC_L3)) {
3235 if (c)
3236 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].l3_id);
3237 else
3238 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3239 }
3240 if (DO_BIC(BIC_Node)) {
3241 if (t)
3242 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id);
3243 else
3244 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3245 }
3246 if (DO_BIC(BIC_Core)) {
3247 if (c)
3248 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_id);
3249 else
3250 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3251 }
3252 if (DO_BIC(BIC_CPU))
3253 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->cpu_id);
3254 if (DO_BIC(BIC_APIC))
3255 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->apic_id);
3256 if (DO_BIC(BIC_X2APIC))
3257 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->x2apic_id);
3258 }
3259
3260 if (DO_BIC(BIC_Avg_MHz))
3261 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float);
3262
3263 if (DO_BIC(BIC_Busy))
3264 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf / tsc));
3265
3266 if (DO_BIC(BIC_Bzy_MHz)) {
3267 if (has_base_hz)
3268 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf);
3269 else
3270 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), tsc / units * t->aperf / t->mperf / interval_float);
3271 }
3272
3273 if (DO_BIC(BIC_TSC_MHz))
3274 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 * t->tsc / units / interval_float);
3275
3276 if (DO_BIC(BIC_IPC))
3277 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 1.0 * t->instr_count / t->aperf);
3278
3279 /* IRQ */
3280 if (DO_BIC(BIC_IRQ)) {
3281 if (sums_need_wide_columns)
3282 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->irq_count);
3283 else
3284 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->irq_count);
3285 }
3286
3287 /* NMI */
3288 if (DO_BIC(BIC_NMI)) {
3289 if (sums_need_wide_columns)
3290 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->nmi_count);
3291 else
3292 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->nmi_count);
3293 }
3294
3295 /* SMI */
3296 if (DO_BIC(BIC_SMI))
3297 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count);
3298
3299 /* LLC Stats */
3300 if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT)) {
3301 if (DO_BIC(BIC_LLC_RPS))
3302 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000);
3303
3304 if (DO_BIC(BIC_LLC_HIT))
3305 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses) / t->llc.references));
3306 }
3307
3308 /* Added Thread Counters */
3309 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3310 if (mp->format == FORMAT_RAW)
3311 outp += print_hex_value(mp->width, &printed, delim, t->counter[i]);
3312 else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3313 outp += print_decimal_value(mp->width, &printed, delim, t->counter[i]);
3314 else if (mp->format == FORMAT_PERCENT) {
3315 if (mp->type == COUNTER_USEC)
3316 outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000);
3317 else
3318 outp += print_float_value(&printed, delim, pct(t->counter[i] / tsc));
3319 }
3320 }
3321
3322 /* Added perf Thread Counters */
3323 for (i = 0, pp = sys.perf_tp; pp; ++i, pp = pp->next) {
3324 if (pp->format == FORMAT_RAW)
3325 outp += print_hex_value(pp->width, &printed, delim, t->perf_counter[i]);
3326 else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3327 outp += print_decimal_value(pp->width, &printed, delim, t->perf_counter[i]);
3328 else if (pp->format == FORMAT_PERCENT) {
3329 if (pp->type == COUNTER_USEC)
3330 outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000);
3331 else
3332 outp += print_float_value(&printed, delim, pct(t->perf_counter[i] / tsc));
3333 }
3334 }
3335
3336 /* Added PMT Thread Counters */
3337 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
3338 const unsigned long value_raw = t->pmt_counter[i];
3339 double value_converted;
3340 switch (ppmt->type) {
3341 case PMT_TYPE_RAW:
3342 outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, t->pmt_counter[i]);
3343 break;
3344
3345 case PMT_TYPE_XTAL_TIME:
3346 value_converted = pct(value_raw / crystal_hz / interval_float);
3347 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted);
3348 break;
3349
3350 case PMT_TYPE_TCORE_CLOCK:
3351 value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float);
3352 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted);
3353 }
3354 }
3355
3356 /* C1 */
3357 if (DO_BIC(BIC_CPU_c1))
3358 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1 / tsc));
3359
3360 /* print per-core data only for 1st thread in core */
3361 if (!is_cpu_first_thread_in_core(t, c))
3362 goto done;
3363
3364 if (DO_BIC(BIC_CPU_c3))
3365 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3 / tsc));
3366 if (DO_BIC(BIC_CPU_c6))
3367 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6 / tsc));
3368 if (DO_BIC(BIC_CPU_c7))
3369 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7 / tsc));
3370
3371 /* Mod%c6 */
3372 if (DO_BIC(BIC_Mod_c6))
3373 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us / tsc));
3374
3375 if (DO_BIC(BIC_CoreTmp))
3376 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c);
3377
3378 /* Core throttle count */
3379 if (DO_BIC(BIC_CORE_THROT_CNT))
3380 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt);
3381
3382 /* Added Core Counters */
3383 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3384 if (mp->format == FORMAT_RAW)
3385 outp += print_hex_value(mp->width, &printed, delim, c->counter[i]);
3386 else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3387 outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]);
3388 else if (mp->format == FORMAT_PERCENT)
3389 outp += print_float_value(&printed, delim, pct(c->counter[i] / tsc));
3390 }
3391
3392 /* Added perf Core counters */
3393 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3394 if (pp->format == FORMAT_RAW)
3395 outp += print_hex_value(pp->width, &printed, delim, c->perf_counter[i]);
3396 else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3397 outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]);
3398 else if (pp->format == FORMAT_PERCENT)
3399 outp += print_float_value(&printed, delim, pct(c->perf_counter[i] / tsc));
3400 }
3401
3402 /* Added PMT Core counters */
3403 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3404 const unsigned long value_raw = c->pmt_counter[i];
3405 double value_converted;
3406 switch (ppmt->type) {
3407 case PMT_TYPE_RAW:
3408 outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, c->pmt_counter[i]);
3409 break;
3410
3411 case PMT_TYPE_XTAL_TIME:
3412 value_converted = pct(value_raw / crystal_hz / interval_float);
3413 outp += print_float_value(&printed, delim, value_converted);
3414 break;
3415
3416 case PMT_TYPE_TCORE_CLOCK:
3417 value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float);
3418 outp += print_float_value(&printed, delim, value_converted);
3419 }
3420 }
3421
3422 if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
3423 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_WATTS, interval_float));
3424 if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
3425 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_JOULES, interval_float));
3426
3427 /* print per-package data only for 1st core in package */
3428 if (!is_cpu_first_core_in_package(t, p))
3429 goto done;
3430
3431 /* PkgTmp */
3432 if (DO_BIC(BIC_PkgTmp))
3433 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->pkg_temp_c);
3434
3435 /* GFXrc6 */
3436 if (DO_BIC(BIC_GFX_rc6)) {
3437 if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */
3438 outp += sprintf(outp, "%s**.**", (printed++ ? delim : ""));
3439 } else {
3440 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->gfx_rc6_ms / 10.0 / interval_float);
3441 }
3442 }
3443
3444 /* GFXMHz */
3445 if (DO_BIC(BIC_GFXMHz))
3446 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_mhz);
3447
3448 /* GFXACTMHz */
3449 if (DO_BIC(BIC_GFXACTMHz))
3450 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_act_mhz);
3451
3452 /* SAMmc6 */
3453 if (DO_BIC(BIC_SAM_mc6)) {
3454 if (p->sam_mc6_ms == -1) { /* detect GFX counter reset */
3455 outp += sprintf(outp, "%s**.**", (printed++ ? delim : ""));
3456 } else {
3457 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->sam_mc6_ms / 10.0 / interval_float);
3458 }
3459 }
3460
3461 /* SAMMHz */
3462 if (DO_BIC(BIC_SAMMHz))
3463 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_mhz);
3464
3465 /* SAMACTMHz */
3466 if (DO_BIC(BIC_SAMACTMHz))
3467 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_act_mhz);
3468
3469 /* Totl%C0, Any%C0 GFX%C0 CPUGFX% */
3470 if (DO_BIC(BIC_Totl_c0))
3471 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */
3472 if (DO_BIC(BIC_Any_c0))
3473 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0 / tsc));
3474 if (DO_BIC(BIC_GFX_c0))
3475 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0 / tsc));
3476 if (DO_BIC(BIC_CPUGFX))
3477 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0 / tsc));
3478
3479 if (DO_BIC(BIC_Pkgpc2))
3480 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2 / tsc));
3481 if (DO_BIC(BIC_Pkgpc3))
3482 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3 / tsc));
3483 if (DO_BIC(BIC_Pkgpc6))
3484 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6 / tsc));
3485 if (DO_BIC(BIC_Pkgpc7))
3486 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7 / tsc));
3487 if (DO_BIC(BIC_Pkgpc8))
3488 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8 / tsc));
3489 if (DO_BIC(BIC_Pkgpc9))
3490 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9 / tsc));
3491 if (DO_BIC(BIC_Pkgpc10))
3492 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10 / tsc));
3493
3494 if (DO_BIC(BIC_Diec6))
3495 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz / interval_float));
3496
3497 if (DO_BIC(BIC_CPU_LPI)) {
3498 if (p->cpu_lpi >= 0)
3499 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0 / interval_float));
3500 else
3501 outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
3502 }
3503 if (DO_BIC(BIC_SYS_LPI)) {
3504 if (p->sys_lpi >= 0)
3505 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0 / interval_float));
3506 else
3507 outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
3508 }
3509
3510 if (DO_BIC(BIC_PkgWatt))
3511 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_WATTS, interval_float));
3512 if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
3513 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_WATTS, interval_float));
3514 if (DO_BIC(BIC_GFXWatt))
3515 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_WATTS, interval_float));
3516 if (DO_BIC(BIC_RAMWatt))
3517 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_WATTS, interval_float));
3518 if (DO_BIC(BIC_Pkg_J))
3519 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_JOULES, interval_float));
3520 if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
3521 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_JOULES, interval_float));
3522 if (DO_BIC(BIC_GFX_J))
3523 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_JOULES, interval_float));
3524 if (DO_BIC(BIC_RAM_J))
3525 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float));
3526 if (DO_BIC(BIC_PKG__))
3527 outp +=
3528 sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float));
3529 if (DO_BIC(BIC_RAM__))
3530 outp +=
3531 sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float));
3532 /* UncMHz */
3533 if (DO_BIC(BIC_UNCORE_MHZ))
3534 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz);
3535
3536 /* Added Package Counters */
3537 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3538 if (mp->format == FORMAT_RAW)
3539 outp += print_hex_value(mp->width, &printed, delim, p->counter[i]);
3540 else if (mp->type == COUNTER_K2M)
3541 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->counter[i] / 1000);
3542 else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3543 outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]);
3544 else if (mp->format == FORMAT_PERCENT)
3545 outp += print_float_value(&printed, delim, pct(p->counter[i] / tsc));
3546 }
3547
3548 /* Added perf Package Counters */
3549 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3550 if (pp->format == FORMAT_RAW)
3551 outp += print_hex_value(pp->width, &printed, delim, p->perf_counter[i]);
3552 else if (pp->type == COUNTER_K2M)
3553 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000);
3554 else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3555 outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]);
3556 else if (pp->format == FORMAT_PERCENT)
3557 outp += print_float_value(&printed, delim, pct(p->perf_counter[i] / tsc));
3558 }
3559
3560 /* Added PMT Package Counters */
3561 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3562 const unsigned long value_raw = p->pmt_counter[i];
3563 double value_converted;
3564 switch (ppmt->type) {
3565 case PMT_TYPE_RAW:
3566 outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, p->pmt_counter[i]);
3567 break;
3568
3569 case PMT_TYPE_XTAL_TIME:
3570 value_converted = pct(value_raw / crystal_hz / interval_float);
3571 outp += print_float_value(&printed, delim, value_converted);
3572 break;
3573
3574 case PMT_TYPE_TCORE_CLOCK:
3575 value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float);
3576 outp += print_float_value(&printed, delim, value_converted);
3577 }
3578 }
3579
3580 if (DO_BIC(BIC_SysWatt) && (t == &average.threads))
3581 outp += sprintf(outp, fmt8, (printed++ ? delim : ""),
3582 rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_WATTS, interval_float));
3583 if (DO_BIC(BIC_Sys_J) && (t == &average.threads))
3584 outp += sprintf(outp, fmt8, (printed++ ? delim : ""),
3585 rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_JOULES, interval_float));
3586
3587 done:
3588 if (*(outp - 1) != '\n')
3589 outp += sprintf(outp, "\n");
3590
3591 return 0;
3592 }
3593
flush_output_stdout(void)3594 void flush_output_stdout(void)
3595 {
3596 FILE *filep;
3597
3598 if (outf == stderr)
3599 filep = stdout;
3600 else
3601 filep = outf;
3602
3603 fputs(output_buffer, filep);
3604 fflush(filep);
3605
3606 outp = output_buffer;
3607 }
3608
flush_output_stderr(void)3609 void flush_output_stderr(void)
3610 {
3611 fputs(output_buffer, outf);
3612 fflush(outf);
3613 outp = output_buffer;
3614 }
3615
format_all_counters(PER_THREAD_PARAMS)3616 void format_all_counters(PER_THREAD_PARAMS)
3617 {
3618 static int count;
3619
3620 if ((!count || (header_iterations && !(count % header_iterations))) || !summary_only)
3621 print_header("\t");
3622
3623 format_counters(&average.threads, &average.cores, &average.packages);
3624
3625 count++;
3626
3627 if (summary_only)
3628 return;
3629
3630 for_all_cpus(format_counters, t, c, p);
3631 }
3632
3633 #define DELTA_WRAP32(new, old) \
3634 old = ((((unsigned long long)new << 32) - ((unsigned long long)old << 32)) >> 32);
3635
delta_package(struct pkg_data * new,struct pkg_data * old)3636 int delta_package(struct pkg_data *new, struct pkg_data *old)
3637 {
3638 int i;
3639 struct msr_counter *mp;
3640 struct perf_counter_info *pp;
3641 struct pmt_counter *ppmt;
3642
3643 if (DO_BIC(BIC_Totl_c0))
3644 old->pkg_wtd_core_c0 = new->pkg_wtd_core_c0 - old->pkg_wtd_core_c0;
3645 if (DO_BIC(BIC_Any_c0))
3646 old->pkg_any_core_c0 = new->pkg_any_core_c0 - old->pkg_any_core_c0;
3647 if (DO_BIC(BIC_GFX_c0))
3648 old->pkg_any_gfxe_c0 = new->pkg_any_gfxe_c0 - old->pkg_any_gfxe_c0;
3649 if (DO_BIC(BIC_CPUGFX))
3650 old->pkg_both_core_gfxe_c0 = new->pkg_both_core_gfxe_c0 - old->pkg_both_core_gfxe_c0;
3651
3652 old->pc2 = new->pc2 - old->pc2;
3653 if (DO_BIC(BIC_Pkgpc3))
3654 old->pc3 = new->pc3 - old->pc3;
3655 if (DO_BIC(BIC_Pkgpc6))
3656 old->pc6 = new->pc6 - old->pc6;
3657 if (DO_BIC(BIC_Pkgpc7))
3658 old->pc7 = new->pc7 - old->pc7;
3659 old->pc8 = new->pc8 - old->pc8;
3660 old->pc9 = new->pc9 - old->pc9;
3661 old->pc10 = new->pc10 - old->pc10;
3662 old->die_c6 = new->die_c6 - old->die_c6;
3663 old->cpu_lpi = new->cpu_lpi - old->cpu_lpi;
3664 old->sys_lpi = new->sys_lpi - old->sys_lpi;
3665 old->pkg_temp_c = new->pkg_temp_c;
3666
3667 /* flag an error when rc6 counter resets/wraps */
3668 if (old->gfx_rc6_ms > new->gfx_rc6_ms)
3669 old->gfx_rc6_ms = -1;
3670 else
3671 old->gfx_rc6_ms = new->gfx_rc6_ms - old->gfx_rc6_ms;
3672
3673 old->uncore_mhz = new->uncore_mhz;
3674 old->gfx_mhz = new->gfx_mhz;
3675 old->gfx_act_mhz = new->gfx_act_mhz;
3676
3677 /* flag an error when mc6 counter resets/wraps */
3678 if (old->sam_mc6_ms > new->sam_mc6_ms)
3679 old->sam_mc6_ms = -1;
3680 else
3681 old->sam_mc6_ms = new->sam_mc6_ms - old->sam_mc6_ms;
3682
3683 old->sam_mhz = new->sam_mhz;
3684 old->sam_act_mhz = new->sam_act_mhz;
3685
3686 old->energy_pkg.raw_value = new->energy_pkg.raw_value - old->energy_pkg.raw_value;
3687 old->energy_cores.raw_value = new->energy_cores.raw_value - old->energy_cores.raw_value;
3688 old->energy_gfx.raw_value = new->energy_gfx.raw_value - old->energy_gfx.raw_value;
3689 old->energy_dram.raw_value = new->energy_dram.raw_value - old->energy_dram.raw_value;
3690 old->rapl_pkg_perf_status.raw_value = new->rapl_pkg_perf_status.raw_value - old->rapl_pkg_perf_status.raw_value;
3691 old->rapl_dram_perf_status.raw_value = new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value;
3692
3693 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3694 if (mp->format == FORMAT_RAW)
3695 old->counter[i] = new->counter[i];
3696 else if (mp->format == FORMAT_AVERAGE)
3697 old->counter[i] = new->counter[i];
3698 else
3699 old->counter[i] = new->counter[i] - old->counter[i];
3700 }
3701
3702 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3703 if (pp->format == FORMAT_RAW)
3704 old->perf_counter[i] = new->perf_counter[i];
3705 else if (pp->format == FORMAT_AVERAGE)
3706 old->perf_counter[i] = new->perf_counter[i];
3707 else
3708 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3709 }
3710
3711 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3712 if (ppmt->format == FORMAT_RAW)
3713 old->pmt_counter[i] = new->pmt_counter[i];
3714 else
3715 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3716 }
3717
3718 return 0;
3719 }
3720
delta_core(struct core_data * new,struct core_data * old)3721 void delta_core(struct core_data *new, struct core_data *old)
3722 {
3723 int i;
3724 struct msr_counter *mp;
3725 struct perf_counter_info *pp;
3726 struct pmt_counter *ppmt;
3727
3728 old->c3 = new->c3 - old->c3;
3729 old->c6 = new->c6 - old->c6;
3730 old->c7 = new->c7 - old->c7;
3731 old->core_temp_c = new->core_temp_c;
3732 old->core_throt_cnt = new->core_throt_cnt - old->core_throt_cnt;
3733 old->mc6_us = new->mc6_us - old->mc6_us;
3734
3735 DELTA_WRAP32(new->core_energy.raw_value, old->core_energy.raw_value);
3736
3737 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3738 if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE)
3739 old->counter[i] = new->counter[i];
3740 else
3741 old->counter[i] = new->counter[i] - old->counter[i];
3742 }
3743
3744 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3745 if (pp->format == FORMAT_RAW)
3746 old->perf_counter[i] = new->perf_counter[i];
3747 else
3748 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3749 }
3750
3751 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3752 if (ppmt->format == FORMAT_RAW)
3753 old->pmt_counter[i] = new->pmt_counter[i];
3754 else
3755 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3756 }
3757 }
3758
soft_c1_residency_display(int bic)3759 int soft_c1_residency_display(int bic)
3760 {
3761 if (!DO_BIC(BIC_CPU_c1) || platform->has_msr_core_c1_res)
3762 return 0;
3763
3764 return DO_BIC_READ(bic);
3765 }
3766
3767 /*
3768 * old = new - old
3769 */
delta_thread(struct thread_data * new,struct thread_data * old,struct core_data * core_delta)3770 int delta_thread(struct thread_data *new, struct thread_data *old, struct core_data *core_delta)
3771 {
3772 int i;
3773 struct msr_counter *mp;
3774 struct perf_counter_info *pp;
3775 struct pmt_counter *ppmt;
3776
3777 /* we run cpuid just the 1st time, copy the results */
3778 if (DO_BIC(BIC_APIC))
3779 new->apic_id = old->apic_id;
3780 if (DO_BIC(BIC_X2APIC))
3781 new->x2apic_id = old->x2apic_id;
3782
3783 /*
3784 * the timestamps from start of measurement interval are in "old"
3785 * the timestamp from end of measurement interval are in "new"
3786 * over-write old w/ new so we can print end of interval values
3787 */
3788
3789 timersub(&new->tv_begin, &old->tv_begin, &old->tv_delta);
3790 old->tv_begin = new->tv_begin;
3791 old->tv_end = new->tv_end;
3792
3793 old->tsc = new->tsc - old->tsc;
3794
3795 /* check for TSC < 1 Mcycles over interval */
3796 if (old->tsc < (1000 * 1000))
3797 errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n"
3798 "You can disable all c-states by booting with \"idle=poll\"\n" "or just the deep ones with \"processor.max_cstate=1\"");
3799
3800 old->c1 = new->c1 - old->c1;
3801
3802 if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz) || DO_BIC(BIC_IPC)
3803 || soft_c1_residency_display(BIC_Avg_MHz)) {
3804 if ((new->aperf > old->aperf) && (new->mperf > old->mperf)) {
3805 old->aperf = new->aperf - old->aperf;
3806 old->mperf = new->mperf - old->mperf;
3807 } else {
3808 return -1;
3809 }
3810 }
3811
3812 if (platform->has_msr_core_c1_res) {
3813 /*
3814 * Some models have a dedicated C1 residency MSR,
3815 * which should be more accurate than the derivation below.
3816 */
3817 } else {
3818 /*
3819 * As counter collection is not atomic,
3820 * it is possible for mperf's non-halted cycles + idle states
3821 * to exceed TSC's all cycles: show c1 = 0% in that case.
3822 */
3823 if ((old->mperf + core_delta->c3 + core_delta->c6 + core_delta->c7) > (old->tsc * tsc_tweak))
3824 old->c1 = 0;
3825 else {
3826 /* normal case, derive c1 */
3827 old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - core_delta->c6 - core_delta->c7;
3828 }
3829 }
3830
3831 if (old->mperf == 0) {
3832 if (debug > 1)
3833 fprintf(outf, "cpu%d MPERF 0!\n", old->cpu_id);
3834 old->mperf = 1; /* divide by 0 protection */
3835 }
3836
3837 if (DO_BIC(BIC_IPC))
3838 old->instr_count = new->instr_count - old->instr_count;
3839
3840 if (DO_BIC(BIC_IRQ))
3841 old->irq_count = new->irq_count - old->irq_count;
3842
3843 if (DO_BIC(BIC_NMI))
3844 old->nmi_count = new->nmi_count - old->nmi_count;
3845
3846 if (DO_BIC(BIC_SMI))
3847 old->smi_count = new->smi_count - old->smi_count;
3848
3849 if (DO_BIC(BIC_LLC_RPS))
3850 old->llc.references = new->llc.references - old->llc.references;
3851
3852 if (DO_BIC(BIC_LLC_HIT))
3853 old->llc.misses = new->llc.misses - old->llc.misses;
3854
3855 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3856 if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE)
3857 old->counter[i] = new->counter[i];
3858 else
3859 old->counter[i] = new->counter[i] - old->counter[i];
3860 }
3861
3862 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
3863 if (pp->format == FORMAT_RAW)
3864 old->perf_counter[i] = new->perf_counter[i];
3865 else
3866 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3867 }
3868
3869 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
3870 if (ppmt->format == FORMAT_RAW)
3871 old->pmt_counter[i] = new->pmt_counter[i];
3872 else
3873 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3874 }
3875
3876 return 0;
3877 }
3878
delta_cpu(struct thread_data * t,struct core_data * c,struct pkg_data * p,struct thread_data * t2,struct core_data * c2,struct pkg_data * p2)3879 int delta_cpu(struct thread_data *t, struct core_data *c, struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2)
3880 {
3881 int retval = 0;
3882
3883 /* calculate core delta only for 1st thread in core */
3884 if (is_cpu_first_thread_in_core(t, c))
3885 delta_core(c, c2);
3886
3887 /* always calculate thread delta */
3888 retval = delta_thread(t, t2, c2); /* c2 is core delta */
3889
3890 /* calculate package delta only for 1st core in package */
3891 if (is_cpu_first_core_in_package(t, p))
3892 retval |= delta_package(p, p2);
3893
3894 return retval;
3895 }
3896
delta_platform(struct platform_counters * new,struct platform_counters * old)3897 void delta_platform(struct platform_counters *new, struct platform_counters *old)
3898 {
3899 old->energy_psys.raw_value = new->energy_psys.raw_value - old->energy_psys.raw_value;
3900 }
3901
rapl_counter_clear(struct rapl_counter * c)3902 void rapl_counter_clear(struct rapl_counter *c)
3903 {
3904 c->raw_value = 0;
3905 c->scale = 0.0;
3906 c->unit = RAPL_UNIT_INVALID;
3907 }
3908
clear_counters(PER_THREAD_PARAMS)3909 void clear_counters(PER_THREAD_PARAMS)
3910 {
3911 int i;
3912 struct msr_counter *mp;
3913
3914 t->tv_begin.tv_sec = 0;
3915 t->tv_begin.tv_usec = 0;
3916 t->tv_end.tv_sec = 0;
3917 t->tv_end.tv_usec = 0;
3918 t->tv_delta.tv_sec = 0;
3919 t->tv_delta.tv_usec = 0;
3920
3921 t->tsc = 0;
3922 t->aperf = 0;
3923 t->mperf = 0;
3924 t->c1 = 0;
3925
3926 t->instr_count = 0;
3927
3928 t->irq_count = 0;
3929 t->nmi_count = 0;
3930 t->smi_count = 0;
3931
3932 t->llc.references = 0;
3933 t->llc.misses = 0;
3934
3935 c->c3 = 0;
3936 c->c6 = 0;
3937 c->c7 = 0;
3938 c->mc6_us = 0;
3939 c->core_temp_c = 0;
3940 rapl_counter_clear(&c->core_energy);
3941 c->core_throt_cnt = 0;
3942
3943 t->llc.references = 0;
3944 t->llc.misses = 0;
3945
3946 p->pkg_wtd_core_c0 = 0;
3947 p->pkg_any_core_c0 = 0;
3948 p->pkg_any_gfxe_c0 = 0;
3949 p->pkg_both_core_gfxe_c0 = 0;
3950
3951 p->pc2 = 0;
3952 if (DO_BIC(BIC_Pkgpc3))
3953 p->pc3 = 0;
3954 if (DO_BIC(BIC_Pkgpc6))
3955 p->pc6 = 0;
3956 if (DO_BIC(BIC_Pkgpc7))
3957 p->pc7 = 0;
3958 p->pc8 = 0;
3959 p->pc9 = 0;
3960 p->pc10 = 0;
3961 p->die_c6 = 0;
3962 p->cpu_lpi = 0;
3963 p->sys_lpi = 0;
3964
3965 rapl_counter_clear(&p->energy_pkg);
3966 rapl_counter_clear(&p->energy_dram);
3967 rapl_counter_clear(&p->energy_cores);
3968 rapl_counter_clear(&p->energy_gfx);
3969 rapl_counter_clear(&p->rapl_pkg_perf_status);
3970 rapl_counter_clear(&p->rapl_dram_perf_status);
3971 p->pkg_temp_c = 0;
3972
3973 p->gfx_rc6_ms = 0;
3974 p->uncore_mhz = 0;
3975 p->gfx_mhz = 0;
3976 p->gfx_act_mhz = 0;
3977 p->sam_mc6_ms = 0;
3978 p->sam_mhz = 0;
3979 p->sam_act_mhz = 0;
3980 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next)
3981 t->counter[i] = 0;
3982
3983 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next)
3984 c->counter[i] = 0;
3985
3986 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next)
3987 p->counter[i] = 0;
3988
3989 memset(&t->perf_counter[0], 0, sizeof(t->perf_counter));
3990 memset(&c->perf_counter[0], 0, sizeof(c->perf_counter));
3991 memset(&p->perf_counter[0], 0, sizeof(p->perf_counter));
3992
3993 memset(&t->pmt_counter[0], 0, ARRAY_SIZE(t->pmt_counter));
3994 memset(&c->pmt_counter[0], 0, ARRAY_SIZE(c->pmt_counter));
3995 memset(&p->pmt_counter[0], 0, ARRAY_SIZE(p->pmt_counter));
3996 }
3997
rapl_counter_accumulate(struct rapl_counter * dst,const struct rapl_counter * src)3998 void rapl_counter_accumulate(struct rapl_counter *dst, const struct rapl_counter *src)
3999 {
4000 /* Copy unit and scale from src if dst is not initialized */
4001 if (dst->unit == RAPL_UNIT_INVALID) {
4002 dst->unit = src->unit;
4003 dst->scale = src->scale;
4004 }
4005
4006 assert(dst->unit == src->unit);
4007 assert(dst->scale == src->scale);
4008
4009 dst->raw_value += src->raw_value;
4010 }
4011
sum_counters(PER_THREAD_PARAMS)4012 int sum_counters(PER_THREAD_PARAMS)
4013 {
4014 int i;
4015 struct msr_counter *mp;
4016 struct perf_counter_info *pp;
4017 struct pmt_counter *ppmt;
4018
4019 /* copy un-changing apic_id's */
4020 if (DO_BIC(BIC_APIC))
4021 average.threads.apic_id = t->apic_id;
4022 if (DO_BIC(BIC_X2APIC))
4023 average.threads.x2apic_id = t->x2apic_id;
4024
4025 /* remember first tv_begin */
4026 if (average.threads.tv_begin.tv_sec == 0)
4027 average.threads.tv_begin = procsysfs_tv_begin;
4028
4029 /* remember last tv_end */
4030 average.threads.tv_end = t->tv_end;
4031
4032 average.threads.tsc += t->tsc;
4033 average.threads.aperf += t->aperf;
4034 average.threads.mperf += t->mperf;
4035 average.threads.c1 += t->c1;
4036
4037 average.threads.instr_count += t->instr_count;
4038
4039 average.threads.irq_count += t->irq_count;
4040 average.threads.nmi_count += t->nmi_count;
4041 average.threads.smi_count += t->smi_count;
4042
4043 average.threads.llc.references += t->llc.references;
4044 average.threads.llc.misses += t->llc.misses;
4045
4046 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4047 if (mp->format == FORMAT_RAW)
4048 continue;
4049 average.threads.counter[i] += t->counter[i];
4050 }
4051
4052 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4053 if (pp->format == FORMAT_RAW)
4054 continue;
4055 average.threads.perf_counter[i] += t->perf_counter[i];
4056 }
4057
4058 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4059 average.threads.pmt_counter[i] += t->pmt_counter[i];
4060 }
4061
4062 /* sum per-core values only for 1st thread in core */
4063 if (!is_cpu_first_thread_in_core(t, c))
4064 return 0;
4065
4066 average.cores.c3 += c->c3;
4067 average.cores.c6 += c->c6;
4068 average.cores.c7 += c->c7;
4069 average.cores.mc6_us += c->mc6_us;
4070
4071 average.cores.core_temp_c = MAX(average.cores.core_temp_c, c->core_temp_c);
4072 average.cores.core_throt_cnt = MAX(average.cores.core_throt_cnt, c->core_throt_cnt);
4073
4074 rapl_counter_accumulate(&average.cores.core_energy, &c->core_energy);
4075
4076 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
4077 if (mp->format == FORMAT_RAW)
4078 continue;
4079 average.cores.counter[i] += c->counter[i];
4080 }
4081
4082 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
4083 if (pp->format == FORMAT_RAW)
4084 continue;
4085 average.cores.perf_counter[i] += c->perf_counter[i];
4086 }
4087
4088 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
4089 average.cores.pmt_counter[i] += c->pmt_counter[i];
4090 }
4091
4092 /* sum per-pkg values only for 1st core in pkg */
4093 if (!is_cpu_first_core_in_package(t, p))
4094 return 0;
4095
4096 if (DO_BIC(BIC_Totl_c0))
4097 average.packages.pkg_wtd_core_c0 += p->pkg_wtd_core_c0;
4098 if (DO_BIC(BIC_Any_c0))
4099 average.packages.pkg_any_core_c0 += p->pkg_any_core_c0;
4100 if (DO_BIC(BIC_GFX_c0))
4101 average.packages.pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0;
4102 if (DO_BIC(BIC_CPUGFX))
4103 average.packages.pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0;
4104
4105 average.packages.pc2 += p->pc2;
4106 if (DO_BIC(BIC_Pkgpc3))
4107 average.packages.pc3 += p->pc3;
4108 if (DO_BIC(BIC_Pkgpc6))
4109 average.packages.pc6 += p->pc6;
4110 if (DO_BIC(BIC_Pkgpc7))
4111 average.packages.pc7 += p->pc7;
4112 average.packages.pc8 += p->pc8;
4113 average.packages.pc9 += p->pc9;
4114 average.packages.pc10 += p->pc10;
4115 average.packages.die_c6 += p->die_c6;
4116
4117 average.packages.cpu_lpi = p->cpu_lpi;
4118 average.packages.sys_lpi = p->sys_lpi;
4119
4120 rapl_counter_accumulate(&average.packages.energy_pkg, &p->energy_pkg);
4121 rapl_counter_accumulate(&average.packages.energy_dram, &p->energy_dram);
4122 rapl_counter_accumulate(&average.packages.energy_cores, &p->energy_cores);
4123 rapl_counter_accumulate(&average.packages.energy_gfx, &p->energy_gfx);
4124
4125 average.packages.gfx_rc6_ms = p->gfx_rc6_ms;
4126 average.packages.uncore_mhz = p->uncore_mhz;
4127 average.packages.gfx_mhz = p->gfx_mhz;
4128 average.packages.gfx_act_mhz = p->gfx_act_mhz;
4129 average.packages.sam_mc6_ms = p->sam_mc6_ms;
4130 average.packages.sam_mhz = p->sam_mhz;
4131 average.packages.sam_act_mhz = p->sam_act_mhz;
4132
4133 average.packages.pkg_temp_c = MAX(average.packages.pkg_temp_c, p->pkg_temp_c);
4134
4135 rapl_counter_accumulate(&average.packages.rapl_pkg_perf_status, &p->rapl_pkg_perf_status);
4136 rapl_counter_accumulate(&average.packages.rapl_dram_perf_status, &p->rapl_dram_perf_status);
4137
4138 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
4139 if ((mp->format == FORMAT_RAW) && (topo.num_packages == 0))
4140 average.packages.counter[i] = p->counter[i];
4141 else
4142 average.packages.counter[i] += p->counter[i];
4143 }
4144
4145 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
4146 if ((pp->format == FORMAT_RAW) && (topo.num_packages == 0))
4147 average.packages.perf_counter[i] = p->perf_counter[i];
4148 else
4149 average.packages.perf_counter[i] += p->perf_counter[i];
4150 }
4151
4152 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
4153 average.packages.pmt_counter[i] += p->pmt_counter[i];
4154 }
4155
4156 return 0;
4157 }
4158
4159 /*
4160 * sum the counters for all cpus in the system
4161 * compute the weighted average
4162 */
compute_average(PER_THREAD_PARAMS)4163 void compute_average(PER_THREAD_PARAMS)
4164 {
4165 int i;
4166 struct msr_counter *mp;
4167 struct perf_counter_info *pp;
4168 struct pmt_counter *ppmt;
4169
4170 clear_counters(&average.threads, &average.cores, &average.packages);
4171
4172 for_all_cpus(sum_counters, t, c, p);
4173
4174 /* Use the global time delta for the average. */
4175 average.threads.tv_delta = tv_delta;
4176
4177 average.threads.tsc /= topo.allowed_cpus;
4178 average.threads.aperf /= topo.allowed_cpus;
4179 average.threads.mperf /= topo.allowed_cpus;
4180 average.threads.instr_count /= topo.allowed_cpus;
4181 average.threads.c1 /= topo.allowed_cpus;
4182
4183 if (average.threads.irq_count > 9999999)
4184 sums_need_wide_columns = 1;
4185 if (average.threads.nmi_count > 9999999)
4186 sums_need_wide_columns = 1;
4187
4188 average.cores.c3 /= topo.allowed_cores;
4189 average.cores.c6 /= topo.allowed_cores;
4190 average.cores.c7 /= topo.allowed_cores;
4191 average.cores.mc6_us /= topo.allowed_cores;
4192
4193 if (DO_BIC(BIC_Totl_c0))
4194 average.packages.pkg_wtd_core_c0 /= topo.allowed_packages;
4195 if (DO_BIC(BIC_Any_c0))
4196 average.packages.pkg_any_core_c0 /= topo.allowed_packages;
4197 if (DO_BIC(BIC_GFX_c0))
4198 average.packages.pkg_any_gfxe_c0 /= topo.allowed_packages;
4199 if (DO_BIC(BIC_CPUGFX))
4200 average.packages.pkg_both_core_gfxe_c0 /= topo.allowed_packages;
4201
4202 average.packages.pc2 /= topo.allowed_packages;
4203 if (DO_BIC(BIC_Pkgpc3))
4204 average.packages.pc3 /= topo.allowed_packages;
4205 if (DO_BIC(BIC_Pkgpc6))
4206 average.packages.pc6 /= topo.allowed_packages;
4207 if (DO_BIC(BIC_Pkgpc7))
4208 average.packages.pc7 /= topo.allowed_packages;
4209
4210 average.packages.pc8 /= topo.allowed_packages;
4211 average.packages.pc9 /= topo.allowed_packages;
4212 average.packages.pc10 /= topo.allowed_packages;
4213 average.packages.die_c6 /= topo.allowed_packages;
4214
4215 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4216 if (mp->format == FORMAT_RAW)
4217 continue;
4218 if (mp->type == COUNTER_ITEMS) {
4219 if (average.threads.counter[i] > 9999999)
4220 sums_need_wide_columns = 1;
4221 continue;
4222 }
4223 average.threads.counter[i] /= topo.allowed_cpus;
4224 }
4225 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
4226 if (mp->format == FORMAT_RAW)
4227 continue;
4228 if (mp->type == COUNTER_ITEMS) {
4229 if (average.cores.counter[i] > 9999999)
4230 sums_need_wide_columns = 1;
4231 }
4232 average.cores.counter[i] /= topo.allowed_cores;
4233 }
4234 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
4235 if (mp->format == FORMAT_RAW)
4236 continue;
4237 if (mp->type == COUNTER_ITEMS) {
4238 if (average.packages.counter[i] > 9999999)
4239 sums_need_wide_columns = 1;
4240 }
4241 average.packages.counter[i] /= topo.allowed_packages;
4242 }
4243
4244 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4245 if (pp->format == FORMAT_RAW)
4246 continue;
4247 if (pp->type == COUNTER_ITEMS) {
4248 if (average.threads.perf_counter[i] > 9999999)
4249 sums_need_wide_columns = 1;
4250 continue;
4251 }
4252 average.threads.perf_counter[i] /= topo.allowed_cpus;
4253 }
4254 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
4255 if (pp->format == FORMAT_RAW)
4256 continue;
4257 if (pp->type == COUNTER_ITEMS) {
4258 if (average.cores.perf_counter[i] > 9999999)
4259 sums_need_wide_columns = 1;
4260 }
4261 average.cores.perf_counter[i] /= topo.allowed_cores;
4262 }
4263 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
4264 if (pp->format == FORMAT_RAW)
4265 continue;
4266 if (pp->type == COUNTER_ITEMS) {
4267 if (average.packages.perf_counter[i] > 9999999)
4268 sums_need_wide_columns = 1;
4269 }
4270 average.packages.perf_counter[i] /= topo.allowed_packages;
4271 }
4272
4273 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4274 average.threads.pmt_counter[i] /= topo.allowed_cpus;
4275 }
4276 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
4277 average.cores.pmt_counter[i] /= topo.allowed_cores;
4278 }
4279 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
4280 average.packages.pmt_counter[i] /= topo.allowed_packages;
4281 }
4282 }
4283
rdtsc(void)4284 static unsigned long long rdtsc(void)
4285 {
4286 unsigned int low, high;
4287
4288 asm volatile ("rdtsc":"=a" (low), "=d"(high));
4289
4290 return low | ((unsigned long long)high) << 32;
4291 }
4292
4293 /*
4294 * Open a file, and exit on failure
4295 */
fopen_or_die(const char * path,const char * mode)4296 FILE *fopen_or_die(const char *path, const char *mode)
4297 {
4298 FILE *filep = fopen(path, mode);
4299
4300 if (!filep)
4301 err(1, "%s: open failed", path);
4302 return filep;
4303 }
4304
4305 /*
4306 * snapshot_sysfs_counter()
4307 *
4308 * return snapshot of given counter
4309 */
snapshot_sysfs_counter(char * path)4310 unsigned long long snapshot_sysfs_counter(char *path)
4311 {
4312 FILE *fp;
4313 int retval;
4314 unsigned long long counter;
4315
4316 fp = fopen_or_die(path, "r");
4317
4318 retval = fscanf(fp, "%lld", &counter);
4319 if (retval != 1)
4320 err(1, "snapshot_sysfs_counter(%s)", path);
4321
4322 fclose(fp);
4323
4324 return counter;
4325 }
4326
get_mp(int cpu,struct msr_counter * mp,unsigned long long * counterp,char * counter_path)4327 int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp, char *counter_path)
4328 {
4329 if (mp->msr_num != 0) {
4330 assert(!no_msr);
4331 if (get_msr(cpu, mp->msr_num, counterp))
4332 return -1;
4333 } else {
4334 char path[128 + PATH_BYTES];
4335
4336 if (mp->flags & SYSFS_PERCPU) {
4337 sprintf(path, "/sys/devices/system/cpu/cpu%d/%s", cpu, mp->sp->path);
4338
4339 *counterp = snapshot_sysfs_counter(path);
4340 } else {
4341 *counterp = snapshot_sysfs_counter(counter_path);
4342 }
4343 }
4344
4345 return 0;
4346 }
4347
get_legacy_uncore_mhz(int package)4348 unsigned long long get_legacy_uncore_mhz(int package)
4349 {
4350 char path[128];
4351 int die;
4352 static int warn_once;
4353
4354 /*
4355 * for this package, use the first die_id that exists
4356 */
4357 for (die = 0; die <= topo.max_die_id; ++die) {
4358
4359 sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", package, die);
4360
4361 if (access(path, R_OK) == 0)
4362 return (snapshot_sysfs_counter(path) / 1000);
4363 }
4364 if (!warn_once) {
4365 warnx("BUG: %s: No %s", __func__, path);
4366 warn_once = 1;
4367 }
4368
4369 return 0;
4370 }
4371
get_epb(int cpu)4372 int get_epb(int cpu)
4373 {
4374 char path[128 + PATH_BYTES];
4375 unsigned long long msr;
4376 int ret, epb = -1;
4377 FILE *fp;
4378
4379 sprintf(path, "/sys/devices/system/cpu/cpu%d/power/energy_perf_bias", cpu);
4380
4381 fp = fopen(path, "r");
4382 if (!fp)
4383 goto msr_fallback;
4384
4385 ret = fscanf(fp, "%d", &epb);
4386 if (ret != 1)
4387 err(1, "%s(%s)", __func__, path);
4388
4389 fclose(fp);
4390
4391 return epb;
4392
4393 msr_fallback:
4394 if (no_msr)
4395 return -1;
4396
4397 get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr);
4398
4399 return msr & 0xf;
4400 }
4401
get_apic_id(struct thread_data * t)4402 void get_apic_id(struct thread_data *t)
4403 {
4404 unsigned int eax, ebx, ecx, edx;
4405
4406 if (DO_BIC(BIC_APIC)) {
4407 eax = ebx = ecx = edx = 0;
4408 __cpuid(1, eax, ebx, ecx, edx);
4409
4410 t->apic_id = (ebx >> 24) & 0xff;
4411 }
4412
4413 if (!DO_BIC(BIC_X2APIC))
4414 return;
4415
4416 if (authentic_amd || hygon_genuine) {
4417 unsigned int topology_extensions;
4418
4419 if (max_extended_level < 0x8000001e)
4420 return;
4421
4422 eax = ebx = ecx = edx = 0;
4423 __cpuid(0x80000001, eax, ebx, ecx, edx);
4424 topology_extensions = ecx & (1 << 22);
4425
4426 if (topology_extensions == 0)
4427 return;
4428
4429 eax = ebx = ecx = edx = 0;
4430 __cpuid(0x8000001e, eax, ebx, ecx, edx);
4431
4432 t->x2apic_id = eax;
4433 return;
4434 }
4435
4436 if (!genuine_intel)
4437 return;
4438
4439 if (max_level < 0xb)
4440 return;
4441
4442 ecx = 0;
4443 __cpuid(0xb, eax, ebx, ecx, edx);
4444 t->x2apic_id = edx;
4445
4446 if (debug && (t->apic_id != (t->x2apic_id & 0xff)))
4447 fprintf(outf, "cpu%d: BIOS BUG: apic 0x%x x2apic 0x%x\n", t->cpu_id, t->apic_id, t->x2apic_id);
4448 }
4449
get_core_throt_cnt(int cpu,unsigned long long * cnt)4450 int get_core_throt_cnt(int cpu, unsigned long long *cnt)
4451 {
4452 char path[128 + PATH_BYTES];
4453 unsigned long long tmp;
4454 FILE *fp;
4455 int ret;
4456
4457 sprintf(path, "/sys/devices/system/cpu/cpu%d/thermal_throttle/core_throttle_count", cpu);
4458 fp = fopen(path, "r");
4459 if (!fp)
4460 return -1;
4461 ret = fscanf(fp, "%lld", &tmp);
4462 fclose(fp);
4463 if (ret != 1)
4464 return -1;
4465 *cnt = tmp;
4466
4467 return 0;
4468 }
4469
read_perf_counter_info(const char * const path,const char * const parse_format,void * value_ptr)4470 static int read_perf_counter_info(const char *const path, const char *const parse_format, void *value_ptr)
4471 {
4472 int fdmt;
4473 int bytes_read;
4474 char buf[64];
4475 int ret = -1;
4476
4477 fdmt = open(path, O_RDONLY, 0);
4478 if (fdmt == -1) {
4479 if (debug)
4480 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4481 ret = -1;
4482 goto cleanup_and_exit;
4483 }
4484
4485 bytes_read = read(fdmt, buf, sizeof(buf) - 1);
4486 if (bytes_read <= 0 || bytes_read >= (int)sizeof(buf)) {
4487 if (debug)
4488 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4489 ret = -1;
4490 goto cleanup_and_exit;
4491 }
4492
4493 buf[bytes_read] = '\0';
4494
4495 if (sscanf(buf, parse_format, value_ptr) != 1) {
4496 if (debug)
4497 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4498 ret = -1;
4499 goto cleanup_and_exit;
4500 }
4501
4502 ret = 0;
4503
4504 cleanup_and_exit:
4505 close(fdmt);
4506 return ret;
4507 }
4508
read_perf_counter_info_n(const char * const path,const char * const parse_format)4509 static unsigned int read_perf_counter_info_n(const char *const path, const char *const parse_format)
4510 {
4511 unsigned int v;
4512 int status;
4513
4514 status = read_perf_counter_info(path, parse_format, &v);
4515 if (status)
4516 v = -1;
4517
4518 return v;
4519 }
4520
read_perf_type(const char * subsys)4521 static unsigned int read_perf_type(const char *subsys)
4522 {
4523 const char *const path_format = "/sys/bus/event_source/devices/%s/type";
4524 const char *const format = "%u";
4525 char path[128];
4526
4527 snprintf(path, sizeof(path), path_format, subsys);
4528
4529 return read_perf_counter_info_n(path, format);
4530 }
4531
read_perf_config(const char * subsys,const char * event_name)4532 static unsigned int read_perf_config(const char *subsys, const char *event_name)
4533 {
4534 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s";
4535 FILE *fconfig = NULL;
4536 char path[128];
4537 char config_str[64];
4538 unsigned int config;
4539 unsigned int umask;
4540 bool has_config = false;
4541 bool has_umask = false;
4542 unsigned int ret = -1;
4543
4544 snprintf(path, sizeof(path), path_format, subsys, event_name);
4545
4546 fconfig = fopen(path, "r");
4547 if (!fconfig)
4548 return -1;
4549
4550 if (fgets(config_str, ARRAY_SIZE(config_str), fconfig) != config_str)
4551 goto cleanup_and_exit;
4552
4553 for (char *pconfig_str = &config_str[0]; pconfig_str;) {
4554 if (sscanf(pconfig_str, "event=%x", &config) == 1) {
4555 has_config = true;
4556 goto next;
4557 }
4558
4559 if (sscanf(pconfig_str, "umask=%x", &umask) == 1) {
4560 has_umask = true;
4561 goto next;
4562 }
4563
4564 next:
4565 pconfig_str = strchr(pconfig_str, ',');
4566 if (pconfig_str) {
4567 *pconfig_str = '\0';
4568 ++pconfig_str;
4569 }
4570 }
4571
4572 if (!has_umask)
4573 umask = 0;
4574
4575 if (has_config)
4576 ret = (umask << 8) | config;
4577
4578 cleanup_and_exit:
4579 fclose(fconfig);
4580 return ret;
4581 }
4582
read_perf_rapl_unit(const char * subsys,const char * event_name)4583 static unsigned int read_perf_rapl_unit(const char *subsys, const char *event_name)
4584 {
4585 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.unit";
4586 const char *const format = "%s";
4587 char path[128];
4588 char unit_buffer[16];
4589
4590 snprintf(path, sizeof(path), path_format, subsys, event_name);
4591
4592 read_perf_counter_info(path, format, &unit_buffer);
4593 if (strcmp("Joules", unit_buffer) == 0)
4594 return RAPL_UNIT_JOULES;
4595
4596 return RAPL_UNIT_INVALID;
4597 }
4598
read_perf_scale(const char * subsys,const char * event_name)4599 static double read_perf_scale(const char *subsys, const char *event_name)
4600 {
4601 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.scale";
4602 const char *const format = "%lf";
4603 char path[128];
4604 double scale;
4605
4606 snprintf(path, sizeof(path), path_format, subsys, event_name);
4607
4608 if (read_perf_counter_info(path, format, &scale))
4609 return 0.0;
4610
4611 return scale;
4612 }
4613
rapl_counter_info_count_perf(const struct rapl_counter_info_t * rci)4614 size_t rapl_counter_info_count_perf(const struct rapl_counter_info_t *rci)
4615 {
4616 size_t ret = 0;
4617
4618 for (int i = 0; i < NUM_RAPL_COUNTERS; ++i)
4619 if (rci->source[i] == COUNTER_SOURCE_PERF)
4620 ++ret;
4621
4622 return ret;
4623 }
4624
cstate_counter_info_count_perf(const struct cstate_counter_info_t * cci)4625 static size_t cstate_counter_info_count_perf(const struct cstate_counter_info_t *cci)
4626 {
4627 size_t ret = 0;
4628
4629 for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i)
4630 if (cci->source[i] == COUNTER_SOURCE_PERF)
4631 ++ret;
4632
4633 return ret;
4634 }
4635
write_rapl_counter(struct rapl_counter * rc,struct rapl_counter_info_t * rci,unsigned int idx)4636 void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci, unsigned int idx)
4637 {
4638 if (rci->source[idx] == COUNTER_SOURCE_NONE)
4639 return;
4640
4641 rc->raw_value = rci->data[idx];
4642 rc->unit = rci->unit[idx];
4643 rc->scale = rci->scale[idx];
4644 }
4645
get_rapl_counters(int cpu,unsigned int domain,struct core_data * c,struct pkg_data * p)4646 int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct pkg_data *p)
4647 {
4648 struct platform_counters *pplat_cnt = p == package_odd ? &platform_counters_odd : &platform_counters_even;
4649 unsigned long long perf_data[NUM_RAPL_COUNTERS + 1];
4650 struct rapl_counter_info_t *rci;
4651
4652 if (debug >= 2)
4653 fprintf(stderr, "%s: cpu%d domain%d\n", __func__, cpu, domain);
4654
4655 assert(rapl_counter_info_perdomain);
4656 assert(domain < rapl_counter_info_perdomain_size);
4657
4658 rci = &rapl_counter_info_perdomain[domain];
4659
4660 /*
4661 * If we have any perf counters to read, read them all now, in bulk
4662 */
4663 if (rci->fd_perf != -1) {
4664 size_t num_perf_counters = rapl_counter_info_count_perf(rci);
4665 const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
4666 const ssize_t actual_read_size = read(rci->fd_perf, &perf_data[0], sizeof(perf_data));
4667
4668 if (actual_read_size != expected_read_size)
4669 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size);
4670 }
4671
4672 for (unsigned int i = 0, pi = 1; i < NUM_RAPL_COUNTERS; ++i) {
4673 switch (rci->source[i]) {
4674 case COUNTER_SOURCE_NONE:
4675 rci->data[i] = 0;
4676 break;
4677
4678 case COUNTER_SOURCE_PERF:
4679 assert(pi < ARRAY_SIZE(perf_data));
4680 assert(rci->fd_perf != -1);
4681
4682 if (debug >= 2)
4683 fprintf(stderr, "Reading rapl counter via perf at %u (%llu %e %lf)\n",
4684 i, perf_data[pi], rci->scale[i], perf_data[pi] * rci->scale[i]);
4685
4686 rci->data[i] = perf_data[pi];
4687
4688 ++pi;
4689 break;
4690
4691 case COUNTER_SOURCE_MSR:
4692 if (debug >= 2)
4693 fprintf(stderr, "Reading rapl counter via msr at %u\n", i);
4694
4695 assert(!no_msr);
4696 if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM) {
4697 if (get_msr_sum(cpu, rci->msr[i], &rci->data[i]))
4698 return -13 - i;
4699 } else {
4700 if (get_msr(cpu, rci->msr[i], &rci->data[i]))
4701 return -13 - i;
4702 }
4703
4704 rci->data[i] &= rci->msr_mask[i];
4705 if (rci->msr_shift[i] >= 0)
4706 rci->data[i] >>= abs(rci->msr_shift[i]);
4707 else
4708 rci->data[i] <<= abs(rci->msr_shift[i]);
4709
4710 break;
4711 }
4712 }
4713
4714 BUILD_BUG_ON(NUM_RAPL_COUNTERS != 8);
4715 write_rapl_counter(&p->energy_pkg, rci, RAPL_RCI_INDEX_ENERGY_PKG);
4716 write_rapl_counter(&p->energy_cores, rci, RAPL_RCI_INDEX_ENERGY_CORES);
4717 write_rapl_counter(&p->energy_dram, rci, RAPL_RCI_INDEX_DRAM);
4718 write_rapl_counter(&p->energy_gfx, rci, RAPL_RCI_INDEX_GFX);
4719 write_rapl_counter(&p->rapl_pkg_perf_status, rci, RAPL_RCI_INDEX_PKG_PERF_STATUS);
4720 write_rapl_counter(&p->rapl_dram_perf_status, rci, RAPL_RCI_INDEX_DRAM_PERF_STATUS);
4721 write_rapl_counter(&c->core_energy, rci, RAPL_RCI_INDEX_CORE_ENERGY);
4722 write_rapl_counter(&pplat_cnt->energy_psys, rci, RAPL_RCI_INDEX_ENERGY_PLATFORM);
4723
4724 return 0;
4725 }
4726
find_sysfs_path_by_id(struct sysfs_path * sp,int id)4727 char *find_sysfs_path_by_id(struct sysfs_path *sp, int id)
4728 {
4729 while (sp) {
4730 if (sp->id == id)
4731 return (sp->path);
4732 sp = sp->next;
4733 }
4734 if (debug)
4735 warnx("%s: id%d not found", __func__, id);
4736 return NULL;
4737 }
4738
get_cstate_counters(unsigned int cpu,PER_THREAD_PARAMS)4739 int get_cstate_counters(unsigned int cpu, PER_THREAD_PARAMS)
4740 {
4741 /*
4742 * Overcommit memory a little bit here,
4743 * but skip calculating exact sizes for the buffers.
4744 */
4745 unsigned long long perf_data[NUM_CSTATE_COUNTERS];
4746 unsigned long long perf_data_core[NUM_CSTATE_COUNTERS + 1];
4747 unsigned long long perf_data_pkg[NUM_CSTATE_COUNTERS + 1];
4748
4749 struct cstate_counter_info_t *cci;
4750
4751 if (debug >= 2)
4752 fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
4753
4754 assert(ccstate_counter_info);
4755 assert(cpu <= ccstate_counter_info_size);
4756
4757 ZERO_ARRAY(perf_data);
4758 ZERO_ARRAY(perf_data_core);
4759 ZERO_ARRAY(perf_data_pkg);
4760
4761 cci = &ccstate_counter_info[cpu];
4762
4763 /*
4764 * If we have any perf counters to read, read them all now, in bulk
4765 */
4766 const size_t num_perf_counters = cstate_counter_info_count_perf(cci);
4767 ssize_t expected_read_size = num_perf_counters * sizeof(unsigned long long);
4768 ssize_t actual_read_size_core = 0, actual_read_size_pkg = 0;
4769
4770 if (cci->fd_perf_core != -1) {
4771 /* Each descriptor read begins with number of counters read. */
4772 expected_read_size += sizeof(unsigned long long);
4773
4774 actual_read_size_core = read(cci->fd_perf_core, &perf_data_core[0], sizeof(perf_data_core));
4775
4776 if (actual_read_size_core <= 0)
4777 err(-1, "%s: read perf %s: %ld", __func__, "core", actual_read_size_core);
4778 }
4779
4780 if (cci->fd_perf_pkg != -1) {
4781 /* Each descriptor read begins with number of counters read. */
4782 expected_read_size += sizeof(unsigned long long);
4783
4784 actual_read_size_pkg = read(cci->fd_perf_pkg, &perf_data_pkg[0], sizeof(perf_data_pkg));
4785
4786 if (actual_read_size_pkg <= 0)
4787 err(-1, "%s: read perf %s: %ld", __func__, "pkg", actual_read_size_pkg);
4788 }
4789
4790 const ssize_t actual_read_size_total = actual_read_size_core + actual_read_size_pkg;
4791
4792 if (actual_read_size_total != expected_read_size)
4793 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size_total);
4794
4795 /*
4796 * Copy ccstate and pcstate data into unified buffer.
4797 *
4798 * Skip first element from core and pkg buffers.
4799 * Kernel puts there how many counters were read.
4800 */
4801 const size_t num_core_counters = perf_data_core[0];
4802 const size_t num_pkg_counters = perf_data_pkg[0];
4803
4804 assert(num_perf_counters == num_core_counters + num_pkg_counters);
4805
4806 /* Copy ccstate perf data */
4807 memcpy(&perf_data[0], &perf_data_core[1], num_core_counters * sizeof(unsigned long long));
4808
4809 /* Copy pcstate perf data */
4810 memcpy(&perf_data[num_core_counters], &perf_data_pkg[1], num_pkg_counters * sizeof(unsigned long long));
4811
4812 for (unsigned int i = 0, pi = 0; i < NUM_CSTATE_COUNTERS; ++i) {
4813 switch (cci->source[i]) {
4814 case COUNTER_SOURCE_NONE:
4815 break;
4816
4817 case COUNTER_SOURCE_PERF:
4818 assert(pi < ARRAY_SIZE(perf_data));
4819 assert(cci->fd_perf_core != -1 || cci->fd_perf_pkg != -1);
4820
4821 if (debug >= 2)
4822 fprintf(stderr, "cstate via %s %u: %llu\n", "perf", i, perf_data[pi]);
4823
4824 cci->data[i] = perf_data[pi];
4825
4826 ++pi;
4827 break;
4828
4829 case COUNTER_SOURCE_MSR:
4830 assert(!no_msr);
4831 if (get_msr(cpu, cci->msr[i], &cci->data[i]))
4832 return -13 - i;
4833
4834 if (debug >= 2)
4835 fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", "msr", cci->msr[i], i, cci->data[i]);
4836
4837 break;
4838 }
4839 }
4840
4841 /*
4842 * Helper to write the data only if the source of
4843 * the counter for the current cpu is not none.
4844 *
4845 * Otherwise we would overwrite core data with 0 (default value),
4846 * when invoked for the thread sibling.
4847 */
4848 #define PERF_COUNTER_WRITE_DATA(out_counter, index) do { \
4849 if (cci->source[index] != COUNTER_SOURCE_NONE) \
4850 out_counter = cci->data[index]; \
4851 } while (0)
4852
4853 BUILD_BUG_ON(NUM_CSTATE_COUNTERS != 11);
4854
4855 PERF_COUNTER_WRITE_DATA(t->c1, CCSTATE_RCI_INDEX_C1_RESIDENCY);
4856 PERF_COUNTER_WRITE_DATA(c->c3, CCSTATE_RCI_INDEX_C3_RESIDENCY);
4857 PERF_COUNTER_WRITE_DATA(c->c6, CCSTATE_RCI_INDEX_C6_RESIDENCY);
4858 PERF_COUNTER_WRITE_DATA(c->c7, CCSTATE_RCI_INDEX_C7_RESIDENCY);
4859
4860 PERF_COUNTER_WRITE_DATA(p->pc2, PCSTATE_RCI_INDEX_C2_RESIDENCY);
4861 PERF_COUNTER_WRITE_DATA(p->pc3, PCSTATE_RCI_INDEX_C3_RESIDENCY);
4862 PERF_COUNTER_WRITE_DATA(p->pc6, PCSTATE_RCI_INDEX_C6_RESIDENCY);
4863 PERF_COUNTER_WRITE_DATA(p->pc7, PCSTATE_RCI_INDEX_C7_RESIDENCY);
4864 PERF_COUNTER_WRITE_DATA(p->pc8, PCSTATE_RCI_INDEX_C8_RESIDENCY);
4865 PERF_COUNTER_WRITE_DATA(p->pc9, PCSTATE_RCI_INDEX_C9_RESIDENCY);
4866 PERF_COUNTER_WRITE_DATA(p->pc10, PCSTATE_RCI_INDEX_C10_RESIDENCY);
4867
4868 #undef PERF_COUNTER_WRITE_DATA
4869
4870 return 0;
4871 }
4872
msr_counter_info_count_perf(const struct msr_counter_info_t * mci)4873 size_t msr_counter_info_count_perf(const struct msr_counter_info_t *mci)
4874 {
4875 size_t ret = 0;
4876
4877 for (int i = 0; i < NUM_MSR_COUNTERS; ++i)
4878 if (mci->source[i] == COUNTER_SOURCE_PERF)
4879 ++ret;
4880
4881 return ret;
4882 }
4883
get_smi_aperf_mperf(unsigned int cpu,struct thread_data * t)4884 int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t)
4885 {
4886 unsigned long long perf_data[NUM_MSR_COUNTERS + 1];
4887
4888 struct msr_counter_info_t *mci;
4889
4890 if (debug >= 2)
4891 fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
4892
4893 assert(msr_counter_info);
4894 assert(cpu <= msr_counter_info_size);
4895
4896 mci = &msr_counter_info[cpu];
4897
4898 ZERO_ARRAY(perf_data);
4899 ZERO_ARRAY(mci->data);
4900
4901 if (mci->fd_perf != -1) {
4902 const size_t num_perf_counters = msr_counter_info_count_perf(mci);
4903 const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
4904 const ssize_t actual_read_size = read(mci->fd_perf, &perf_data[0], sizeof(perf_data));
4905
4906 if (actual_read_size != expected_read_size)
4907 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size);
4908 }
4909
4910 for (unsigned int i = 0, pi = 1; i < NUM_MSR_COUNTERS; ++i) {
4911 switch (mci->source[i]) {
4912 case COUNTER_SOURCE_NONE:
4913 break;
4914
4915 case COUNTER_SOURCE_PERF:
4916 assert(pi < ARRAY_SIZE(perf_data));
4917 assert(mci->fd_perf != -1);
4918
4919 if (debug >= 2)
4920 fprintf(stderr, "Reading msr counter via perf at %u: %llu\n", i, perf_data[pi]);
4921
4922 mci->data[i] = perf_data[pi];
4923
4924 ++pi;
4925 break;
4926
4927 case COUNTER_SOURCE_MSR:
4928 assert(!no_msr);
4929
4930 if (get_msr(cpu, mci->msr[i], &mci->data[i]))
4931 return -2 - i;
4932
4933 mci->data[i] &= mci->msr_mask[i];
4934
4935 if (debug >= 2)
4936 fprintf(stderr, "Reading msr counter via msr at %u: %llu\n", i, mci->data[i]);
4937
4938 break;
4939 }
4940 }
4941
4942 BUILD_BUG_ON(NUM_MSR_COUNTERS != 3);
4943 t->aperf = mci->data[MSR_RCI_INDEX_APERF];
4944 t->mperf = mci->data[MSR_RCI_INDEX_MPERF];
4945 t->smi_count = mci->data[MSR_RCI_INDEX_SMI];
4946
4947 return 0;
4948 }
4949
perf_counter_info_read_values(struct perf_counter_info * pp,int cpu,unsigned long long * out,size_t out_size)4950 int perf_counter_info_read_values(struct perf_counter_info *pp, int cpu, unsigned long long *out, size_t out_size)
4951 {
4952 unsigned int domain;
4953 unsigned long long value;
4954 int fd_counter;
4955
4956 for (size_t i = 0; pp; ++i, pp = pp->next) {
4957 domain = cpu_to_domain(pp, cpu);
4958 assert(domain < pp->num_domains);
4959
4960 fd_counter = pp->fd_perf_per_domain[domain];
4961
4962 if (fd_counter == -1)
4963 continue;
4964
4965 if (read(fd_counter, &value, sizeof(value)) != sizeof(value))
4966 return 1;
4967
4968 assert(i < out_size);
4969 out[i] = value * pp->scale;
4970 }
4971
4972 return 0;
4973 }
4974
pmt_gen_value_mask(unsigned int lsb,unsigned int msb)4975 unsigned long pmt_gen_value_mask(unsigned int lsb, unsigned int msb)
4976 {
4977 unsigned long mask;
4978
4979 if (msb == 63)
4980 mask = 0xffffffffffffffff;
4981 else
4982 mask = ((1 << (msb + 1)) - 1);
4983
4984 mask -= (1 << lsb) - 1;
4985
4986 return mask;
4987 }
4988
pmt_read_counter(struct pmt_counter * ppmt,unsigned int domain_id)4989 unsigned long pmt_read_counter(struct pmt_counter *ppmt, unsigned int domain_id)
4990 {
4991 if (domain_id >= ppmt->num_domains)
4992 return 0;
4993
4994 const unsigned long *pmmio = ppmt->domains[domain_id].pcounter;
4995 const unsigned long value = pmmio ? *pmmio : 0;
4996 const unsigned long value_mask = pmt_gen_value_mask(ppmt->lsb, ppmt->msb);
4997 const unsigned long value_shift = ppmt->lsb;
4998
4999 return (value & value_mask) >> value_shift;
5000 }
5001
5002 /* Rapl domain enumeration helpers */
get_rapl_num_domains(void)5003 static inline int get_rapl_num_domains(void)
5004 {
5005 int num_packages = topo.max_package_id + 1;
5006 int num_cores_per_package;
5007 int num_cores;
5008
5009 if (!platform->has_per_core_rapl)
5010 return num_packages;
5011
5012 num_cores_per_package = topo.max_core_id + 1;
5013 num_cores = num_cores_per_package * num_packages;
5014
5015 return num_cores;
5016 }
5017
get_rapl_domain_id(int cpu)5018 static inline int get_rapl_domain_id(int cpu)
5019 {
5020 int nr_cores_per_package = topo.max_core_id + 1;
5021 int rapl_core_id;
5022
5023 if (!platform->has_per_core_rapl)
5024 return cpus[cpu].physical_package_id;
5025
5026 /* Compute the system-wide unique core-id for @cpu */
5027 rapl_core_id = cpus[cpu].physical_core_id;
5028 rapl_core_id += cpus[cpu].physical_package_id * nr_cores_per_package;
5029
5030 return rapl_core_id;
5031 }
5032
5033 /*
5034 * get_counters(...)
5035 * migrate to cpu
5036 * acquire and record local counters for that cpu
5037 */
get_counters(PER_THREAD_PARAMS)5038 int get_counters(PER_THREAD_PARAMS)
5039 {
5040 int cpu = t->cpu_id;
5041 unsigned long long msr;
5042 struct msr_counter *mp;
5043 struct pmt_counter *pp;
5044 int i;
5045 int status;
5046
5047 if (cpu_migrate(cpu)) {
5048 fprintf(outf, "%s: Could not migrate to CPU %d\n", __func__, cpu);
5049 return -1;
5050 }
5051
5052 gettimeofday(&t->tv_begin, (struct timezone *)NULL);
5053
5054 if (first_counter_read)
5055 get_apic_id(t);
5056
5057 t->tsc = rdtsc(); /* we are running on local CPU of interest */
5058
5059 get_smi_aperf_mperf(cpu, t);
5060
5061 if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT))
5062 get_perf_llc_stats(cpu, &t->llc);
5063
5064 if (DO_BIC(BIC_IPC))
5065 if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long))
5066 return -4;
5067
5068 if (DO_BIC(BIC_IRQ))
5069 t->irq_count = irqs_per_cpu[cpu];
5070 if (DO_BIC(BIC_NMI))
5071 t->nmi_count = nmi_per_cpu[cpu];
5072
5073 get_cstate_counters(cpu, t, c, p);
5074
5075 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
5076 if (get_mp(cpu, mp, &t->counter[i], mp->sp->path))
5077 return -10;
5078 }
5079
5080 if (perf_counter_info_read_values(sys.perf_tp, cpu, t->perf_counter, MAX_ADDED_THREAD_COUNTERS))
5081 return -10;
5082
5083 for (i = 0, pp = sys.pmt_tp; pp; i++, pp = pp->next)
5084 t->pmt_counter[i] = pmt_read_counter(pp, t->cpu_id);
5085
5086 /* collect core counters only for 1st thread in core */
5087 if (!is_cpu_first_thread_in_core(t, c))
5088 goto done;
5089
5090 if (platform->has_per_core_rapl) {
5091 status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p);
5092 if (status != 0)
5093 return status;
5094 }
5095
5096 if (DO_BIC(BIC_CPU_c7) && t->is_atom) {
5097 /*
5098 * For Atom CPUs that has core cstate deeper than c6,
5099 * MSR_CORE_C6_RESIDENCY returns residency of cc6 and deeper.
5100 * Minus CC7 (and deeper cstates) residency to get
5101 * accturate cc6 residency.
5102 */
5103 c->c6 -= c->c7;
5104 }
5105
5106 if (DO_BIC(BIC_Mod_c6))
5107 if (get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us))
5108 return -8;
5109
5110 if (DO_BIC(BIC_CoreTmp)) {
5111 if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
5112 return -9;
5113 c->core_temp_c = tj_max - ((msr >> 16) & 0x7F);
5114 }
5115
5116 if (DO_BIC(BIC_CORE_THROT_CNT))
5117 get_core_throt_cnt(cpu, &c->core_throt_cnt);
5118
5119 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
5120 if (get_mp(cpu, mp, &c->counter[i], mp->sp->path))
5121 return -10;
5122 }
5123
5124 if (perf_counter_info_read_values(sys.perf_cp, cpu, c->perf_counter, MAX_ADDED_CORE_COUNTERS))
5125 return -10;
5126
5127 for (i = 0, pp = sys.pmt_cp; pp; i++, pp = pp->next)
5128 c->pmt_counter[i] = pmt_read_counter(pp, c->core_id);
5129
5130 /* collect package counters only for 1st core in package */
5131 if (!is_cpu_first_core_in_package(t, p))
5132 goto done;
5133
5134 if (DO_BIC(BIC_Totl_c0)) {
5135 if (get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0))
5136 return -10;
5137 }
5138 if (DO_BIC(BIC_Any_c0)) {
5139 if (get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0))
5140 return -11;
5141 }
5142 if (DO_BIC(BIC_GFX_c0)) {
5143 if (get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0))
5144 return -12;
5145 }
5146 if (DO_BIC(BIC_CPUGFX)) {
5147 if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0))
5148 return -13;
5149 }
5150
5151 if (DO_BIC(BIC_CPU_LPI))
5152 p->cpu_lpi = cpuidle_cur_cpu_lpi_us;
5153 if (DO_BIC(BIC_SYS_LPI))
5154 p->sys_lpi = cpuidle_cur_sys_lpi_us;
5155
5156 if (!platform->has_per_core_rapl) {
5157 status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p);
5158 if (status != 0)
5159 return status;
5160 }
5161
5162 if (DO_BIC(BIC_PkgTmp)) {
5163 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
5164 return -17;
5165 p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F);
5166 }
5167
5168 if (DO_BIC(BIC_UNCORE_MHZ))
5169 p->uncore_mhz = get_legacy_uncore_mhz(p->package_id);
5170
5171 if (DO_BIC(BIC_GFX_rc6))
5172 p->gfx_rc6_ms = gfx_info[GFX_rc6].val_ull;
5173
5174 if (DO_BIC(BIC_GFXMHz))
5175 p->gfx_mhz = gfx_info[GFX_MHz].val;
5176
5177 if (DO_BIC(BIC_GFXACTMHz))
5178 p->gfx_act_mhz = gfx_info[GFX_ACTMHz].val;
5179
5180 if (DO_BIC(BIC_SAM_mc6))
5181 p->sam_mc6_ms = gfx_info[SAM_mc6].val_ull;
5182
5183 if (DO_BIC(BIC_SAMMHz))
5184 p->sam_mhz = gfx_info[SAM_MHz].val;
5185
5186 if (DO_BIC(BIC_SAMACTMHz))
5187 p->sam_act_mhz = gfx_info[SAM_ACTMHz].val;
5188
5189 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
5190 char *path = NULL;
5191
5192 if (mp->msr_num == 0) {
5193 path = find_sysfs_path_by_id(mp->sp, p->package_id);
5194 if (path == NULL) {
5195 warnx("%s: package_id %d not found", __func__, p->package_id);
5196 return -10;
5197 }
5198 }
5199 if (get_mp(cpu, mp, &p->counter[i], path))
5200 return -10;
5201 }
5202
5203 if (perf_counter_info_read_values(sys.perf_pp, cpu, p->perf_counter, MAX_ADDED_PACKAGE_COUNTERS))
5204 return -10;
5205
5206 for (i = 0, pp = sys.pmt_pp; pp; i++, pp = pp->next)
5207 p->pmt_counter[i] = pmt_read_counter(pp, p->package_id);
5208
5209 done:
5210 gettimeofday(&t->tv_end, (struct timezone *)NULL);
5211
5212 return 0;
5213 }
5214
5215 int pkg_cstate_limit = PCLUKN;
5216 char *pkg_cstate_limit_strings[] = { "unknown", "reserved", "pc0", "pc1", "pc2",
5217 "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited"
5218 };
5219
5220 int nhm_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5221 PCLRSV, PCLRSV
5222 };
5223
5224 int snb_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5225 PCLRSV, PCLRSV
5226 };
5227
5228 int hsw_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5229 PCLRSV, PCLRSV
5230 };
5231
5232 int slv_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5233 PCL__6, PCL__7
5234 };
5235
5236 int amt_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5237 PCLRSV, PCLRSV
5238 };
5239
5240 int phi_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5241 PCLRSV, PCLRSV
5242 };
5243
5244 int glm_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5245 PCLRSV, PCLRSV
5246 };
5247
5248 int skx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5249 PCLRSV, PCLRSV
5250 };
5251
5252 int icx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5253 PCLRSV, PCLRSV
5254 };
5255
probe_cst_limit(void)5256 void probe_cst_limit(void)
5257 {
5258 unsigned long long msr;
5259 int *pkg_cstate_limits;
5260
5261 if (!platform->has_nhm_msrs || no_msr)
5262 return;
5263
5264 switch (platform->cst_limit) {
5265 case CST_LIMIT_NHM:
5266 pkg_cstate_limits = nhm_pkg_cstate_limits;
5267 break;
5268 case CST_LIMIT_SNB:
5269 pkg_cstate_limits = snb_pkg_cstate_limits;
5270 break;
5271 case CST_LIMIT_HSW:
5272 pkg_cstate_limits = hsw_pkg_cstate_limits;
5273 break;
5274 case CST_LIMIT_SKX:
5275 pkg_cstate_limits = skx_pkg_cstate_limits;
5276 break;
5277 case CST_LIMIT_ICX:
5278 pkg_cstate_limits = icx_pkg_cstate_limits;
5279 break;
5280 case CST_LIMIT_SLV:
5281 pkg_cstate_limits = slv_pkg_cstate_limits;
5282 break;
5283 case CST_LIMIT_AMT:
5284 pkg_cstate_limits = amt_pkg_cstate_limits;
5285 break;
5286 case CST_LIMIT_KNL:
5287 pkg_cstate_limits = phi_pkg_cstate_limits;
5288 break;
5289 case CST_LIMIT_GMT:
5290 pkg_cstate_limits = glm_pkg_cstate_limits;
5291 break;
5292 default:
5293 return;
5294 }
5295
5296 get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
5297 pkg_cstate_limit = pkg_cstate_limits[msr & 0xF];
5298 }
5299
dump_platform_info(void)5300 static void dump_platform_info(void)
5301 {
5302 unsigned long long msr;
5303 unsigned int ratio;
5304
5305 if (!platform->has_nhm_msrs || no_msr)
5306 return;
5307
5308 get_msr(base_cpu, MSR_PLATFORM_INFO, &msr);
5309
5310 fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", base_cpu, msr);
5311
5312 ratio = (msr >> 40) & 0xFF;
5313 fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n", ratio, bclk, ratio * bclk);
5314
5315 ratio = (msr >> 8) & 0xFF;
5316 fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
5317 }
5318
dump_power_ctl(void)5319 static void dump_power_ctl(void)
5320 {
5321 unsigned long long msr;
5322
5323 if (!platform->has_nhm_msrs || no_msr)
5324 return;
5325
5326 get_msr(base_cpu, MSR_IA32_POWER_CTL, &msr);
5327 fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", base_cpu, msr, msr & 0x2 ? "EN" : "DIS");
5328
5329 /* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */
5330 if (platform->has_cst_prewake_bit)
5331 fprintf(outf, "C-state Pre-wake: %sabled\n", msr & 0x40000000 ? "DIS" : "EN");
5332
5333 return;
5334 }
5335
dump_turbo_ratio_limit2(void)5336 static void dump_turbo_ratio_limit2(void)
5337 {
5338 unsigned long long msr;
5339 unsigned int ratio;
5340
5341 get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT2, &msr);
5342
5343 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", base_cpu, msr);
5344
5345 ratio = (msr >> 8) & 0xFF;
5346 if (ratio)
5347 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 18 active cores\n", ratio, bclk, ratio * bclk);
5348
5349 ratio = (msr >> 0) & 0xFF;
5350 if (ratio)
5351 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 17 active cores\n", ratio, bclk, ratio * bclk);
5352 return;
5353 }
5354
dump_turbo_ratio_limit1(void)5355 static void dump_turbo_ratio_limit1(void)
5356 {
5357 unsigned long long msr;
5358 unsigned int ratio;
5359
5360 get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &msr);
5361
5362 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, msr);
5363
5364 ratio = (msr >> 56) & 0xFF;
5365 if (ratio)
5366 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 16 active cores\n", ratio, bclk, ratio * bclk);
5367
5368 ratio = (msr >> 48) & 0xFF;
5369 if (ratio)
5370 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 15 active cores\n", ratio, bclk, ratio * bclk);
5371
5372 ratio = (msr >> 40) & 0xFF;
5373 if (ratio)
5374 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 14 active cores\n", ratio, bclk, ratio * bclk);
5375
5376 ratio = (msr >> 32) & 0xFF;
5377 if (ratio)
5378 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 13 active cores\n", ratio, bclk, ratio * bclk);
5379
5380 ratio = (msr >> 24) & 0xFF;
5381 if (ratio)
5382 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 12 active cores\n", ratio, bclk, ratio * bclk);
5383
5384 ratio = (msr >> 16) & 0xFF;
5385 if (ratio)
5386 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 11 active cores\n", ratio, bclk, ratio * bclk);
5387
5388 ratio = (msr >> 8) & 0xFF;
5389 if (ratio)
5390 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 10 active cores\n", ratio, bclk, ratio * bclk);
5391
5392 ratio = (msr >> 0) & 0xFF;
5393 if (ratio)
5394 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 9 active cores\n", ratio, bclk, ratio * bclk);
5395 return;
5396 }
5397
dump_turbo_ratio_limits(int trl_msr_offset)5398 static void dump_turbo_ratio_limits(int trl_msr_offset)
5399 {
5400 unsigned long long msr, core_counts;
5401 int shift;
5402
5403 get_msr(base_cpu, trl_msr_offset, &msr);
5404 fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n",
5405 base_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr);
5406
5407 if (platform->trl_msrs & TRL_CORECOUNT) {
5408 get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts);
5409 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", base_cpu, core_counts);
5410 } else {
5411 core_counts = 0x0807060504030201;
5412 }
5413
5414 for (shift = 56; shift >= 0; shift -= 8) {
5415 unsigned int ratio, group_size;
5416
5417 ratio = (msr >> shift) & 0xFF;
5418 group_size = (core_counts >> shift) & 0xFF;
5419 if (ratio)
5420 fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio, bclk, ratio * bclk, group_size);
5421 }
5422
5423 return;
5424 }
5425
dump_atom_turbo_ratio_limits(void)5426 static void dump_atom_turbo_ratio_limits(void)
5427 {
5428 unsigned long long msr;
5429 unsigned int ratio;
5430
5431 get_msr(base_cpu, MSR_ATOM_CORE_RATIOS, &msr);
5432 fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF);
5433
5434 ratio = (msr >> 0) & 0x3F;
5435 if (ratio)
5436 fprintf(outf, "%d * %.1f = %.1f MHz minimum operating frequency\n", ratio, bclk, ratio * bclk);
5437
5438 ratio = (msr >> 8) & 0x3F;
5439 if (ratio)
5440 fprintf(outf, "%d * %.1f = %.1f MHz low frequency mode (LFM)\n", ratio, bclk, ratio * bclk);
5441
5442 ratio = (msr >> 16) & 0x3F;
5443 if (ratio)
5444 fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
5445
5446 get_msr(base_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr);
5447 fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", base_cpu, msr & 0xFFFFFFFF);
5448
5449 ratio = (msr >> 24) & 0x3F;
5450 if (ratio)
5451 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 4 active cores\n", ratio, bclk, ratio * bclk);
5452
5453 ratio = (msr >> 16) & 0x3F;
5454 if (ratio)
5455 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 3 active cores\n", ratio, bclk, ratio * bclk);
5456
5457 ratio = (msr >> 8) & 0x3F;
5458 if (ratio)
5459 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 2 active cores\n", ratio, bclk, ratio * bclk);
5460
5461 ratio = (msr >> 0) & 0x3F;
5462 if (ratio)
5463 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 1 active core\n", ratio, bclk, ratio * bclk);
5464 }
5465
dump_knl_turbo_ratio_limits(void)5466 static void dump_knl_turbo_ratio_limits(void)
5467 {
5468 const unsigned int buckets_no = 7;
5469
5470 unsigned long long msr;
5471 int delta_cores, delta_ratio;
5472 int i, b_nr;
5473 unsigned int cores[buckets_no];
5474 unsigned int ratio[buckets_no];
5475
5476 get_msr(base_cpu, MSR_TURBO_RATIO_LIMIT, &msr);
5477
5478 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", base_cpu, msr);
5479
5480 /*
5481 * Turbo encoding in KNL is as follows:
5482 * [0] -- Reserved
5483 * [7:1] -- Base value of number of active cores of bucket 1.
5484 * [15:8] -- Base value of freq ratio of bucket 1.
5485 * [20:16] -- +ve delta of number of active cores of bucket 2.
5486 * i.e. active cores of bucket 2 =
5487 * active cores of bucket 1 + delta
5488 * [23:21] -- Negative delta of freq ratio of bucket 2.
5489 * i.e. freq ratio of bucket 2 =
5490 * freq ratio of bucket 1 - delta
5491 * [28:24]-- +ve delta of number of active cores of bucket 3.
5492 * [31:29]-- -ve delta of freq ratio of bucket 3.
5493 * [36:32]-- +ve delta of number of active cores of bucket 4.
5494 * [39:37]-- -ve delta of freq ratio of bucket 4.
5495 * [44:40]-- +ve delta of number of active cores of bucket 5.
5496 * [47:45]-- -ve delta of freq ratio of bucket 5.
5497 * [52:48]-- +ve delta of number of active cores of bucket 6.
5498 * [55:53]-- -ve delta of freq ratio of bucket 6.
5499 * [60:56]-- +ve delta of number of active cores of bucket 7.
5500 * [63:61]-- -ve delta of freq ratio of bucket 7.
5501 */
5502
5503 b_nr = 0;
5504 cores[b_nr] = (msr & 0xFF) >> 1;
5505 ratio[b_nr] = (msr >> 8) & 0xFF;
5506
5507 for (i = 16; i < 64; i += 8) {
5508 delta_cores = (msr >> i) & 0x1F;
5509 delta_ratio = (msr >> (i + 5)) & 0x7;
5510
5511 cores[b_nr + 1] = cores[b_nr] + delta_cores;
5512 ratio[b_nr + 1] = ratio[b_nr] - delta_ratio;
5513 b_nr++;
5514 }
5515
5516 for (i = buckets_no - 1; i >= 0; i--)
5517 if (i > 0 ? ratio[i] != ratio[i - 1] : 1)
5518 fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio[i], bclk, ratio[i] * bclk, cores[i]);
5519 }
5520
dump_cst_cfg(void)5521 static void dump_cst_cfg(void)
5522 {
5523 unsigned long long msr;
5524
5525 if (!platform->has_nhm_msrs || no_msr)
5526 return;
5527
5528 get_msr(base_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
5529
5530 fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", base_cpu, msr);
5531
5532 fprintf(outf, " (%s%s%s%s%slocked, pkg-cstate-limit=%d (%s)",
5533 (msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "",
5534 (msr & SNB_C1_AUTO_UNDEMOTE) ? "UNdemote-C1, " : "",
5535 (msr & NHM_C3_AUTO_DEMOTE) ? "demote-C3, " : "",
5536 (msr & NHM_C1_AUTO_DEMOTE) ? "demote-C1, " : "",
5537 (msr & (1 << 15)) ? "" : "UN", (unsigned int)msr & 0xF, pkg_cstate_limit_strings[pkg_cstate_limit]);
5538
5539 #define AUTOMATIC_CSTATE_CONVERSION (1UL << 16)
5540 if (platform->has_cst_auto_convension) {
5541 fprintf(outf, ", automatic c-state conversion=%s", (msr & AUTOMATIC_CSTATE_CONVERSION) ? "on" : "off");
5542 }
5543
5544 fprintf(outf, ")\n");
5545
5546 return;
5547 }
5548
dump_config_tdp(void)5549 static void dump_config_tdp(void)
5550 {
5551 unsigned long long msr;
5552
5553 get_msr(base_cpu, MSR_CONFIG_TDP_NOMINAL, &msr);
5554 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", base_cpu, msr);
5555 fprintf(outf, " (base_ratio=%d)\n", (unsigned int)msr & 0xFF);
5556
5557 get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr);
5558 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", base_cpu, msr);
5559 if (msr) {
5560 fprintf(outf, "PKG_MIN_PWR_LVL1=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5561 fprintf(outf, "PKG_MAX_PWR_LVL1=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5562 fprintf(outf, "LVL1_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5563 fprintf(outf, "PKG_TDP_LVL1=%d", (unsigned int)(msr) & 0x7FFF);
5564 }
5565 fprintf(outf, ")\n");
5566
5567 get_msr(base_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr);
5568 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", base_cpu, msr);
5569 if (msr) {
5570 fprintf(outf, "PKG_MIN_PWR_LVL2=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5571 fprintf(outf, "PKG_MAX_PWR_LVL2=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5572 fprintf(outf, "LVL2_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5573 fprintf(outf, "PKG_TDP_LVL2=%d", (unsigned int)(msr) & 0x7FFF);
5574 }
5575 fprintf(outf, ")\n");
5576
5577 get_msr(base_cpu, MSR_CONFIG_TDP_CONTROL, &msr);
5578 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", base_cpu, msr);
5579 if ((msr) & 0x3)
5580 fprintf(outf, "TDP_LEVEL=%d ", (unsigned int)(msr) & 0x3);
5581 fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5582 fprintf(outf, ")\n");
5583
5584 get_msr(base_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr);
5585 fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", base_cpu, msr);
5586 fprintf(outf, "MAX_NON_TURBO_RATIO=%d", (unsigned int)(msr) & 0xFF);
5587 fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5588 fprintf(outf, ")\n");
5589 }
5590
5591 unsigned int irtl_time_units[] = { 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };
5592
print_irtl(void)5593 void print_irtl(void)
5594 {
5595 unsigned long long msr;
5596
5597 if (!platform->has_irtl_msrs || no_msr)
5598 return;
5599
5600 if (platform->supported_cstates & PC3) {
5601 get_msr(base_cpu, MSR_PKGC3_IRTL, &msr);
5602 fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", base_cpu, msr);
5603 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5604 }
5605
5606 if (platform->supported_cstates & PC6) {
5607 get_msr(base_cpu, MSR_PKGC6_IRTL, &msr);
5608 fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", base_cpu, msr);
5609 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5610 }
5611
5612 if (platform->supported_cstates & PC7) {
5613 get_msr(base_cpu, MSR_PKGC7_IRTL, &msr);
5614 fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", base_cpu, msr);
5615 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5616 }
5617
5618 if (platform->supported_cstates & PC8) {
5619 get_msr(base_cpu, MSR_PKGC8_IRTL, &msr);
5620 fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", base_cpu, msr);
5621 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5622 }
5623
5624 if (platform->supported_cstates & PC9) {
5625 get_msr(base_cpu, MSR_PKGC9_IRTL, &msr);
5626 fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", base_cpu, msr);
5627 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5628 }
5629
5630 if (platform->supported_cstates & PC10) {
5631 get_msr(base_cpu, MSR_PKGC10_IRTL, &msr);
5632 fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", base_cpu, msr);
5633 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5634 }
5635 }
5636
free_fd_percpu(void)5637 void free_fd_percpu(void)
5638 {
5639 int i;
5640
5641 if (!fd_percpu)
5642 return;
5643
5644 for (i = 0; i < topo.max_cpu_num + 1; ++i) {
5645 if (fd_percpu[i] != 0)
5646 close(fd_percpu[i]);
5647 }
5648
5649 free(fd_percpu);
5650 fd_percpu = NULL;
5651 }
5652
free_fd_instr_count_percpu(void)5653 void free_fd_instr_count_percpu(void)
5654 {
5655 if (!fd_instr_count_percpu)
5656 return;
5657
5658 for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5659 if (fd_instr_count_percpu[i] != 0)
5660 close(fd_instr_count_percpu[i]);
5661 }
5662
5663 free(fd_instr_count_percpu);
5664 fd_instr_count_percpu = NULL;
5665 }
5666
free_fd_llc_percpu(void)5667 void free_fd_llc_percpu(void)
5668 {
5669 if (!fd_llc_percpu)
5670 return;
5671
5672 for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5673 if (fd_llc_percpu[i] != 0)
5674 close(fd_llc_percpu[i]);
5675 }
5676
5677 free(fd_llc_percpu);
5678 fd_llc_percpu = NULL;
5679 }
5680
free_fd_cstate(void)5681 void free_fd_cstate(void)
5682 {
5683 if (!ccstate_counter_info)
5684 return;
5685
5686 const int counter_info_num = ccstate_counter_info_size;
5687
5688 for (int counter_id = 0; counter_id < counter_info_num; ++counter_id) {
5689 if (ccstate_counter_info[counter_id].fd_perf_core != -1)
5690 close(ccstate_counter_info[counter_id].fd_perf_core);
5691
5692 if (ccstate_counter_info[counter_id].fd_perf_pkg != -1)
5693 close(ccstate_counter_info[counter_id].fd_perf_pkg);
5694 }
5695
5696 free(ccstate_counter_info);
5697 ccstate_counter_info = NULL;
5698 ccstate_counter_info_size = 0;
5699 }
5700
free_fd_msr(void)5701 void free_fd_msr(void)
5702 {
5703 if (!msr_counter_info)
5704 return;
5705
5706 for (int cpu = 0; cpu < topo.max_cpu_num; ++cpu) {
5707 if (msr_counter_info[cpu].fd_perf != -1)
5708 close(msr_counter_info[cpu].fd_perf);
5709 }
5710
5711 free(msr_counter_info);
5712 msr_counter_info = NULL;
5713 msr_counter_info_size = 0;
5714 }
5715
free_fd_rapl_percpu(void)5716 void free_fd_rapl_percpu(void)
5717 {
5718 if (!rapl_counter_info_perdomain)
5719 return;
5720
5721 const int num_domains = rapl_counter_info_perdomain_size;
5722
5723 for (int domain_id = 0; domain_id < num_domains; ++domain_id) {
5724 if (rapl_counter_info_perdomain[domain_id].fd_perf != -1)
5725 close(rapl_counter_info_perdomain[domain_id].fd_perf);
5726 }
5727
5728 free(rapl_counter_info_perdomain);
5729 rapl_counter_info_perdomain = NULL;
5730 rapl_counter_info_perdomain_size = 0;
5731 }
5732
free_fd_added_perf_counters_(struct perf_counter_info * pp)5733 void free_fd_added_perf_counters_(struct perf_counter_info *pp)
5734 {
5735 if (!pp)
5736 return;
5737
5738 if (!pp->fd_perf_per_domain)
5739 return;
5740
5741 while (pp) {
5742 for (size_t domain = 0; domain < pp->num_domains; ++domain) {
5743 if (pp->fd_perf_per_domain[domain] != -1) {
5744 close(pp->fd_perf_per_domain[domain]);
5745 pp->fd_perf_per_domain[domain] = -1;
5746 }
5747 }
5748
5749 free(pp->fd_perf_per_domain);
5750 pp->fd_perf_per_domain = NULL;
5751
5752 pp = pp->next;
5753 }
5754 }
5755
free_fd_added_perf_counters(void)5756 void free_fd_added_perf_counters(void)
5757 {
5758 free_fd_added_perf_counters_(sys.perf_tp);
5759 free_fd_added_perf_counters_(sys.perf_cp);
5760 free_fd_added_perf_counters_(sys.perf_pp);
5761 }
5762
free_all_buffers(void)5763 void free_all_buffers(void)
5764 {
5765 int i;
5766
5767 CPU_FREE(cpu_present_set);
5768 cpu_present_set = NULL;
5769 cpu_present_setsize = 0;
5770
5771 CPU_FREE(cpu_effective_set);
5772 cpu_effective_set = NULL;
5773 cpu_effective_setsize = 0;
5774
5775 CPU_FREE(cpu_allowed_set);
5776 cpu_allowed_set = NULL;
5777 cpu_allowed_setsize = 0;
5778
5779 CPU_FREE(cpu_affinity_set);
5780 cpu_affinity_set = NULL;
5781 cpu_affinity_setsize = 0;
5782
5783 free(thread_even);
5784 free(core_even);
5785 free(package_even);
5786
5787 thread_even = NULL;
5788 core_even = NULL;
5789 package_even = NULL;
5790
5791 free(thread_odd);
5792 free(core_odd);
5793 free(package_odd);
5794
5795 thread_odd = NULL;
5796 core_odd = NULL;
5797 package_odd = NULL;
5798
5799 free(output_buffer);
5800 output_buffer = NULL;
5801 outp = NULL;
5802
5803 free_fd_percpu();
5804 free_fd_instr_count_percpu();
5805 free_fd_llc_percpu();
5806 free_fd_msr();
5807 free_fd_rapl_percpu();
5808 free_fd_cstate();
5809 free_fd_added_perf_counters();
5810
5811 free(irq_column_2_cpu);
5812 free(irqs_per_cpu);
5813 free(nmi_per_cpu);
5814
5815 for (i = 0; i <= topo.max_cpu_num; ++i) {
5816 if (cpus[i].put_ids)
5817 CPU_FREE(cpus[i].put_ids);
5818 }
5819 free(cpus);
5820 }
5821
5822 /*
5823 * Parse a file containing a single int.
5824 * Return 0 if file can not be opened
5825 * Exit if file can be opened, but can not be parsed
5826 */
parse_int_file(const char * fmt,...)5827 int parse_int_file(const char *fmt, ...)
5828 {
5829 va_list args;
5830 char path[PATH_MAX];
5831 FILE *filep;
5832 int value;
5833
5834 va_start(args, fmt);
5835 vsnprintf(path, sizeof(path), fmt, args);
5836 va_end(args);
5837 filep = fopen(path, "r");
5838 if (!filep)
5839 return 0;
5840 if (fscanf(filep, "%d", &value) != 1)
5841 err(1, "%s: failed to parse number from file", path);
5842 fclose(filep);
5843 return value;
5844 }
5845
5846 /*
5847 * cpu_is_first_core_in_package(cpu)
5848 * return 1 if given CPU is 1st core in package
5849 */
cpu_is_first_core_in_package(int cpu)5850 int cpu_is_first_core_in_package(int cpu)
5851 {
5852 return cpu == parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_siblings_list", cpu);
5853 }
5854
get_physical_package_id(int cpu)5855 int get_physical_package_id(int cpu)
5856 {
5857 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu);
5858 }
5859
get_die_id(int cpu)5860 int get_die_id(int cpu)
5861 {
5862 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/die_id", cpu);
5863 }
5864
get_l3_id(int cpu)5865 int get_l3_id(int cpu)
5866 {
5867 return parse_int_file("/sys/devices/system/cpu/cpu%d/cache/index3/id", cpu);
5868 }
5869
get_core_id(int cpu)5870 int get_core_id(int cpu)
5871 {
5872 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_id", cpu);
5873 }
5874
set_node_data(void)5875 void set_node_data(void)
5876 {
5877 int pkg, node, lnode, cpu, cpux;
5878 int cpu_count;
5879
5880 /* initialize logical_node_id */
5881 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu)
5882 cpus[cpu].logical_node_id = -1;
5883
5884 cpu_count = 0;
5885 for (pkg = 0; pkg < topo.num_packages; pkg++) {
5886 lnode = 0;
5887 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
5888 if (cpus[cpu].physical_package_id != pkg)
5889 continue;
5890 /* find a cpu with an unset logical_node_id */
5891 if (cpus[cpu].logical_node_id != -1)
5892 continue;
5893 cpus[cpu].logical_node_id = lnode;
5894 node = cpus[cpu].physical_node_id;
5895 cpu_count++;
5896 /*
5897 * find all matching cpus on this pkg and set
5898 * the logical_node_id
5899 */
5900 for (cpux = cpu; cpux <= topo.max_cpu_num; cpux++) {
5901 if ((cpus[cpux].physical_package_id == pkg) && (cpus[cpux].physical_node_id == node)) {
5902 cpus[cpux].logical_node_id = lnode;
5903 cpu_count++;
5904 }
5905 }
5906 lnode++;
5907 if (lnode > topo.nodes_per_pkg)
5908 topo.nodes_per_pkg = lnode;
5909 }
5910 if (cpu_count >= topo.max_cpu_num)
5911 break;
5912 }
5913 }
5914
get_physical_node_id(struct cpu_topology * thiscpu)5915 int get_physical_node_id(struct cpu_topology *thiscpu)
5916 {
5917 char path[80];
5918 FILE *filep;
5919 int i;
5920 int cpu = thiscpu->logical_cpu_id;
5921
5922 for (i = 0; i <= topo.max_cpu_num; i++) {
5923 sprintf(path, "/sys/devices/system/cpu/cpu%d/node%i/cpulist", cpu, i);
5924 filep = fopen(path, "r");
5925 if (!filep)
5926 continue;
5927 fclose(filep);
5928 return i;
5929 }
5930 return -1;
5931 }
5932
parse_cpu_str(char * cpu_str,cpu_set_t * cpu_set,int cpu_set_size)5933 static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size)
5934 {
5935 unsigned int start, end;
5936 char *next = cpu_str;
5937
5938 while (next && *next) {
5939
5940 if (*next == '-') /* no negative cpu numbers */
5941 return 1;
5942
5943 if (*next == '\0' || *next == '\n')
5944 break;
5945
5946 start = strtoul(next, &next, 10);
5947
5948 if (start >= CPU_SUBSET_MAXCPUS)
5949 return 1;
5950 CPU_SET_S(start, cpu_set_size, cpu_set);
5951
5952 if (*next == '\0' || *next == '\n')
5953 break;
5954
5955 if (*next == ',') {
5956 next += 1;
5957 continue;
5958 }
5959
5960 if (*next == '-') {
5961 next += 1; /* start range */
5962 } else if (*next == '.') {
5963 next += 1;
5964 if (*next == '.')
5965 next += 1; /* start range */
5966 else
5967 return 1;
5968 }
5969
5970 end = strtoul(next, &next, 10);
5971 if (end <= start)
5972 return 1;
5973
5974 while (++start <= end) {
5975 if (start >= CPU_SUBSET_MAXCPUS)
5976 return 1;
5977 CPU_SET_S(start, cpu_set_size, cpu_set);
5978 }
5979
5980 if (*next == ',')
5981 next += 1;
5982 else if (*next != '\0' && *next != '\n')
5983 return 1;
5984 }
5985
5986 return 0;
5987 }
5988
get_thread_siblings(struct cpu_topology * thiscpu)5989 int get_thread_siblings(struct cpu_topology *thiscpu)
5990 {
5991 char path[80], character;
5992 FILE *filep;
5993 unsigned long map;
5994 int so, shift, sib_core;
5995 int cpu = thiscpu->logical_cpu_id;
5996 int offset = topo.max_cpu_num + 1;
5997 size_t size;
5998 int thread_id = 0;
5999
6000 thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1));
6001 if (thiscpu->thread_id < 0)
6002 thiscpu->thread_id = thread_id++;
6003 if (!thiscpu->put_ids)
6004 return -1;
6005
6006 size = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
6007 CPU_ZERO_S(size, thiscpu->put_ids);
6008
6009 sprintf(path, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", cpu);
6010 filep = fopen(path, "r");
6011
6012 if (!filep) {
6013 warnx("%s: open failed", path);
6014 return -1;
6015 }
6016 do {
6017 offset -= BITMASK_SIZE;
6018 if (fscanf(filep, "%lx%c", &map, &character) != 2)
6019 err(1, "%s: failed to parse file", path);
6020 for (shift = 0; shift < BITMASK_SIZE; shift++) {
6021 if ((map >> shift) & 0x1) {
6022 so = shift + offset;
6023 sib_core = get_core_id(so);
6024 if (sib_core == thiscpu->physical_core_id) {
6025 CPU_SET_S(so, size, thiscpu->put_ids);
6026 if ((so != cpu) && (cpus[so].thread_id < 0))
6027 cpus[so].thread_id = thread_id++;
6028 }
6029 }
6030 }
6031 } while (character == ',');
6032 fclose(filep);
6033
6034 return CPU_COUNT_S(size, thiscpu->put_ids);
6035 }
6036
6037 /*
6038 * run func(thread, core, package) in topology order
6039 * skip non-present cpus
6040 */
6041
for_all_cpus_2(int (func)(struct thread_data *,struct core_data *,struct pkg_data *,struct thread_data *,struct core_data *,struct pkg_data *),struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base,struct thread_data * thread_base2,struct core_data * core_base2,struct pkg_data * pkg_base2)6042 int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *,
6043 struct pkg_data *, struct thread_data *, struct core_data *,
6044 struct pkg_data *), struct thread_data *thread_base,
6045 struct core_data *core_base, struct pkg_data *pkg_base,
6046 struct thread_data *thread_base2, struct core_data *core_base2, struct pkg_data *pkg_base2)
6047 {
6048 int retval, pkg_no, node_no, core_no, thread_no;
6049
6050 retval = 0;
6051
6052 for (pkg_no = 0; pkg_no < topo.num_packages; ++pkg_no) {
6053 for (node_no = 0; node_no < topo.nodes_per_pkg; ++node_no) {
6054 for (core_no = 0; core_no < topo.cores_per_node; ++core_no) {
6055 for (thread_no = 0; thread_no < topo.threads_per_core; ++thread_no) {
6056 struct thread_data *t, *t2;
6057 struct core_data *c, *c2;
6058
6059 t = GET_THREAD(thread_base, thread_no, core_no, node_no, pkg_no);
6060
6061 if (cpu_is_not_allowed(t->cpu_id))
6062 continue;
6063
6064 t2 = GET_THREAD(thread_base2, thread_no, core_no, node_no, pkg_no);
6065
6066 c = GET_CORE(core_base, core_no, node_no, pkg_no);
6067 c2 = GET_CORE(core_base2, core_no, node_no, pkg_no);
6068
6069 retval |= func(t, c, &pkg_base[pkg_no], t2, c2, &pkg_base2[pkg_no]);
6070 }
6071 }
6072 }
6073 }
6074 return retval;
6075 }
6076
6077 /*
6078 * run func(cpu) on every cpu in /proc/stat
6079 * return max_cpu number
6080 */
for_all_proc_cpus(int (func)(int))6081 int for_all_proc_cpus(int (func) (int))
6082 {
6083 FILE *fp;
6084 int cpu_num;
6085 int retval;
6086
6087 fp = fopen_or_die(proc_stat, "r");
6088
6089 retval = fscanf(fp, "cpu %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n");
6090 if (retval != 0)
6091 err(1, "%s: failed to parse format", proc_stat);
6092
6093 while (1) {
6094 retval = fscanf(fp, "cpu%u %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n", &cpu_num);
6095 if (retval != 1)
6096 break;
6097
6098 retval = func(cpu_num);
6099 if (retval) {
6100 fclose(fp);
6101 return (retval);
6102 }
6103 }
6104 fclose(fp);
6105 return 0;
6106 }
6107
6108 #define PATH_EFFECTIVE_CPUS "/sys/fs/cgroup/cpuset.cpus.effective"
6109
6110 static char cpu_effective_str[1024];
6111
update_effective_str(bool startup)6112 static int update_effective_str(bool startup)
6113 {
6114 FILE *fp;
6115 char *pos;
6116 char buf[1024];
6117 int ret;
6118
6119 if (cpu_effective_str[0] == '\0' && !startup)
6120 return 0;
6121
6122 fp = fopen(PATH_EFFECTIVE_CPUS, "r");
6123 if (!fp)
6124 return 0;
6125
6126 pos = fgets(buf, 1024, fp);
6127 if (!pos)
6128 err(1, "%s: file read failed\n", PATH_EFFECTIVE_CPUS);
6129
6130 fclose(fp);
6131
6132 ret = strncmp(cpu_effective_str, buf, 1024);
6133 if (!ret)
6134 return 0;
6135
6136 strncpy(cpu_effective_str, buf, 1024);
6137 return 1;
6138 }
6139
update_effective_set(bool startup)6140 static void update_effective_set(bool startup)
6141 {
6142 update_effective_str(startup);
6143
6144 if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize))
6145 err(1, "%s: cpu str malformat %s\n", PATH_EFFECTIVE_CPUS, cpu_effective_str);
6146 }
6147
6148 void linux_perf_init(void);
6149 void msr_perf_init(void);
6150 void rapl_perf_init(void);
6151 void cstate_perf_init(void);
6152 void perf_llc_init(void);
6153 void added_perf_counters_init(void);
6154 void pmt_init(void);
6155
re_initialize(void)6156 void re_initialize(void)
6157 {
6158 free_all_buffers();
6159 setup_all_buffers(false);
6160 linux_perf_init();
6161 msr_perf_init();
6162 rapl_perf_init();
6163 cstate_perf_init();
6164 perf_llc_init();
6165 added_perf_counters_init();
6166 pmt_init();
6167 fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus);
6168 }
6169
set_max_cpu_num(void)6170 void set_max_cpu_num(void)
6171 {
6172 FILE *filep;
6173 int base_cpu;
6174 unsigned long dummy;
6175 char pathname[64];
6176
6177 base_cpu = sched_getcpu();
6178 if (base_cpu < 0)
6179 err(1, "cannot find calling cpu ID");
6180 sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", base_cpu);
6181
6182 filep = fopen_or_die(pathname, "r");
6183 topo.max_cpu_num = 0;
6184 while (fscanf(filep, "%lx,", &dummy) == 1)
6185 topo.max_cpu_num += BITMASK_SIZE;
6186 fclose(filep);
6187 topo.max_cpu_num--; /* 0 based */
6188 }
6189
6190 /*
6191 * count_cpus()
6192 * remember the last one seen, it will be the max
6193 */
count_cpus(int cpu)6194 int count_cpus(int cpu)
6195 {
6196 UNUSED(cpu);
6197
6198 topo.num_cpus++;
6199 return 0;
6200 }
6201
mark_cpu_present(int cpu)6202 int mark_cpu_present(int cpu)
6203 {
6204 CPU_SET_S(cpu, cpu_present_setsize, cpu_present_set);
6205 return 0;
6206 }
6207
init_thread_id(int cpu)6208 int init_thread_id(int cpu)
6209 {
6210 cpus[cpu].thread_id = -1;
6211 return 0;
6212 }
6213
set_my_cpu_type(void)6214 int set_my_cpu_type(void)
6215 {
6216 unsigned int eax, ebx, ecx, edx;
6217 unsigned int max_level;
6218
6219 __cpuid(0, max_level, ebx, ecx, edx);
6220
6221 if (max_level < CPUID_LEAF_MODEL_ID)
6222 return 0;
6223
6224 __cpuid(CPUID_LEAF_MODEL_ID, eax, ebx, ecx, edx);
6225
6226 return (eax >> CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT);
6227 }
6228
set_cpu_hybrid_type(int cpu)6229 int set_cpu_hybrid_type(int cpu)
6230 {
6231 if (cpu_migrate(cpu))
6232 return -1;
6233
6234 int type = set_my_cpu_type();
6235
6236 cpus[cpu].type = type;
6237 return 0;
6238 }
6239
6240 /*
6241 * snapshot_proc_interrupts()
6242 *
6243 * read and record summary of /proc/interrupts
6244 *
6245 * return 1 if config change requires a restart, else return 0
6246 */
snapshot_proc_interrupts(void)6247 int snapshot_proc_interrupts(void)
6248 {
6249 static FILE *fp;
6250 int column, retval;
6251
6252 if (fp == NULL)
6253 fp = fopen_or_die("/proc/interrupts", "r");
6254 else
6255 rewind(fp);
6256
6257 /* read 1st line of /proc/interrupts to get cpu* name for each column */
6258 for (column = 0; column < topo.num_cpus; ++column) {
6259 int cpu_number;
6260
6261 retval = fscanf(fp, " CPU%d", &cpu_number);
6262 if (retval != 1)
6263 break;
6264
6265 if (cpu_number > topo.max_cpu_num) {
6266 warn("/proc/interrupts: cpu%d: > %d", cpu_number, topo.max_cpu_num);
6267 return 1;
6268 }
6269
6270 irq_column_2_cpu[column] = cpu_number;
6271 irqs_per_cpu[cpu_number] = 0;
6272 nmi_per_cpu[cpu_number] = 0;
6273 }
6274
6275 /* read /proc/interrupt count lines and sum up irqs per cpu */
6276 while (1) {
6277 int column;
6278 char buf[64];
6279 int this_row_is_nmi = 0;
6280
6281 retval = fscanf(fp, " %s:", buf); /* irq# "N:" */
6282 if (retval != 1)
6283 break;
6284
6285 if (strncmp(buf, "NMI", strlen("NMI")) == 0)
6286 this_row_is_nmi = 1;
6287
6288 /* read the count per cpu */
6289 for (column = 0; column < topo.num_cpus; ++column) {
6290
6291 int cpu_number, irq_count;
6292
6293 retval = fscanf(fp, " %d", &irq_count);
6294
6295 if (retval != 1)
6296 break;
6297
6298 cpu_number = irq_column_2_cpu[column];
6299 irqs_per_cpu[cpu_number] += irq_count;
6300 if (this_row_is_nmi)
6301 nmi_per_cpu[cpu_number] += irq_count;
6302 }
6303 while (getc(fp) != '\n') ; /* flush interrupt description */
6304
6305 }
6306 return 0;
6307 }
6308
6309 /*
6310 * snapshot_graphics()
6311 *
6312 * record snapshot of specified graphics sysfs knob
6313 *
6314 * return 1 if config change requires a restart, else return 0
6315 */
snapshot_graphics(int idx)6316 int snapshot_graphics(int idx)
6317 {
6318 int retval;
6319
6320 rewind(gfx_info[idx].fp);
6321 fflush(gfx_info[idx].fp);
6322
6323 switch (idx) {
6324 case GFX_rc6:
6325 case SAM_mc6:
6326 retval = fscanf(gfx_info[idx].fp, "%lld", &gfx_info[idx].val_ull);
6327 if (retval != 1)
6328 err(1, "rc6");
6329 return 0;
6330 case GFX_MHz:
6331 case GFX_ACTMHz:
6332 case SAM_MHz:
6333 case SAM_ACTMHz:
6334 retval = fscanf(gfx_info[idx].fp, "%d", &gfx_info[idx].val);
6335 if (retval != 1)
6336 err(1, "MHz");
6337 return 0;
6338 default:
6339 return -EINVAL;
6340 }
6341 }
6342
6343 /*
6344 * snapshot_cpu_lpi()
6345 *
6346 * record snapshot of
6347 * /sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us
6348 */
snapshot_cpu_lpi_us(void)6349 int snapshot_cpu_lpi_us(void)
6350 {
6351 FILE *fp;
6352 int retval;
6353
6354 fp = fopen_or_die("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", "r");
6355
6356 retval = fscanf(fp, "%lld", &cpuidle_cur_cpu_lpi_us);
6357 if (retval != 1) {
6358 fprintf(stderr, "Disabling Low Power Idle CPU output\n");
6359 BIC_NOT_PRESENT(BIC_CPU_LPI);
6360 fclose(fp);
6361 return -1;
6362 }
6363
6364 fclose(fp);
6365
6366 return 0;
6367 }
6368
6369 /*
6370 * snapshot_sys_lpi()
6371 *
6372 * record snapshot of sys_lpi_file
6373 */
snapshot_sys_lpi_us(void)6374 int snapshot_sys_lpi_us(void)
6375 {
6376 FILE *fp;
6377 int retval;
6378
6379 fp = fopen_or_die(sys_lpi_file, "r");
6380
6381 retval = fscanf(fp, "%lld", &cpuidle_cur_sys_lpi_us);
6382 if (retval != 1) {
6383 fprintf(stderr, "Disabling Low Power Idle System output\n");
6384 BIC_NOT_PRESENT(BIC_SYS_LPI);
6385 fclose(fp);
6386 return -1;
6387 }
6388 fclose(fp);
6389
6390 return 0;
6391 }
6392
6393 /*
6394 * snapshot /proc and /sys files
6395 *
6396 * return 1 if configuration restart needed, else return 0
6397 */
snapshot_proc_sysfs_files(void)6398 int snapshot_proc_sysfs_files(void)
6399 {
6400 gettimeofday(&procsysfs_tv_begin, (struct timezone *)NULL);
6401
6402 if (DO_BIC(BIC_IRQ) || DO_BIC(BIC_NMI))
6403 if (snapshot_proc_interrupts())
6404 return 1;
6405
6406 if (DO_BIC(BIC_GFX_rc6))
6407 snapshot_graphics(GFX_rc6);
6408
6409 if (DO_BIC(BIC_GFXMHz))
6410 snapshot_graphics(GFX_MHz);
6411
6412 if (DO_BIC(BIC_GFXACTMHz))
6413 snapshot_graphics(GFX_ACTMHz);
6414
6415 if (DO_BIC(BIC_SAM_mc6))
6416 snapshot_graphics(SAM_mc6);
6417
6418 if (DO_BIC(BIC_SAMMHz))
6419 snapshot_graphics(SAM_MHz);
6420
6421 if (DO_BIC(BIC_SAMACTMHz))
6422 snapshot_graphics(SAM_ACTMHz);
6423
6424 if (DO_BIC(BIC_CPU_LPI))
6425 snapshot_cpu_lpi_us();
6426
6427 if (DO_BIC(BIC_SYS_LPI))
6428 snapshot_sys_lpi_us();
6429
6430 return 0;
6431 }
6432
6433 int exit_requested;
6434
signal_handler(int signal)6435 static void signal_handler(int signal)
6436 {
6437 switch (signal) {
6438 case SIGINT:
6439 exit_requested = 1;
6440 if (debug)
6441 fprintf(stderr, " SIGINT\n");
6442 break;
6443 case SIGUSR1:
6444 if (debug > 1)
6445 fprintf(stderr, "SIGUSR1\n");
6446 break;
6447 }
6448 }
6449
setup_signal_handler(void)6450 void setup_signal_handler(void)
6451 {
6452 struct sigaction sa;
6453
6454 memset(&sa, 0, sizeof(sa));
6455
6456 sa.sa_handler = &signal_handler;
6457
6458 if (sigaction(SIGINT, &sa, NULL) < 0)
6459 err(1, "sigaction SIGINT");
6460 if (sigaction(SIGUSR1, &sa, NULL) < 0)
6461 err(1, "sigaction SIGUSR1");
6462 }
6463
do_sleep(void)6464 void do_sleep(void)
6465 {
6466 struct timeval tout;
6467 struct timespec rest;
6468 fd_set readfds;
6469 int retval;
6470
6471 FD_ZERO(&readfds);
6472 FD_SET(0, &readfds);
6473
6474 if (ignore_stdin) {
6475 nanosleep(&interval_ts, NULL);
6476 return;
6477 }
6478
6479 tout = interval_tv;
6480 retval = select(1, &readfds, NULL, NULL, &tout);
6481
6482 if (retval == 1) {
6483 switch (getc(stdin)) {
6484 case 'q':
6485 exit_requested = 1;
6486 break;
6487 case EOF:
6488 /*
6489 * 'stdin' is a pipe closed on the other end. There
6490 * won't be any further input.
6491 */
6492 ignore_stdin = 1;
6493 /* Sleep the rest of the time */
6494 rest.tv_sec = (tout.tv_sec + tout.tv_usec / 1000000);
6495 rest.tv_nsec = (tout.tv_usec % 1000000) * 1000;
6496 nanosleep(&rest, NULL);
6497 }
6498 }
6499 }
6500
get_msr_sum(int cpu,off_t offset,unsigned long long * msr)6501 int get_msr_sum(int cpu, off_t offset, unsigned long long *msr)
6502 {
6503 int ret, idx;
6504 unsigned long long msr_cur, msr_last;
6505
6506 assert(!no_msr);
6507
6508 if (!per_cpu_msr_sum)
6509 return 1;
6510
6511 idx = offset_to_idx(offset);
6512 if (idx < 0)
6513 return idx;
6514 /* get_msr_sum() = sum + (get_msr() - last) */
6515 ret = get_msr(cpu, offset, &msr_cur);
6516 if (ret)
6517 return ret;
6518 msr_last = per_cpu_msr_sum[cpu].entries[idx].last;
6519 DELTA_WRAP32(msr_cur, msr_last);
6520 *msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum;
6521
6522 return 0;
6523 }
6524
6525 timer_t timerid;
6526
6527 /* Timer callback, update the sum of MSRs periodically. */
update_msr_sum(PER_THREAD_PARAMS)6528 static int update_msr_sum(PER_THREAD_PARAMS)
6529 {
6530 int i, ret;
6531 int cpu = t->cpu_id;
6532
6533 UNUSED(c);
6534 UNUSED(p);
6535
6536 assert(!no_msr);
6537
6538 for (i = IDX_PKG_ENERGY; i < IDX_COUNT; i++) {
6539 unsigned long long msr_cur, msr_last;
6540 off_t offset;
6541
6542 if (!idx_valid(i))
6543 continue;
6544 offset = idx_to_offset(i);
6545 if (offset < 0)
6546 continue;
6547 ret = get_msr(cpu, offset, &msr_cur);
6548 if (ret) {
6549 fprintf(outf, "Can not update msr(0x%llx)\n", (unsigned long long)offset);
6550 continue;
6551 }
6552
6553 msr_last = per_cpu_msr_sum[cpu].entries[i].last;
6554 per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff;
6555
6556 DELTA_WRAP32(msr_cur, msr_last);
6557 per_cpu_msr_sum[cpu].entries[i].sum += msr_last;
6558 }
6559 return 0;
6560 }
6561
msr_record_handler(union sigval v)6562 static void msr_record_handler(union sigval v)
6563 {
6564 UNUSED(v);
6565
6566 for_all_cpus(update_msr_sum, EVEN_COUNTERS);
6567 }
6568
msr_sum_record(void)6569 void msr_sum_record(void)
6570 {
6571 struct itimerspec its;
6572 struct sigevent sev;
6573
6574 per_cpu_msr_sum = calloc(topo.max_cpu_num + 1, sizeof(struct msr_sum_array));
6575 if (!per_cpu_msr_sum) {
6576 fprintf(outf, "Can not allocate memory for long time MSR.\n");
6577 return;
6578 }
6579 /*
6580 * Signal handler might be restricted, so use thread notifier instead.
6581 */
6582 memset(&sev, 0, sizeof(struct sigevent));
6583 sev.sigev_notify = SIGEV_THREAD;
6584 sev.sigev_notify_function = msr_record_handler;
6585
6586 sev.sigev_value.sival_ptr = &timerid;
6587 if (timer_create(CLOCK_REALTIME, &sev, &timerid) == -1) {
6588 fprintf(outf, "Can not create timer.\n");
6589 goto release_msr;
6590 }
6591
6592 its.it_value.tv_sec = 0;
6593 its.it_value.tv_nsec = 1;
6594 /*
6595 * A wraparound time has been calculated early.
6596 * Some sources state that the peak power for a
6597 * microprocessor is usually 1.5 times the TDP rating,
6598 * use 2 * TDP for safety.
6599 */
6600 its.it_interval.tv_sec = rapl_joule_counter_range / 2;
6601 its.it_interval.tv_nsec = 0;
6602
6603 if (timer_settime(timerid, 0, &its, NULL) == -1) {
6604 fprintf(outf, "Can not set timer.\n");
6605 goto release_timer;
6606 }
6607 return;
6608
6609 release_timer:
6610 timer_delete(timerid);
6611 release_msr:
6612 free(per_cpu_msr_sum);
6613 per_cpu_msr_sum = NULL;
6614 }
6615
6616 /*
6617 * set_my_sched_priority(pri)
6618 * return previous priority on success
6619 * return value < -20 on failure
6620 */
set_my_sched_priority(int priority)6621 int set_my_sched_priority(int priority)
6622 {
6623 int retval;
6624 int original_priority;
6625
6626 errno = 0;
6627 original_priority = getpriority(PRIO_PROCESS, 0);
6628 if (errno && (original_priority == -1))
6629 return -21;
6630
6631 retval = setpriority(PRIO_PROCESS, 0, priority);
6632 if (retval)
6633 return -21;
6634
6635 errno = 0;
6636 retval = getpriority(PRIO_PROCESS, 0);
6637 if (retval != priority)
6638 return -21;
6639
6640 return original_priority;
6641 }
6642
turbostat_loop()6643 void turbostat_loop()
6644 {
6645 int retval;
6646 int restarted = 0;
6647 unsigned int done_iters = 0;
6648
6649 setup_signal_handler();
6650
6651 /*
6652 * elevate own priority for interval mode
6653 *
6654 * ignore on error - we probably don't have permission to set it, but
6655 * it's not a big deal
6656 */
6657 set_my_sched_priority(-20);
6658
6659 restart:
6660 restarted++;
6661
6662 snapshot_proc_sysfs_files();
6663 retval = for_all_cpus(get_counters, EVEN_COUNTERS);
6664 first_counter_read = 0;
6665 if (retval < -1) {
6666 exit(retval);
6667 } else if (retval == -1) {
6668 if (restarted > 10) {
6669 exit(retval);
6670 }
6671 re_initialize();
6672 goto restart;
6673 }
6674 restarted = 0;
6675 done_iters = 0;
6676 gettimeofday(&tv_even, (struct timezone *)NULL);
6677
6678 while (1) {
6679 if (for_all_proc_cpus(cpu_is_not_present)) {
6680 re_initialize();
6681 goto restart;
6682 }
6683 if (update_effective_str(false)) {
6684 re_initialize();
6685 goto restart;
6686 }
6687 do_sleep();
6688 if (snapshot_proc_sysfs_files())
6689 goto restart;
6690 retval = for_all_cpus(get_counters, ODD_COUNTERS);
6691 if (retval < -1) {
6692 exit(retval);
6693 } else if (retval == -1) {
6694 re_initialize();
6695 goto restart;
6696 }
6697 gettimeofday(&tv_odd, (struct timezone *)NULL);
6698 timersub(&tv_odd, &tv_even, &tv_delta);
6699 if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) {
6700 re_initialize();
6701 goto restart;
6702 }
6703 delta_platform(&platform_counters_odd, &platform_counters_even);
6704 compute_average(EVEN_COUNTERS);
6705 format_all_counters(EVEN_COUNTERS);
6706 flush_output_stdout();
6707 if (exit_requested)
6708 break;
6709 if (num_iterations && ++done_iters >= num_iterations)
6710 break;
6711 do_sleep();
6712 if (snapshot_proc_sysfs_files())
6713 goto restart;
6714 retval = for_all_cpus(get_counters, EVEN_COUNTERS);
6715 if (retval < -1) {
6716 exit(retval);
6717 } else if (retval == -1) {
6718 re_initialize();
6719 goto restart;
6720 }
6721 gettimeofday(&tv_even, (struct timezone *)NULL);
6722 timersub(&tv_even, &tv_odd, &tv_delta);
6723 if (for_all_cpus_2(delta_cpu, EVEN_COUNTERS, ODD_COUNTERS)) {
6724 re_initialize();
6725 goto restart;
6726 }
6727 delta_platform(&platform_counters_even, &platform_counters_odd);
6728 compute_average(ODD_COUNTERS);
6729 format_all_counters(ODD_COUNTERS);
6730 flush_output_stdout();
6731 if (exit_requested)
6732 break;
6733 if (num_iterations && ++done_iters >= num_iterations)
6734 break;
6735 }
6736 }
6737
probe_dev_msr(void)6738 int probe_dev_msr(void)
6739 {
6740 struct stat sb;
6741 char pathname[32];
6742
6743 sprintf(pathname, "/dev/msr%d", base_cpu);
6744 return !stat(pathname, &sb);
6745 }
6746
probe_dev_cpu_msr(void)6747 int probe_dev_cpu_msr(void)
6748 {
6749 struct stat sb;
6750 char pathname[32];
6751
6752 sprintf(pathname, "/dev/cpu/%d/msr", base_cpu);
6753 return !stat(pathname, &sb);
6754 }
6755
probe_msr_driver(void)6756 int probe_msr_driver(void)
6757 {
6758 if (probe_dev_msr()) {
6759 use_android_msr_path = 1;
6760 return 1;
6761 }
6762 return probe_dev_cpu_msr();
6763 }
6764
check_msr_driver(void)6765 void check_msr_driver(void)
6766 {
6767 if (probe_msr_driver())
6768 return;
6769
6770 if (system("/sbin/modprobe msr > /dev/null 2>&1"))
6771 no_msr = 1;
6772
6773 if (!probe_msr_driver())
6774 no_msr = 1;
6775 }
6776
6777 /*
6778 * check for CAP_SYS_RAWIO
6779 * return 0 on success
6780 * return 1 on fail
6781 */
check_for_cap_sys_rawio(void)6782 int check_for_cap_sys_rawio(void)
6783 {
6784 cap_t caps;
6785 cap_flag_value_t cap_flag_value;
6786 int ret = 0;
6787
6788 caps = cap_get_proc();
6789 if (caps == NULL) {
6790 /*
6791 * CONFIG_MULTIUSER=n kernels have no cap_get_proc()
6792 * Allow them to continue and attempt to access MSRs
6793 */
6794 if (errno == ENOSYS)
6795 return 0;
6796
6797 return 1;
6798 }
6799
6800 if (cap_get_flag(caps, CAP_SYS_RAWIO, CAP_EFFECTIVE, &cap_flag_value)) {
6801 ret = 1;
6802 goto free_and_exit;
6803 }
6804
6805 if (cap_flag_value != CAP_SET) {
6806 ret = 1;
6807 goto free_and_exit;
6808 }
6809
6810 free_and_exit:
6811 if (cap_free(caps) == -1)
6812 err(-6, "cap_free\n");
6813
6814 return ret;
6815 }
6816
check_msr_permission(void)6817 void check_msr_permission(void)
6818 {
6819 int failed = 0;
6820 char pathname[32];
6821
6822 if (no_msr)
6823 return;
6824
6825 /* check for CAP_SYS_RAWIO */
6826 failed += check_for_cap_sys_rawio();
6827
6828 /* test file permissions */
6829 sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", base_cpu);
6830 if (euidaccess(pathname, R_OK)) {
6831 failed++;
6832 }
6833
6834 if (failed) {
6835 warnx("Failed to access %s. Some of the counters may not be available\n"
6836 "\tRun as root to enable them or use %s to disable the access explicitly", pathname, "--no-msr");
6837 no_msr = 1;
6838 }
6839 }
6840
probe_bclk(void)6841 void probe_bclk(void)
6842 {
6843 unsigned long long msr;
6844 unsigned int base_ratio;
6845
6846 if (!platform->has_nhm_msrs || no_msr)
6847 return;
6848
6849 if (platform->bclk_freq == BCLK_100MHZ)
6850 bclk = 100.00;
6851 else if (platform->bclk_freq == BCLK_133MHZ)
6852 bclk = 133.33;
6853 else if (platform->bclk_freq == BCLK_SLV)
6854 bclk = slm_bclk();
6855 else
6856 return;
6857
6858 get_msr(base_cpu, MSR_PLATFORM_INFO, &msr);
6859 base_ratio = (msr >> 8) & 0xFF;
6860
6861 base_hz = base_ratio * bclk * 1000000;
6862 has_base_hz = 1;
6863
6864 if (platform->enable_tsc_tweak)
6865 tsc_tweak = base_hz / tsc_hz;
6866 }
6867
remove_underbar(char * s)6868 static void remove_underbar(char *s)
6869 {
6870 char *to = s;
6871
6872 while (*s) {
6873 if (*s != '_')
6874 *to++ = *s;
6875 s++;
6876 }
6877
6878 *to = 0;
6879 }
6880
dump_turbo_ratio_info(void)6881 static void dump_turbo_ratio_info(void)
6882 {
6883 if (!has_turbo)
6884 return;
6885
6886 if (!platform->has_nhm_msrs || no_msr)
6887 return;
6888
6889 if (platform->trl_msrs & TRL_LIMIT2)
6890 dump_turbo_ratio_limit2();
6891
6892 if (platform->trl_msrs & TRL_LIMIT1)
6893 dump_turbo_ratio_limit1();
6894
6895 if (platform->trl_msrs & TRL_BASE) {
6896 dump_turbo_ratio_limits(MSR_TURBO_RATIO_LIMIT);
6897
6898 if (is_hybrid)
6899 dump_turbo_ratio_limits(MSR_SECONDARY_TURBO_RATIO_LIMIT);
6900 }
6901
6902 if (platform->trl_msrs & TRL_ATOM)
6903 dump_atom_turbo_ratio_limits();
6904
6905 if (platform->trl_msrs & TRL_KNL)
6906 dump_knl_turbo_ratio_limits();
6907
6908 if (platform->has_config_tdp)
6909 dump_config_tdp();
6910 }
6911
read_sysfs_int(char * path)6912 static int read_sysfs_int(char *path)
6913 {
6914 FILE *input;
6915 int retval = -1;
6916
6917 input = fopen(path, "r");
6918 if (input == NULL) {
6919 if (debug)
6920 fprintf(outf, "NSFOD %s\n", path);
6921 return (-1);
6922 }
6923 if (fscanf(input, "%d", &retval) != 1)
6924 err(1, "%s: failed to read int from file", path);
6925 fclose(input);
6926
6927 return (retval);
6928 }
6929
dump_sysfs_file(char * path)6930 static void dump_sysfs_file(char *path)
6931 {
6932 FILE *input;
6933 char cpuidle_buf[64];
6934
6935 input = fopen(path, "r");
6936 if (input == NULL) {
6937 if (debug)
6938 fprintf(outf, "NSFOD %s\n", path);
6939 return;
6940 }
6941 if (!fgets(cpuidle_buf, sizeof(cpuidle_buf), input))
6942 err(1, "%s: failed to read file", path);
6943 fclose(input);
6944
6945 fprintf(outf, "%s: %s", strrchr(path, '/') + 1, cpuidle_buf);
6946 }
6947
probe_intel_uncore_frequency_legacy(void)6948 static void probe_intel_uncore_frequency_legacy(void)
6949 {
6950 int i, j;
6951 char path[256];
6952
6953 for (i = 0; i < topo.num_packages; ++i) {
6954 for (j = 0; j <= topo.max_die_id; ++j) {
6955 int k, l;
6956 char path_base[128];
6957
6958 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, j);
6959
6960 sprintf(path, "%s/current_freq_khz", path_base);
6961 if (access(path, R_OK))
6962 continue;
6963
6964 BIC_PRESENT(BIC_UNCORE_MHZ);
6965
6966 if (quiet)
6967 return;
6968
6969 sprintf(path, "%s/min_freq_khz", path_base);
6970 k = read_sysfs_int(path);
6971 sprintf(path, "%s/max_freq_khz", path_base);
6972 l = read_sysfs_int(path);
6973 fprintf(outf, "Uncore Frequency package%d die%d: %d - %d MHz ", i, j, k / 1000, l / 1000);
6974
6975 sprintf(path, "%s/initial_min_freq_khz", path_base);
6976 k = read_sysfs_int(path);
6977 sprintf(path, "%s/initial_max_freq_khz", path_base);
6978 l = read_sysfs_int(path);
6979 fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
6980
6981 sprintf(path, "%s/current_freq_khz", path_base);
6982 k = read_sysfs_int(path);
6983 fprintf(outf, " %d MHz\n", k / 1000);
6984 }
6985 }
6986 }
6987
probe_intel_uncore_frequency_cluster(void)6988 static void probe_intel_uncore_frequency_cluster(void)
6989 {
6990 int i, uncore_max_id;
6991 char path[256];
6992 char path_base[128];
6993
6994 if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00/current_freq_khz", R_OK))
6995 return;
6996
6997 for (uncore_max_id = 0;; ++uncore_max_id) {
6998
6999 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", uncore_max_id);
7000
7001 /* uncore## start at 00 and skips no numbers, so stop upon first missing */
7002 if (access(path_base, R_OK)) {
7003 uncore_max_id -= 1;
7004 break;
7005 }
7006 }
7007 for (i = uncore_max_id; i >= 0; --i) {
7008 int k, l;
7009 int package_id, domain_id, cluster_id;
7010 char name_buf[16];
7011
7012 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i);
7013
7014 if (access(path_base, R_OK))
7015 err(1, "%s: %s\n", __func__, path_base);
7016
7017 sprintf(path, "%s/package_id", path_base);
7018 package_id = read_sysfs_int(path);
7019
7020 sprintf(path, "%s/domain_id", path_base);
7021 domain_id = read_sysfs_int(path);
7022
7023 sprintf(path, "%s/fabric_cluster_id", path_base);
7024 cluster_id = read_sysfs_int(path);
7025
7026 sprintf(path, "%s/current_freq_khz", path_base);
7027 sprintf(name_buf, "UMHz%d.%d", domain_id, cluster_id);
7028
7029 /*
7030 * Once add_couter() is called, that counter is always read
7031 * and reported -- So it is effectively (enabled & present).
7032 * Only call add_counter() here if legacy BIC_UNCORE_MHZ (UncMHz)
7033 * is (enabled). Since we are in this routine, we
7034 * know we will not probe and set (present) the legacy counter.
7035 *
7036 * This allows "--show/--hide UncMHz" to be effective for
7037 * the clustered MHz counters, as a group.
7038 */
7039 if BIC_IS_ENABLED
7040 (BIC_UNCORE_MHZ)
7041 add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, package_id);
7042
7043 if (quiet)
7044 continue;
7045
7046 sprintf(path, "%s/min_freq_khz", path_base);
7047 k = read_sysfs_int(path);
7048 sprintf(path, "%s/max_freq_khz", path_base);
7049 l = read_sysfs_int(path);
7050 fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", package_id, domain_id, cluster_id, k / 1000, l / 1000);
7051
7052 sprintf(path, "%s/initial_min_freq_khz", path_base);
7053 k = read_sysfs_int(path);
7054 sprintf(path, "%s/initial_max_freq_khz", path_base);
7055 l = read_sysfs_int(path);
7056 fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
7057
7058 sprintf(path, "%s/current_freq_khz", path_base);
7059 k = read_sysfs_int(path);
7060 fprintf(outf, " %d MHz\n", k / 1000);
7061 }
7062 }
7063
probe_intel_uncore_frequency(void)7064 static void probe_intel_uncore_frequency(void)
7065 {
7066 if (!genuine_intel)
7067 return;
7068
7069 if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00", R_OK) == 0)
7070 probe_intel_uncore_frequency_cluster();
7071 else
7072 probe_intel_uncore_frequency_legacy();
7073 }
7074
set_graphics_fp(char * path,int idx)7075 static void set_graphics_fp(char *path, int idx)
7076 {
7077 if (!access(path, R_OK))
7078 gfx_info[idx].fp = fopen_or_die(path, "r");
7079 }
7080
7081 /* Enlarge this if there are /sys/class/drm/card2 ... */
7082 #define GFX_MAX_CARDS 2
7083
probe_graphics(void)7084 static void probe_graphics(void)
7085 {
7086 char path[PATH_MAX];
7087 int i;
7088
7089 /* Xe graphics sysfs knobs */
7090 if (!access("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", R_OK)) {
7091 FILE *fp;
7092 char buf[8];
7093 bool gt0_is_gt;
7094
7095 fp = fopen("/sys/class/drm/card0/device/tile0/gt0/gtidle/name", "r");
7096 if (!fp)
7097 goto next;
7098
7099 if (!fread(buf, sizeof(char), 7, fp)) {
7100 fclose(fp);
7101 goto next;
7102 }
7103 fclose(fp);
7104
7105 if (!strncmp(buf, "gt0-rc", strlen("gt0-rc")))
7106 gt0_is_gt = true;
7107 else if (!strncmp(buf, "gt0-mc", strlen("gt0-mc")))
7108 gt0_is_gt = false;
7109 else
7110 goto next;
7111
7112 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", gt0_is_gt ? GFX_rc6 : SAM_mc6);
7113
7114 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/cur_freq", gt0_is_gt ? GFX_MHz : SAM_MHz);
7115
7116 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/act_freq", gt0_is_gt ? GFX_ACTMHz : SAM_ACTMHz);
7117
7118 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/gtidle/idle_residency_ms", gt0_is_gt ? SAM_mc6 : GFX_rc6);
7119
7120 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/cur_freq", gt0_is_gt ? SAM_MHz : GFX_MHz);
7121
7122 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/act_freq", gt0_is_gt ? SAM_ACTMHz : GFX_ACTMHz);
7123
7124 goto end;
7125 }
7126
7127 next:
7128 /* New i915 graphics sysfs knobs */
7129 for (i = 0; i < GFX_MAX_CARDS; i++) {
7130 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i);
7131 if (!access(path, R_OK))
7132 break;
7133 }
7134
7135 if (i == GFX_MAX_CARDS)
7136 goto legacy_i915;
7137
7138 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i);
7139 set_graphics_fp(path, GFX_rc6);
7140
7141 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rps_cur_freq_mhz", i);
7142 set_graphics_fp(path, GFX_MHz);
7143
7144 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rps_act_freq_mhz", i);
7145 set_graphics_fp(path, GFX_ACTMHz);
7146
7147 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rc6_residency_ms", i);
7148 set_graphics_fp(path, SAM_mc6);
7149
7150 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rps_cur_freq_mhz", i);
7151 set_graphics_fp(path, SAM_MHz);
7152
7153 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rps_act_freq_mhz", i);
7154 set_graphics_fp(path, SAM_ACTMHz);
7155
7156 goto end;
7157
7158 legacy_i915:
7159 /* Fall back to traditional i915 graphics sysfs knobs */
7160 set_graphics_fp("/sys/class/drm/card0/power/rc6_residency_ms", GFX_rc6);
7161
7162 set_graphics_fp("/sys/class/drm/card0/gt_cur_freq_mhz", GFX_MHz);
7163 if (!gfx_info[GFX_MHz].fp)
7164 set_graphics_fp("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", GFX_MHz);
7165
7166 set_graphics_fp("/sys/class/drm/card0/gt_act_freq_mhz", GFX_ACTMHz);
7167 if (!gfx_info[GFX_ACTMHz].fp)
7168 set_graphics_fp("/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz", GFX_ACTMHz);
7169
7170 end:
7171 if (gfx_info[GFX_rc6].fp)
7172 BIC_PRESENT(BIC_GFX_rc6);
7173 if (gfx_info[GFX_MHz].fp)
7174 BIC_PRESENT(BIC_GFXMHz);
7175 if (gfx_info[GFX_ACTMHz].fp)
7176 BIC_PRESENT(BIC_GFXACTMHz);
7177 if (gfx_info[SAM_mc6].fp)
7178 BIC_PRESENT(BIC_SAM_mc6);
7179 if (gfx_info[SAM_MHz].fp)
7180 BIC_PRESENT(BIC_SAMMHz);
7181 if (gfx_info[SAM_ACTMHz].fp)
7182 BIC_PRESENT(BIC_SAMACTMHz);
7183 }
7184
dump_sysfs_cstate_config(void)7185 static void dump_sysfs_cstate_config(void)
7186 {
7187 char path[64];
7188 char name_buf[16];
7189 char desc[64];
7190 FILE *input;
7191 int state;
7192 char *sp;
7193
7194 if (access("/sys/devices/system/cpu/cpuidle", R_OK)) {
7195 fprintf(outf, "cpuidle not loaded\n");
7196 return;
7197 }
7198
7199 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_driver");
7200 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor");
7201 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor_ro");
7202
7203 for (state = 0; state < 10; ++state) {
7204
7205 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state);
7206 input = fopen(path, "r");
7207 if (input == NULL)
7208 continue;
7209 if (!fgets(name_buf, sizeof(name_buf), input))
7210 err(1, "%s: failed to read file", path);
7211
7212 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
7213 sp = strchr(name_buf, '-');
7214 if (!sp)
7215 sp = strchrnul(name_buf, '\n');
7216 *sp = '\0';
7217 fclose(input);
7218
7219 remove_underbar(name_buf);
7220
7221 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", base_cpu, state);
7222 input = fopen(path, "r");
7223 if (input == NULL)
7224 continue;
7225 if (!fgets(desc, sizeof(desc), input))
7226 err(1, "%s: failed to read file", path);
7227
7228 fprintf(outf, "cpu%d: %s: %s", base_cpu, name_buf, desc);
7229 fclose(input);
7230 }
7231 }
7232
dump_sysfs_pstate_config(void)7233 static void dump_sysfs_pstate_config(void)
7234 {
7235 char path[64];
7236 char driver_buf[64];
7237 char governor_buf[64];
7238 FILE *input;
7239 int turbo;
7240
7241 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", base_cpu);
7242 input = fopen(path, "r");
7243 if (input == NULL) {
7244 fprintf(outf, "NSFOD %s\n", path);
7245 return;
7246 }
7247 if (!fgets(driver_buf, sizeof(driver_buf), input))
7248 err(1, "%s: failed to read file", path);
7249 fclose(input);
7250
7251 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", base_cpu);
7252 input = fopen(path, "r");
7253 if (input == NULL) {
7254 fprintf(outf, "NSFOD %s\n", path);
7255 return;
7256 }
7257 if (!fgets(governor_buf, sizeof(governor_buf), input))
7258 err(1, "%s: failed to read file", path);
7259 fclose(input);
7260
7261 fprintf(outf, "cpu%d: cpufreq driver: %s", base_cpu, driver_buf);
7262 fprintf(outf, "cpu%d: cpufreq governor: %s", base_cpu, governor_buf);
7263
7264 sprintf(path, "/sys/devices/system/cpu/cpufreq/boost");
7265 input = fopen(path, "r");
7266 if (input != NULL) {
7267 if (fscanf(input, "%d", &turbo) != 1)
7268 err(1, "%s: failed to parse number from file", path);
7269 fprintf(outf, "cpufreq boost: %d\n", turbo);
7270 fclose(input);
7271 }
7272
7273 sprintf(path, "/sys/devices/system/cpu/intel_pstate/no_turbo");
7274 input = fopen(path, "r");
7275 if (input != NULL) {
7276 if (fscanf(input, "%d", &turbo) != 1)
7277 err(1, "%s: failed to parse number from file", path);
7278 fprintf(outf, "cpufreq intel_pstate no_turbo: %d\n", turbo);
7279 fclose(input);
7280 }
7281 }
7282
7283 /*
7284 * print_epb()
7285 * Decode the ENERGY_PERF_BIAS MSR
7286 */
print_epb(PER_THREAD_PARAMS)7287 int print_epb(PER_THREAD_PARAMS)
7288 {
7289 char *epb_string;
7290 int cpu, epb;
7291
7292 UNUSED(c);
7293 UNUSED(p);
7294
7295 if (!has_epb)
7296 return 0;
7297
7298 cpu = t->cpu_id;
7299
7300 /* EPB is per-package */
7301 if (!is_cpu_first_thread_in_package(t, c, p))
7302 return 0;
7303
7304 if (cpu_migrate(cpu)) {
7305 fprintf(outf, "print_epb: Could not migrate to CPU %d\n", cpu);
7306 return -1;
7307 }
7308
7309 epb = get_epb(cpu);
7310 if (epb < 0)
7311 return 0;
7312
7313 switch (epb) {
7314 case ENERGY_PERF_BIAS_PERFORMANCE:
7315 epb_string = "performance";
7316 break;
7317 case ENERGY_PERF_BIAS_NORMAL:
7318 epb_string = "balanced";
7319 break;
7320 case ENERGY_PERF_BIAS_POWERSAVE:
7321 epb_string = "powersave";
7322 break;
7323 default:
7324 epb_string = "custom";
7325 break;
7326 }
7327 fprintf(outf, "cpu%d: EPB: %d (%s)\n", cpu, epb, epb_string);
7328
7329 return 0;
7330 }
7331
7332 /*
7333 * print_hwp()
7334 * Decode the MSR_HWP_CAPABILITIES
7335 */
print_hwp(PER_THREAD_PARAMS)7336 int print_hwp(PER_THREAD_PARAMS)
7337 {
7338 unsigned long long msr;
7339 int cpu;
7340
7341 UNUSED(c);
7342 UNUSED(p);
7343
7344 if (no_msr)
7345 return 0;
7346
7347 if (!has_hwp)
7348 return 0;
7349
7350 cpu = t->cpu_id;
7351
7352 /* MSR_HWP_CAPABILITIES is per-package */
7353 if (!is_cpu_first_thread_in_package(t, c, p))
7354 return 0;
7355
7356 if (cpu_migrate(cpu)) {
7357 fprintf(outf, "print_hwp: Could not migrate to CPU %d\n", cpu);
7358 return -1;
7359 }
7360
7361 if (get_msr(cpu, MSR_PM_ENABLE, &msr))
7362 return 0;
7363
7364 fprintf(outf, "cpu%d: MSR_PM_ENABLE: 0x%08llx (%sHWP)\n", cpu, msr, (msr & (1 << 0)) ? "" : "No-");
7365
7366 /* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */
7367 if ((msr & (1 << 0)) == 0)
7368 return 0;
7369
7370 if (get_msr(cpu, MSR_HWP_CAPABILITIES, &msr))
7371 return 0;
7372
7373 fprintf(outf, "cpu%d: MSR_HWP_CAPABILITIES: 0x%08llx "
7374 "(high %d guar %d eff %d low %d)\n",
7375 cpu, msr,
7376 (unsigned int)HWP_HIGHEST_PERF(msr),
7377 (unsigned int)HWP_GUARANTEED_PERF(msr), (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr));
7378
7379 if (get_msr(cpu, MSR_HWP_REQUEST, &msr))
7380 return 0;
7381
7382 fprintf(outf, "cpu%d: MSR_HWP_REQUEST: 0x%08llx "
7383 "(min %d max %d des %d epp 0x%x window 0x%x pkg 0x%x)\n",
7384 cpu, msr,
7385 (unsigned int)(((msr) >> 0) & 0xff),
7386 (unsigned int)(((msr) >> 8) & 0xff),
7387 (unsigned int)(((msr) >> 16) & 0xff),
7388 (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1));
7389
7390 if (has_hwp_pkg) {
7391 if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr))
7392 return 0;
7393
7394 fprintf(outf, "cpu%d: MSR_HWP_REQUEST_PKG: 0x%08llx "
7395 "(min %d max %d des %d epp 0x%x window 0x%x)\n",
7396 cpu, msr,
7397 (unsigned int)(((msr) >> 0) & 0xff),
7398 (unsigned int)(((msr) >> 8) & 0xff),
7399 (unsigned int)(((msr) >> 16) & 0xff), (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3));
7400 }
7401 if (has_hwp_notify) {
7402 if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr))
7403 return 0;
7404
7405 fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx "
7406 "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis");
7407 }
7408 if (get_msr(cpu, MSR_HWP_STATUS, &msr))
7409 return 0;
7410
7411 fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx "
7412 "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-");
7413
7414 return 0;
7415 }
7416
7417 /*
7418 * print_perf_limit()
7419 */
print_perf_limit(PER_THREAD_PARAMS)7420 int print_perf_limit(PER_THREAD_PARAMS)
7421 {
7422 unsigned long long msr;
7423 int cpu;
7424
7425 UNUSED(c);
7426 UNUSED(p);
7427
7428 if (no_msr)
7429 return 0;
7430
7431 cpu = t->cpu_id;
7432
7433 /* per-package */
7434 if (!is_cpu_first_thread_in_package(t, c, p))
7435 return 0;
7436
7437 if (cpu_migrate(cpu)) {
7438 fprintf(outf, "print_perf_limit: Could not migrate to CPU %d\n", cpu);
7439 return -1;
7440 }
7441
7442 if (platform->plr_msrs & PLR_CORE) {
7443 get_msr(cpu, MSR_CORE_PERF_LIMIT_REASONS, &msr);
7444 fprintf(outf, "cpu%d: MSR_CORE_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7445 fprintf(outf, " (Active: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)",
7446 (msr & 1 << 15) ? "bit15, " : "",
7447 (msr & 1 << 14) ? "bit14, " : "",
7448 (msr & 1 << 13) ? "Transitions, " : "",
7449 (msr & 1 << 12) ? "MultiCoreTurbo, " : "",
7450 (msr & 1 << 11) ? "PkgPwrL2, " : "",
7451 (msr & 1 << 10) ? "PkgPwrL1, " : "",
7452 (msr & 1 << 9) ? "CorePwr, " : "",
7453 (msr & 1 << 8) ? "Amps, " : "",
7454 (msr & 1 << 6) ? "VR-Therm, " : "",
7455 (msr & 1 << 5) ? "Auto-HWP, " : "",
7456 (msr & 1 << 4) ? "Graphics, " : "",
7457 (msr & 1 << 2) ? "bit2, " : "", (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : "");
7458 fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n",
7459 (msr & 1 << 31) ? "bit31, " : "",
7460 (msr & 1 << 30) ? "bit30, " : "",
7461 (msr & 1 << 29) ? "Transitions, " : "",
7462 (msr & 1 << 28) ? "MultiCoreTurbo, " : "",
7463 (msr & 1 << 27) ? "PkgPwrL2, " : "",
7464 (msr & 1 << 26) ? "PkgPwrL1, " : "",
7465 (msr & 1 << 25) ? "CorePwr, " : "",
7466 (msr & 1 << 24) ? "Amps, " : "",
7467 (msr & 1 << 22) ? "VR-Therm, " : "",
7468 (msr & 1 << 21) ? "Auto-HWP, " : "",
7469 (msr & 1 << 20) ? "Graphics, " : "",
7470 (msr & 1 << 18) ? "bit18, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : "");
7471
7472 }
7473 if (platform->plr_msrs & PLR_GFX) {
7474 get_msr(cpu, MSR_GFX_PERF_LIMIT_REASONS, &msr);
7475 fprintf(outf, "cpu%d: MSR_GFX_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7476 fprintf(outf, " (Active: %s%s%s%s%s%s%s%s)",
7477 (msr & 1 << 0) ? "PROCHOT, " : "",
7478 (msr & 1 << 1) ? "ThermStatus, " : "",
7479 (msr & 1 << 4) ? "Graphics, " : "",
7480 (msr & 1 << 6) ? "VR-Therm, " : "",
7481 (msr & 1 << 8) ? "Amps, " : "",
7482 (msr & 1 << 9) ? "GFXPwr, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
7483 fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s)\n",
7484 (msr & 1 << 16) ? "PROCHOT, " : "",
7485 (msr & 1 << 17) ? "ThermStatus, " : "",
7486 (msr & 1 << 20) ? "Graphics, " : "",
7487 (msr & 1 << 22) ? "VR-Therm, " : "",
7488 (msr & 1 << 24) ? "Amps, " : "",
7489 (msr & 1 << 25) ? "GFXPwr, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
7490 }
7491 if (platform->plr_msrs & PLR_RING) {
7492 get_msr(cpu, MSR_RING_PERF_LIMIT_REASONS, &msr);
7493 fprintf(outf, "cpu%d: MSR_RING_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7494 fprintf(outf, " (Active: %s%s%s%s%s%s)",
7495 (msr & 1 << 0) ? "PROCHOT, " : "",
7496 (msr & 1 << 1) ? "ThermStatus, " : "",
7497 (msr & 1 << 6) ? "VR-Therm, " : "",
7498 (msr & 1 << 8) ? "Amps, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
7499 fprintf(outf, " (Logged: %s%s%s%s%s%s)\n",
7500 (msr & 1 << 16) ? "PROCHOT, " : "",
7501 (msr & 1 << 17) ? "ThermStatus, " : "",
7502 (msr & 1 << 22) ? "VR-Therm, " : "",
7503 (msr & 1 << 24) ? "Amps, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
7504 }
7505 return 0;
7506 }
7507
7508 #define RAPL_POWER_GRANULARITY 0x7FFF /* 15 bit power granularity */
7509 #define RAPL_TIME_GRANULARITY 0x3F /* 6 bit time granularity */
7510
get_quirk_tdp(void)7511 double get_quirk_tdp(void)
7512 {
7513 if (platform->rapl_quirk_tdp)
7514 return platform->rapl_quirk_tdp;
7515
7516 return 135.0;
7517 }
7518
get_tdp_intel(void)7519 double get_tdp_intel(void)
7520 {
7521 unsigned long long msr;
7522
7523 if (valid_rapl_msrs & RAPL_PKG_POWER_INFO)
7524 if (!get_msr(base_cpu, MSR_PKG_POWER_INFO, &msr))
7525 return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units;
7526 return get_quirk_tdp();
7527 }
7528
get_tdp_amd(void)7529 double get_tdp_amd(void)
7530 {
7531 return get_quirk_tdp();
7532 }
7533
rapl_probe_intel(void)7534 void rapl_probe_intel(void)
7535 {
7536 unsigned long long msr;
7537 unsigned int time_unit;
7538 double tdp;
7539
7540 if (rapl_joules) {
7541 CLR_BIC(BIC_SysWatt, &bic_enabled);
7542 CLR_BIC(BIC_PkgWatt, &bic_enabled);
7543 CLR_BIC(BIC_CorWatt, &bic_enabled);
7544 CLR_BIC(BIC_RAMWatt, &bic_enabled);
7545 CLR_BIC(BIC_GFXWatt, &bic_enabled);
7546 } else {
7547 CLR_BIC(BIC_Sys_J, &bic_enabled);
7548 CLR_BIC(BIC_Pkg_J, &bic_enabled);
7549 CLR_BIC(BIC_Cor_J, &bic_enabled);
7550 CLR_BIC(BIC_RAM_J, &bic_enabled);
7551 CLR_BIC(BIC_GFX_J, &bic_enabled);
7552 }
7553
7554 if (!valid_rapl_msrs || no_msr)
7555 return;
7556
7557 if (!(valid_rapl_msrs & RAPL_PKG_PERF_STATUS))
7558 CLR_BIC(BIC_PKG__, &bic_enabled);
7559 if (!(valid_rapl_msrs & RAPL_DRAM_PERF_STATUS))
7560 CLR_BIC(BIC_RAM__, &bic_enabled);
7561
7562 /* units on package 0, verify later other packages match */
7563 if (get_msr(base_cpu, MSR_RAPL_POWER_UNIT, &msr))
7564 return;
7565
7566 rapl_power_units = 1.0 / (1 << (msr & 0xF));
7567 if (platform->has_rapl_divisor)
7568 rapl_energy_units = 1.0 * (1 << (msr >> 8 & 0x1F)) / 1000000;
7569 else
7570 rapl_energy_units = 1.0 / (1 << (msr >> 8 & 0x1F));
7571
7572 if (platform->has_fixed_rapl_unit)
7573 rapl_dram_energy_units = (15.3 / 1000000);
7574 else
7575 rapl_dram_energy_units = rapl_energy_units;
7576
7577 if (platform->has_fixed_rapl_psys_unit)
7578 rapl_psys_energy_units = 1.0;
7579 else
7580 rapl_psys_energy_units = rapl_energy_units;
7581
7582 time_unit = msr >> 16 & 0xF;
7583 if (time_unit == 0)
7584 time_unit = 0xA;
7585
7586 rapl_time_units = 1.0 / (1 << (time_unit));
7587
7588 tdp = get_tdp_intel();
7589
7590 rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7591 if (!quiet)
7592 fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7593 }
7594
rapl_probe_amd(void)7595 void rapl_probe_amd(void)
7596 {
7597 unsigned long long msr;
7598 double tdp;
7599
7600 if (rapl_joules) {
7601 CLR_BIC(BIC_SysWatt, &bic_enabled);
7602 CLR_BIC(BIC_CorWatt, &bic_enabled);
7603 } else {
7604 CLR_BIC(BIC_Pkg_J, &bic_enabled);
7605 CLR_BIC(BIC_Cor_J, &bic_enabled);
7606 }
7607
7608 if (!valid_rapl_msrs || no_msr)
7609 return;
7610
7611 if (get_msr(base_cpu, MSR_RAPL_PWR_UNIT, &msr))
7612 return;
7613
7614 rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf));
7615 rapl_energy_units = ldexp(1.0, -(msr >> 8 & 0x1f));
7616 rapl_power_units = ldexp(1.0, -(msr & 0xf));
7617
7618 tdp = get_tdp_amd();
7619
7620 rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7621 if (!quiet)
7622 fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7623 }
7624
print_power_limit_msr(int cpu,unsigned long long msr,char * label)7625 void print_power_limit_msr(int cpu, unsigned long long msr, char *label)
7626 {
7627 fprintf(outf, "cpu%d: %s: %sabled (%0.3f Watts, %f sec, clamp %sabled)\n",
7628 cpu, label,
7629 ((msr >> 15) & 1) ? "EN" : "DIS",
7630 ((msr >> 0) & 0x7FFF) * rapl_power_units,
7631 (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, (((msr >> 16) & 1) ? "EN" : "DIS"));
7632
7633 return;
7634 }
7635
fread_int(char * path,int * val)7636 static int fread_int(char *path, int *val)
7637 {
7638 FILE *filep;
7639 int ret;
7640
7641 filep = fopen(path, "r");
7642 if (!filep)
7643 return -1;
7644
7645 ret = fscanf(filep, "%d", val);
7646 fclose(filep);
7647 return ret;
7648 }
7649
fread_ull(char * path,unsigned long long * val)7650 static int fread_ull(char *path, unsigned long long *val)
7651 {
7652 FILE *filep;
7653 int ret;
7654
7655 filep = fopen(path, "r");
7656 if (!filep)
7657 return -1;
7658
7659 ret = fscanf(filep, "%llu", val);
7660 fclose(filep);
7661 return ret;
7662 }
7663
fread_str(char * path,char * buf,int size)7664 static int fread_str(char *path, char *buf, int size)
7665 {
7666 FILE *filep;
7667 int ret;
7668 char *cp;
7669
7670 filep = fopen(path, "r");
7671 if (!filep)
7672 return -1;
7673
7674 ret = fread(buf, 1, size, filep);
7675 fclose(filep);
7676
7677 /* replace '\n' with '\0' */
7678 cp = strchr(buf, '\n');
7679 if (cp != NULL)
7680 *cp = '\0';
7681
7682 return ret;
7683 }
7684
7685 #define PATH_RAPL_SYSFS "/sys/class/powercap"
7686
dump_one_domain(char * domain_path)7687 static int dump_one_domain(char *domain_path)
7688 {
7689 char path[PATH_MAX];
7690 char str[PATH_MAX];
7691 unsigned long long val;
7692 int constraint;
7693 int enable;
7694 int ret;
7695
7696 snprintf(path, PATH_MAX, "%s/name", domain_path);
7697 ret = fread_str(path, str, PATH_MAX);
7698 if (ret <= 0)
7699 return -1;
7700
7701 fprintf(outf, "%s: %s", domain_path + strlen(PATH_RAPL_SYSFS) + 1, str);
7702
7703 snprintf(path, PATH_MAX, "%s/enabled", domain_path);
7704 ret = fread_int(path, &enable);
7705 if (ret <= 0)
7706 return -1;
7707
7708 if (!enable) {
7709 fputs(" disabled\n", outf);
7710 return 0;
7711 }
7712
7713 for (constraint = 0;; constraint++) {
7714 snprintf(path, PATH_MAX, "%s/constraint_%d_time_window_us", domain_path, constraint);
7715 ret = fread_ull(path, &val);
7716 if (ret <= 0)
7717 break;
7718
7719 if (val > 1000000)
7720 fprintf(outf, " %0.1fs", (double)val / 1000000);
7721 else if (val > 1000)
7722 fprintf(outf, " %0.1fms", (double)val / 1000);
7723 else
7724 fprintf(outf, " %0.1fus", (double)val);
7725
7726 snprintf(path, PATH_MAX, "%s/constraint_%d_power_limit_uw", domain_path, constraint);
7727 ret = fread_ull(path, &val);
7728 if (ret > 0 && val)
7729 fprintf(outf, ":%lluW", val / 1000000);
7730
7731 snprintf(path, PATH_MAX, "%s/constraint_%d_max_power_uw", domain_path, constraint);
7732 ret = fread_ull(path, &val);
7733 if (ret > 0 && val)
7734 fprintf(outf, ",max:%lluW", val / 1000000);
7735 }
7736 fputc('\n', outf);
7737
7738 return 0;
7739 }
7740
print_rapl_sysfs(void)7741 static int print_rapl_sysfs(void)
7742 {
7743 DIR *dir, *cdir;
7744 struct dirent *entry, *centry;
7745 char path[PATH_MAX];
7746 char str[PATH_MAX];
7747
7748 if ((dir = opendir(PATH_RAPL_SYSFS)) == NULL) {
7749 warn("open %s failed", PATH_RAPL_SYSFS);
7750 return 1;
7751 }
7752
7753 while ((entry = readdir(dir)) != NULL) {
7754 if (strlen(entry->d_name) > 100)
7755 continue;
7756
7757 if (strncmp(entry->d_name, "intel-rapl", strlen("intel-rapl")))
7758 continue;
7759
7760 snprintf(path, PATH_MAX, "%s/%s/name", PATH_RAPL_SYSFS, entry->d_name);
7761
7762 /* Parse top level domains first, including package and psys */
7763 fread_str(path, str, PATH_MAX);
7764 if (strncmp(str, "package", strlen("package")) && strncmp(str, "psys", strlen("psys")))
7765 continue;
7766
7767 snprintf(path, PATH_MAX, "%s/%s", PATH_RAPL_SYSFS, entry->d_name);
7768 if ((cdir = opendir(path)) == NULL) {
7769 perror("opendir() error");
7770 return 1;
7771 }
7772
7773 dump_one_domain(path);
7774
7775 while ((centry = readdir(cdir)) != NULL) {
7776 if (strncmp(centry->d_name, "intel-rapl", strlen("intel-rapl")))
7777 continue;
7778 snprintf(path, PATH_MAX, "%s/%s/%s", PATH_RAPL_SYSFS, entry->d_name, centry->d_name);
7779 dump_one_domain(path);
7780 }
7781 closedir(cdir);
7782 }
7783
7784 closedir(dir);
7785 return 0;
7786 }
7787
print_rapl(PER_THREAD_PARAMS)7788 int print_rapl(PER_THREAD_PARAMS)
7789 {
7790 unsigned long long msr;
7791 const char *msr_name;
7792 int cpu;
7793
7794 UNUSED(c);
7795 UNUSED(p);
7796
7797 if (!valid_rapl_msrs)
7798 return 0;
7799
7800 /* RAPL counters are per package, so print only for 1st thread/package */
7801 if (!is_cpu_first_thread_in_package(t, c, p))
7802 return 0;
7803
7804 cpu = t->cpu_id;
7805 if (cpu_migrate(cpu)) {
7806 fprintf(outf, "print_rapl: Could not migrate to CPU %d\n", cpu);
7807 return -1;
7808 }
7809
7810 if (valid_rapl_msrs & RAPL_AMD_F17H) {
7811 msr_name = "MSR_RAPL_PWR_UNIT";
7812 if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr))
7813 return -1;
7814 } else {
7815 msr_name = "MSR_RAPL_POWER_UNIT";
7816 if (get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr))
7817 return -1;
7818 }
7819
7820 fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr,
7821 rapl_power_units, rapl_energy_units, rapl_time_units);
7822
7823 if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) {
7824
7825 if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr))
7826 return -5;
7827
7828 fprintf(outf, "cpu%d: MSR_PKG_POWER_INFO: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
7829 cpu, msr,
7830 ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7831 ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7832 ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
7833
7834 }
7835 if (valid_rapl_msrs & RAPL_PKG) {
7836
7837 if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr))
7838 return -9;
7839
7840 fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 63) & 1 ? "" : "UN");
7841
7842 print_power_limit_msr(cpu, msr, "PKG Limit #1");
7843 fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%0.3f Watts, %f* sec, clamp %sabled)\n",
7844 cpu,
7845 ((msr >> 47) & 1) ? "EN" : "DIS",
7846 ((msr >> 32) & 0x7FFF) * rapl_power_units,
7847 (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, ((msr >> 48) & 1) ? "EN" : "DIS");
7848
7849 if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr))
7850 return -9;
7851
7852 fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr);
7853 fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n",
7854 cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN");
7855 }
7856
7857 if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) {
7858 if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr))
7859 return -6;
7860
7861 fprintf(outf, "cpu%d: MSR_DRAM_POWER_INFO,: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
7862 cpu, msr,
7863 ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7864 ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
7865 ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
7866 }
7867 if (valid_rapl_msrs & RAPL_DRAM) {
7868 if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr))
7869 return -9;
7870 fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
7871
7872 print_power_limit_msr(cpu, msr, "DRAM Limit");
7873 }
7874 if (valid_rapl_msrs & RAPL_CORE_POLICY) {
7875 if (get_msr(cpu, MSR_PP0_POLICY, &msr))
7876 return -7;
7877
7878 fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF);
7879 }
7880 if (valid_rapl_msrs & RAPL_CORE_POWER_LIMIT) {
7881 if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr))
7882 return -9;
7883 fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
7884 print_power_limit_msr(cpu, msr, "Cores Limit");
7885 }
7886 if (valid_rapl_msrs & RAPL_GFX) {
7887 if (get_msr(cpu, MSR_PP1_POLICY, &msr))
7888 return -8;
7889
7890 fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF);
7891
7892 if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr))
7893 return -9;
7894 fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
7895 print_power_limit_msr(cpu, msr, "GFX Limit");
7896 }
7897 return 0;
7898 }
7899
7900 /*
7901 * probe_rapl_msrs
7902 *
7903 * initialize global valid_rapl_msrs to platform->plat_rapl_msrs
7904 * only if PKG_ENERGY counter is enumerated and reads non-zero
7905 */
probe_rapl_msrs(void)7906 void probe_rapl_msrs(void)
7907 {
7908 int ret;
7909 off_t offset;
7910 unsigned long long msr_value;
7911
7912 if (no_msr)
7913 return;
7914
7915 if ((platform->plat_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H)) == 0)
7916 return;
7917
7918 offset = idx_to_offset(IDX_PKG_ENERGY);
7919 if (offset < 0)
7920 return;
7921
7922 ret = get_msr(base_cpu, offset, &msr_value);
7923 if (ret) {
7924 if (debug)
7925 fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset);
7926 return;
7927 }
7928 if (msr_value == 0) {
7929 if (debug)
7930 fprintf(outf, "RAPL_PKG_ENERGY MSR(0x%llx) == ZERO: disabling all RAPL MSRs\n", (unsigned long long)offset);
7931 return;
7932 }
7933
7934 valid_rapl_msrs = platform->plat_rapl_msrs; /* success */
7935 }
7936
7937 /*
7938 * probe_rapl()
7939 *
7940 * sets rapl_power_units, rapl_energy_units, rapl_time_units
7941 */
probe_rapl(void)7942 void probe_rapl(void)
7943 {
7944 probe_rapl_msrs();
7945
7946 if (genuine_intel)
7947 rapl_probe_intel();
7948 if (authentic_amd || hygon_genuine)
7949 rapl_probe_amd();
7950
7951 if (quiet)
7952 return;
7953
7954 print_rapl_sysfs();
7955
7956 if (!valid_rapl_msrs || no_msr)
7957 return;
7958
7959 for_all_cpus(print_rapl, ODD_COUNTERS);
7960 }
7961
7962 /*
7963 * MSR_IA32_TEMPERATURE_TARGET indicates the temperature where
7964 * the Thermal Control Circuit (TCC) activates.
7965 * This is usually equal to tjMax.
7966 *
7967 * Older processors do not have this MSR, so there we guess,
7968 * but also allow cmdline over-ride with -T.
7969 *
7970 * Several MSR temperature values are in units of degrees-C
7971 * below this value, including the Digital Thermal Sensor (DTS),
7972 * Package Thermal Management Sensor (PTM), and thermal event thresholds.
7973 */
set_temperature_target(PER_THREAD_PARAMS)7974 int set_temperature_target(PER_THREAD_PARAMS)
7975 {
7976 unsigned long long msr;
7977 unsigned int tcc_default, tcc_offset;
7978 int cpu;
7979
7980 UNUSED(c);
7981 UNUSED(p);
7982
7983 /* tj_max is used only for dts or ptm */
7984 if (!(do_dts || do_ptm))
7985 return 0;
7986
7987 /* this is a per-package concept */
7988 if (!is_cpu_first_thread_in_package(t, c, p))
7989 return 0;
7990
7991 cpu = t->cpu_id;
7992 if (cpu_migrate(cpu)) {
7993 fprintf(outf, "Could not migrate to CPU %d\n", cpu);
7994 return -1;
7995 }
7996
7997 if (tj_max_override != 0) {
7998 tj_max = tj_max_override;
7999 fprintf(outf, "cpu%d: Using cmdline TCC Target (%d C)\n", cpu, tj_max);
8000 return 0;
8001 }
8002
8003 /* Temperature Target MSR is Nehalem and newer only */
8004 if (!platform->has_nhm_msrs || no_msr)
8005 goto guess;
8006
8007 if (get_msr(base_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr))
8008 goto guess;
8009
8010 tcc_default = (msr >> 16) & 0xFF;
8011
8012 if (!quiet) {
8013 int bits = platform->tcc_offset_bits;
8014 unsigned long long enabled = 0;
8015
8016 if (bits && !get_msr(base_cpu, MSR_PLATFORM_INFO, &enabled))
8017 enabled = (enabled >> 30) & 1;
8018
8019 if (bits && enabled) {
8020 tcc_offset = (msr >> 24) & GENMASK(bits - 1, 0);
8021 fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C) (%d default - %d offset)\n",
8022 cpu, msr, tcc_default - tcc_offset, tcc_default, tcc_offset);
8023 } else {
8024 fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C)\n", cpu, msr, tcc_default);
8025 }
8026 }
8027
8028 if (!tcc_default)
8029 goto guess;
8030
8031 tj_max = tcc_default;
8032
8033 return 0;
8034
8035 guess:
8036 tj_max = TJMAX_DEFAULT;
8037 fprintf(outf, "cpu%d: Guessing tjMax %d C, Please use -T to specify\n", cpu, tj_max);
8038
8039 return 0;
8040 }
8041
print_thermal(PER_THREAD_PARAMS)8042 int print_thermal(PER_THREAD_PARAMS)
8043 {
8044 unsigned long long msr;
8045 unsigned int dts, dts2;
8046 int cpu;
8047
8048 UNUSED(c);
8049 UNUSED(p);
8050
8051 if (no_msr)
8052 return 0;
8053
8054 if (!(do_dts || do_ptm))
8055 return 0;
8056
8057 cpu = t->cpu_id;
8058
8059 /* DTS is per-core, no need to print for each thread */
8060 if (!is_cpu_first_thread_in_core(t, c))
8061 return 0;
8062
8063 if (cpu_migrate(cpu)) {
8064 fprintf(outf, "print_thermal: Could not migrate to CPU %d\n", cpu);
8065 return -1;
8066 }
8067
8068 if (do_ptm && is_cpu_first_core_in_package(t, p)) {
8069 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
8070 return 0;
8071
8072 dts = (msr >> 16) & 0x7F;
8073 fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_STATUS: 0x%08llx (%d C)\n", cpu, msr, tj_max - dts);
8074
8075 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr))
8076 return 0;
8077
8078 dts = (msr >> 16) & 0x7F;
8079 dts2 = (msr >> 8) & 0x7F;
8080 fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
8081 }
8082
8083 if (do_dts && debug) {
8084 unsigned int resolution;
8085
8086 if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
8087 return 0;
8088
8089 dts = (msr >> 16) & 0x7F;
8090 resolution = (msr >> 27) & 0xF;
8091 fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", cpu, msr, tj_max - dts, resolution);
8092
8093 if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr))
8094 return 0;
8095
8096 dts = (msr >> 16) & 0x7F;
8097 dts2 = (msr >> 8) & 0x7F;
8098 fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
8099 }
8100
8101 return 0;
8102 }
8103
probe_thermal(void)8104 void probe_thermal(void)
8105 {
8106 if (!access("/sys/devices/system/cpu/cpu0/thermal_throttle/core_throttle_count", R_OK))
8107 BIC_PRESENT(BIC_CORE_THROT_CNT);
8108 else
8109 BIC_NOT_PRESENT(BIC_CORE_THROT_CNT);
8110
8111 for_all_cpus(set_temperature_target, ODD_COUNTERS);
8112
8113 if (quiet)
8114 return;
8115
8116 for_all_cpus(print_thermal, ODD_COUNTERS);
8117 }
8118
get_cpu_type(PER_THREAD_PARAMS)8119 int get_cpu_type(PER_THREAD_PARAMS)
8120 {
8121 unsigned int eax, ebx, ecx, edx;
8122
8123 UNUSED(c);
8124 UNUSED(p);
8125
8126 if (!genuine_intel)
8127 return 0;
8128
8129 if (cpu_migrate(t->cpu_id)) {
8130 fprintf(outf, "Could not migrate to CPU %d\n", t->cpu_id);
8131 return -1;
8132 }
8133
8134 if (max_level < 0x1a)
8135 return 0;
8136
8137 __cpuid(0x1a, eax, ebx, ecx, edx);
8138 eax = (eax >> 24) & 0xFF;
8139 if (eax == 0x20)
8140 t->is_atom = true;
8141 return 0;
8142 }
8143
decode_feature_control_msr(void)8144 void decode_feature_control_msr(void)
8145 {
8146 unsigned long long msr;
8147
8148 if (no_msr)
8149 return;
8150
8151 if (!get_msr(base_cpu, MSR_IA32_FEAT_CTL, &msr))
8152 fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n",
8153 base_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : "");
8154 }
8155
decode_misc_enable_msr(void)8156 void decode_misc_enable_msr(void)
8157 {
8158 unsigned long long msr;
8159
8160 if (no_msr)
8161 return;
8162
8163 if (!genuine_intel)
8164 return;
8165
8166 if (!get_msr(base_cpu, MSR_IA32_MISC_ENABLE, &msr))
8167 fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n",
8168 base_cpu, msr,
8169 msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-",
8170 msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-",
8171 msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-",
8172 msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : "");
8173 }
8174
decode_misc_feature_control(void)8175 void decode_misc_feature_control(void)
8176 {
8177 unsigned long long msr;
8178
8179 if (no_msr)
8180 return;
8181
8182 if (!platform->has_msr_misc_feature_control)
8183 return;
8184
8185 if (!get_msr(base_cpu, MSR_MISC_FEATURE_CONTROL, &msr))
8186 fprintf(outf,
8187 "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n",
8188 base_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "",
8189 msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : "");
8190 }
8191
8192 /*
8193 * Decode MSR_MISC_PWR_MGMT
8194 *
8195 * Decode the bits according to the Nehalem documentation
8196 * bit[0] seems to continue to have same meaning going forward
8197 * bit[1] less so...
8198 */
decode_misc_pwr_mgmt_msr(void)8199 void decode_misc_pwr_mgmt_msr(void)
8200 {
8201 unsigned long long msr;
8202
8203 if (no_msr)
8204 return;
8205
8206 if (!platform->has_msr_misc_pwr_mgmt)
8207 return;
8208
8209 if (!get_msr(base_cpu, MSR_MISC_PWR_MGMT, &msr))
8210 fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n",
8211 base_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS");
8212 }
8213
8214 /*
8215 * Decode MSR_CC6_DEMOTION_POLICY_CONFIG, MSR_MC6_DEMOTION_POLICY_CONFIG
8216 *
8217 * This MSRs are present on Silvermont processors,
8218 * Intel Atom processor E3000 series (Baytrail), and friends.
8219 */
decode_c6_demotion_policy_msr(void)8220 void decode_c6_demotion_policy_msr(void)
8221 {
8222 unsigned long long msr;
8223
8224 if (no_msr)
8225 return;
8226
8227 if (!platform->has_msr_c6_demotion_policy_config)
8228 return;
8229
8230 if (!get_msr(base_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr))
8231 fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n",
8232 base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
8233
8234 if (!get_msr(base_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr))
8235 fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n",
8236 base_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
8237 }
8238
print_dev_latency(void)8239 void print_dev_latency(void)
8240 {
8241 char *path = "/dev/cpu_dma_latency";
8242 int fd;
8243 int value;
8244 int retval;
8245
8246 fd = open(path, O_RDONLY);
8247 if (fd < 0) {
8248 if (debug)
8249 warnx("Read %s failed", path);
8250 return;
8251 }
8252
8253 retval = read(fd, (void *)&value, sizeof(int));
8254 if (retval != sizeof(int)) {
8255 warn("read failed %s", path);
8256 close(fd);
8257 return;
8258 }
8259 fprintf(outf, "/dev/cpu_dma_latency: %d usec (%s)\n", value, value == 2000000000 ? "default" : "constrained");
8260
8261 close(fd);
8262 }
8263
has_perf_instr_count_access(void)8264 static int has_perf_instr_count_access(void)
8265 {
8266 int fd;
8267
8268 if (no_perf)
8269 return 0;
8270
8271 fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0);
8272 if (fd != -1)
8273 close(fd);
8274
8275 if (fd == -1)
8276 warnx("Failed to access %s. Some of the counters may not be available\n"
8277 "\tRun as root to enable them or use %s to disable the access explicitly", "perf instructions retired counter",
8278 "'--hide IPC' or '--no-perf'");
8279
8280 return (fd != -1);
8281 }
8282
add_rapl_perf_counter(int cpu,struct rapl_counter_info_t * rci,const struct rapl_counter_arch_info * cai,double * scale_,enum rapl_unit * unit_)8283 int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, double *scale_, enum rapl_unit *unit_)
8284 {
8285 int ret = -1;
8286
8287 if (no_perf)
8288 return -1;
8289
8290 if (!cai->perf_name)
8291 return -1;
8292
8293 const double scale = read_perf_scale(cai->perf_subsys, cai->perf_name);
8294
8295 if (scale == 0.0)
8296 goto end;
8297
8298 const enum rapl_unit unit = read_perf_rapl_unit(cai->perf_subsys, cai->perf_name);
8299
8300 if (unit == RAPL_UNIT_INVALID)
8301 goto end;
8302
8303 const unsigned int rapl_type = read_perf_type(cai->perf_subsys);
8304 const unsigned int rapl_energy_pkg_config = read_perf_config(cai->perf_subsys, cai->perf_name);
8305
8306 ret = open_perf_counter(cpu, rapl_type, rapl_energy_pkg_config, rci->fd_perf, PERF_FORMAT_GROUP);
8307 if (ret == -1)
8308 goto end;
8309
8310 /* If it's the first counter opened, make it a group descriptor */
8311 if (rci->fd_perf == -1)
8312 rci->fd_perf = ret;
8313
8314 *scale_ = scale;
8315 *unit_ = unit;
8316
8317 end:
8318 if (debug >= 2)
8319 fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
8320
8321 return ret;
8322 }
8323
8324 /*
8325 * Linux-perf manages the HW instructions-retired counter
8326 * by enabling when requested, and hiding rollover
8327 */
linux_perf_init(void)8328 void linux_perf_init(void)
8329 {
8330 if (access("/proc/sys/kernel/perf_event_paranoid", F_OK))
8331 return;
8332
8333 if (BIC_IS_ENABLED(BIC_IPC) && cpuid_has_aperf_mperf) {
8334 fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8335 if (fd_instr_count_percpu == NULL)
8336 err(-1, "calloc fd_instr_count_percpu");
8337 }
8338 if (BIC_IS_ENABLED(BIC_LLC_RPS)) {
8339 fd_llc_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8340 if (fd_llc_percpu == NULL)
8341 err(-1, "calloc fd_llc_percpu");
8342 }
8343 }
8344
rapl_perf_init(void)8345 void rapl_perf_init(void)
8346 {
8347 const unsigned int num_domains = get_rapl_num_domains();
8348 bool *domain_visited = calloc(num_domains, sizeof(bool));
8349
8350 rapl_counter_info_perdomain = calloc(num_domains, sizeof(*rapl_counter_info_perdomain));
8351 if (rapl_counter_info_perdomain == NULL)
8352 err(-1, "calloc rapl_counter_info_percpu");
8353 rapl_counter_info_perdomain_size = num_domains;
8354
8355 /*
8356 * Initialize rapl_counter_info_percpu
8357 */
8358 for (unsigned int domain_id = 0; domain_id < num_domains; ++domain_id) {
8359 struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[domain_id];
8360
8361 rci->fd_perf = -1;
8362 for (size_t i = 0; i < NUM_RAPL_COUNTERS; ++i) {
8363 rci->data[i] = 0;
8364 rci->source[i] = COUNTER_SOURCE_NONE;
8365 }
8366 }
8367
8368 /*
8369 * Open/probe the counters
8370 * If can't get it via perf, fallback to MSR
8371 */
8372 for (size_t i = 0; i < ARRAY_SIZE(rapl_counter_arch_infos); ++i) {
8373
8374 const struct rapl_counter_arch_info *const cai = &rapl_counter_arch_infos[i];
8375 bool has_counter = 0;
8376 double scale;
8377 enum rapl_unit unit;
8378 unsigned int next_domain;
8379
8380 if (!BIC_IS_ENABLED(cai->bic_number))
8381 continue;
8382
8383 memset(domain_visited, 0, num_domains * sizeof(*domain_visited));
8384
8385 for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
8386
8387 if (cpu_is_not_allowed(cpu))
8388 continue;
8389
8390 /* Skip already seen and handled RAPL domains */
8391 next_domain = get_rapl_domain_id(cpu);
8392
8393 assert(next_domain < num_domains);
8394
8395 if (domain_visited[next_domain])
8396 continue;
8397
8398 domain_visited[next_domain] = 1;
8399
8400 if ((cai->flags & RAPL_COUNTER_FLAG_PLATFORM_COUNTER) && (cpu != base_cpu))
8401 continue;
8402
8403 struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[next_domain];
8404
8405 /*
8406 * rapl_counter_arch_infos[] can have multiple entries describing the same
8407 * counter, due to the difference from different platforms/Vendors.
8408 * E.g. rapl_counter_arch_infos[0] and rapl_counter_arch_infos[1] share the
8409 * same perf_subsys and perf_name, but with different MSR address.
8410 * rapl_counter_arch_infos[0] is for Intel and rapl_counter_arch_infos[1]
8411 * is for AMD.
8412 * In this case, it is possible that multiple rapl_counter_arch_infos[]
8413 * entries are probed just because their perf/msr is duplicate and valid.
8414 *
8415 * Thus need a check to avoid re-probe the same counters.
8416 */
8417 if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE)
8418 break;
8419
8420 /* Use perf API for this counter */
8421 if (add_rapl_perf_counter(cpu, rci, cai, &scale, &unit) != -1) {
8422 rci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8423 rci->scale[cai->rci_index] = scale * cai->compat_scale;
8424 rci->unit[cai->rci_index] = unit;
8425 rci->flags[cai->rci_index] = cai->flags;
8426
8427 /* Use MSR for this counter */
8428 } else if (add_rapl_msr_counter(cpu, cai) >= 0) {
8429 rci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8430 rci->msr[cai->rci_index] = cai->msr;
8431 rci->msr_mask[cai->rci_index] = cai->msr_mask;
8432 rci->msr_shift[cai->rci_index] = cai->msr_shift;
8433 rci->unit[cai->rci_index] = RAPL_UNIT_JOULES;
8434 rci->scale[cai->rci_index] = *cai->platform_rapl_msr_scale * cai->compat_scale;
8435 rci->flags[cai->rci_index] = cai->flags;
8436 }
8437
8438 if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE)
8439 has_counter = 1;
8440 }
8441
8442 /* If any CPU has access to the counter, make it present */
8443 if (has_counter)
8444 BIC_PRESENT(cai->bic_number);
8445 }
8446
8447 free(domain_visited);
8448 }
8449
8450 /* Assumes msr_counter_info is populated */
has_amperf_access(void)8451 static int has_amperf_access(void)
8452 {
8453 return cpuid_has_aperf_mperf && msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present &&
8454 msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present;
8455 }
8456
get_cstate_perf_group_fd(struct cstate_counter_info_t * cci,const char * group_name)8457 int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *group_name)
8458 {
8459 if (strcmp(group_name, "cstate_core") == 0)
8460 return &cci->fd_perf_core;
8461
8462 if (strcmp(group_name, "cstate_pkg") == 0)
8463 return &cci->fd_perf_pkg;
8464
8465 return NULL;
8466 }
8467
add_cstate_perf_counter(int cpu,struct cstate_counter_info_t * cci,const struct cstate_counter_arch_info * cai)8468 int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai)
8469 {
8470 int ret = -1;
8471
8472 if (no_perf)
8473 return -1;
8474
8475 if (!cai->perf_name)
8476 return -1;
8477
8478 int *pfd_group = get_cstate_perf_group_fd(cci, cai->perf_subsys);
8479
8480 if (pfd_group == NULL)
8481 goto end;
8482
8483 const unsigned int type = read_perf_type(cai->perf_subsys);
8484 const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name);
8485
8486 ret = open_perf_counter(cpu, type, config, *pfd_group, PERF_FORMAT_GROUP);
8487
8488 if (ret == -1)
8489 goto end;
8490
8491 /* If it's the first counter opened, make it a group descriptor */
8492 if (*pfd_group == -1)
8493 *pfd_group = ret;
8494
8495 end:
8496 if (debug >= 2)
8497 fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
8498
8499 return ret;
8500 }
8501
add_msr_perf_counter(int cpu,struct msr_counter_info_t * cci,const struct msr_counter_arch_info * cai)8502 int add_msr_perf_counter(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai)
8503 {
8504 int ret = -1;
8505
8506 if (no_perf)
8507 return -1;
8508
8509 if (!cai->perf_name)
8510 return -1;
8511
8512 const unsigned int type = read_perf_type(cai->perf_subsys);
8513 const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name);
8514
8515 ret = open_perf_counter(cpu, type, config, cci->fd_perf, PERF_FORMAT_GROUP);
8516
8517 if (ret == -1)
8518 goto end;
8519
8520 /* If it's the first counter opened, make it a group descriptor */
8521 if (cci->fd_perf == -1)
8522 cci->fd_perf = ret;
8523
8524 end:
8525 if (debug)
8526 fprintf(stderr, "%s: %s/%s: %d (cpu: %d)\n", __func__, cai->perf_subsys, cai->perf_name, ret, cpu);
8527
8528 return ret;
8529 }
8530
msr_perf_init_(void)8531 void msr_perf_init_(void)
8532 {
8533 const int mci_num = topo.max_cpu_num + 1;
8534
8535 msr_counter_info = calloc(mci_num, sizeof(*msr_counter_info));
8536 if (!msr_counter_info)
8537 err(1, "calloc msr_counter_info");
8538 msr_counter_info_size = mci_num;
8539
8540 for (int cpu = 0; cpu < mci_num; ++cpu)
8541 msr_counter_info[cpu].fd_perf = -1;
8542
8543 for (int cidx = 0; cidx < NUM_MSR_COUNTERS; ++cidx) {
8544
8545 struct msr_counter_arch_info *cai = &msr_counter_arch_infos[cidx];
8546
8547 cai->present = false;
8548
8549 for (int cpu = 0; cpu < mci_num; ++cpu) {
8550
8551 struct msr_counter_info_t *const cci = &msr_counter_info[cpu];
8552
8553 if (cpu_is_not_allowed(cpu))
8554 continue;
8555
8556 if (cai->needed) {
8557 /* Use perf API for this counter */
8558 if (add_msr_perf_counter(cpu, cci, cai) != -1) {
8559 cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8560 cai->present = true;
8561
8562 /* User MSR for this counter */
8563 } else if (add_msr_counter(cpu, cai->msr) >= 0) {
8564 cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8565 cci->msr[cai->rci_index] = cai->msr;
8566 cci->msr_mask[cai->rci_index] = cai->msr_mask;
8567 cai->present = true;
8568 }
8569 }
8570 }
8571 }
8572 }
8573
8574 /* Initialize data for reading perf counters from the MSR group. */
msr_perf_init(void)8575 void msr_perf_init(void)
8576 {
8577 bool need_amperf = false, need_smi = false;
8578 const bool need_soft_c1 = (!platform->has_msr_core_c1_res) && (platform->supported_cstates & CC1);
8579
8580 need_amperf = BIC_IS_ENABLED(BIC_Avg_MHz) || BIC_IS_ENABLED(BIC_Busy) || BIC_IS_ENABLED(BIC_Bzy_MHz)
8581 || BIC_IS_ENABLED(BIC_IPC) || need_soft_c1;
8582
8583 if (BIC_IS_ENABLED(BIC_SMI))
8584 need_smi = true;
8585
8586 /* Enable needed counters */
8587 msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].needed = need_amperf;
8588 msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].needed = need_amperf;
8589 msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].needed = need_smi;
8590
8591 msr_perf_init_();
8592
8593 const bool has_amperf = has_amperf_access();
8594 const bool has_smi = msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].present;
8595
8596 has_aperf_access = has_amperf;
8597
8598 if (has_amperf) {
8599 BIC_PRESENT(BIC_Avg_MHz);
8600 BIC_PRESENT(BIC_Busy);
8601 BIC_PRESENT(BIC_Bzy_MHz);
8602 BIC_PRESENT(BIC_SMI);
8603 }
8604
8605 if (has_smi)
8606 BIC_PRESENT(BIC_SMI);
8607 }
8608
cstate_perf_init_(bool soft_c1)8609 void cstate_perf_init_(bool soft_c1)
8610 {
8611 bool has_counter;
8612 bool *cores_visited = NULL, *pkg_visited = NULL;
8613 const int cores_visited_elems = topo.max_core_id + 1;
8614 const int pkg_visited_elems = topo.max_package_id + 1;
8615 const int cci_num = topo.max_cpu_num + 1;
8616
8617 ccstate_counter_info = calloc(cci_num, sizeof(*ccstate_counter_info));
8618 if (!ccstate_counter_info)
8619 err(1, "calloc ccstate_counter_arch_info");
8620 ccstate_counter_info_size = cci_num;
8621
8622 cores_visited = calloc(cores_visited_elems, sizeof(*cores_visited));
8623 if (!cores_visited)
8624 err(1, "calloc cores_visited");
8625
8626 pkg_visited = calloc(pkg_visited_elems, sizeof(*pkg_visited));
8627 if (!pkg_visited)
8628 err(1, "calloc pkg_visited");
8629
8630 /* Initialize cstate_counter_info_percpu */
8631 for (int cpu = 0; cpu < cci_num; ++cpu) {
8632 ccstate_counter_info[cpu].fd_perf_core = -1;
8633 ccstate_counter_info[cpu].fd_perf_pkg = -1;
8634 }
8635
8636 for (int cidx = 0; cidx < NUM_CSTATE_COUNTERS; ++cidx) {
8637 has_counter = false;
8638 memset(cores_visited, 0, cores_visited_elems * sizeof(*cores_visited));
8639 memset(pkg_visited, 0, pkg_visited_elems * sizeof(*pkg_visited));
8640
8641 const struct cstate_counter_arch_info *cai = &ccstate_counter_arch_infos[cidx];
8642
8643 for (int cpu = 0; cpu < cci_num; ++cpu) {
8644
8645 struct cstate_counter_info_t *const cci = &ccstate_counter_info[cpu];
8646
8647 if (cpu_is_not_allowed(cpu))
8648 continue;
8649
8650 const int core_id = cpus[cpu].physical_core_id;
8651 const int pkg_id = cpus[cpu].physical_package_id;
8652
8653 assert(core_id < cores_visited_elems);
8654 assert(pkg_id < pkg_visited_elems);
8655
8656 const bool per_thread = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD;
8657 const bool per_core = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_CORE;
8658
8659 if (!per_thread && cores_visited[core_id])
8660 continue;
8661
8662 if (!per_core && pkg_visited[pkg_id])
8663 continue;
8664
8665 const bool counter_needed = BIC_IS_ENABLED(cai->bic_number) ||
8666 (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY));
8667 const bool counter_supported = (platform->supported_cstates & cai->feature_mask);
8668
8669 if (counter_needed && counter_supported) {
8670 /* Use perf API for this counter */
8671 if (add_cstate_perf_counter(cpu, cci, cai) != -1) {
8672
8673 cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8674
8675 /* User MSR for this counter */
8676 } else if (pkg_cstate_limit >= cai->pkg_cstate_limit && add_msr_counter(cpu, cai->msr) >= 0) {
8677 cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8678 cci->msr[cai->rci_index] = cai->msr;
8679 }
8680 }
8681
8682 if (cci->source[cai->rci_index] != COUNTER_SOURCE_NONE) {
8683 has_counter = true;
8684 cores_visited[core_id] = true;
8685 pkg_visited[pkg_id] = true;
8686 }
8687 }
8688
8689 /* If any CPU has access to the counter, make it present */
8690 if (has_counter)
8691 BIC_PRESENT(cai->bic_number);
8692 }
8693
8694 free(cores_visited);
8695 free(pkg_visited);
8696 }
8697
cstate_perf_init(void)8698 void cstate_perf_init(void)
8699 {
8700 /*
8701 * If we don't have a C1 residency MSR, we calculate it "in software",
8702 * but we need APERF, MPERF too.
8703 */
8704 const bool soft_c1 = !platform->has_msr_core_c1_res && has_amperf_access()
8705 && platform->supported_cstates & CC1;
8706
8707 if (soft_c1)
8708 BIC_PRESENT(BIC_CPU_c1);
8709
8710 cstate_perf_init_(soft_c1);
8711 }
8712
probe_cstates(void)8713 void probe_cstates(void)
8714 {
8715 probe_cst_limit();
8716
8717 if (platform->has_msr_module_c6_res_ms)
8718 BIC_PRESENT(BIC_Mod_c6);
8719
8720 if (platform->has_ext_cst_msrs && !no_msr) {
8721 BIC_PRESENT(BIC_Totl_c0);
8722 BIC_PRESENT(BIC_Any_c0);
8723 BIC_PRESENT(BIC_GFX_c0);
8724 BIC_PRESENT(BIC_CPUGFX);
8725 }
8726
8727 if (quiet)
8728 return;
8729
8730 dump_power_ctl();
8731 dump_cst_cfg();
8732 decode_c6_demotion_policy_msr();
8733 print_dev_latency();
8734 dump_sysfs_cstate_config();
8735 print_irtl();
8736 }
8737
probe_lpi(void)8738 void probe_lpi(void)
8739 {
8740 if (!access("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", R_OK))
8741 BIC_PRESENT(BIC_CPU_LPI);
8742 else
8743 BIC_NOT_PRESENT(BIC_CPU_LPI);
8744
8745 if (!access(sys_lpi_file_sysfs, R_OK)) {
8746 sys_lpi_file = sys_lpi_file_sysfs;
8747 BIC_PRESENT(BIC_SYS_LPI);
8748 } else if (!access(sys_lpi_file_debugfs, R_OK)) {
8749 sys_lpi_file = sys_lpi_file_debugfs;
8750 BIC_PRESENT(BIC_SYS_LPI);
8751 } else {
8752 sys_lpi_file_sysfs = NULL;
8753 BIC_NOT_PRESENT(BIC_SYS_LPI);
8754 }
8755
8756 }
8757
probe_pstates(void)8758 void probe_pstates(void)
8759 {
8760 probe_bclk();
8761
8762 if (quiet)
8763 return;
8764
8765 dump_platform_info();
8766 dump_turbo_ratio_info();
8767 dump_sysfs_pstate_config();
8768 decode_misc_pwr_mgmt_msr();
8769
8770 for_all_cpus(print_hwp, ODD_COUNTERS);
8771 for_all_cpus(print_epb, ODD_COUNTERS);
8772 for_all_cpus(print_perf_limit, ODD_COUNTERS);
8773 }
8774
process_cpuid()8775 void process_cpuid()
8776 {
8777 unsigned int eax, ebx, ecx, edx;
8778 unsigned int fms, family, model, stepping, ecx_flags, edx_flags;
8779 unsigned long long ucode_patch = 0;
8780 bool ucode_patch_valid = false;
8781
8782 eax = ebx = ecx = edx = 0;
8783
8784 __cpuid(0, max_level, ebx, ecx, edx);
8785
8786 if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
8787 genuine_intel = 1;
8788 else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
8789 authentic_amd = 1;
8790 else if (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e)
8791 hygon_genuine = 1;
8792
8793 if (!quiet)
8794 fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", (char *)&ebx, (char *)&edx, (char *)&ecx, max_level);
8795
8796 __cpuid(1, fms, ebx, ecx, edx);
8797 family = (fms >> 8) & 0xf;
8798 model = (fms >> 4) & 0xf;
8799 stepping = fms & 0xf;
8800 if (family == 0xf)
8801 family += (fms >> 20) & 0xff;
8802 if (family >= 6)
8803 model += ((fms >> 16) & 0xf) << 4;
8804 ecx_flags = ecx;
8805 edx_flags = edx;
8806
8807 if (!no_msr) {
8808 if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch))
8809 warnx("get_msr(UCODE)");
8810 else
8811 ucode_patch_valid = true;
8812 }
8813
8814 /*
8815 * check max extended function levels of CPUID.
8816 * This is needed to check for invariant TSC.
8817 * This check is valid for both Intel and AMD.
8818 */
8819 ebx = ecx = edx = 0;
8820 __cpuid(0x80000000, max_extended_level, ebx, ecx, edx);
8821
8822 if (!quiet) {
8823 fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", family, model, stepping, family, model, stepping);
8824 if (ucode_patch_valid)
8825 fprintf(outf, " microcode 0x%x", (unsigned int)((ucode_patch >> 32) & 0xFFFFFFFF));
8826 fputc('\n', outf);
8827
8828 fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level);
8829 fprintf(outf, "CPUID(1): %s %s %s %s %s %s %s %s %s %s\n",
8830 ecx_flags & (1 << 0) ? "SSE3" : "-",
8831 ecx_flags & (1 << 3) ? "MONITOR" : "-",
8832 ecx_flags & (1 << 6) ? "SMX" : "-",
8833 ecx_flags & (1 << 7) ? "EIST" : "-",
8834 ecx_flags & (1 << 8) ? "TM2" : "-",
8835 edx_flags & (1 << 4) ? "TSC" : "-",
8836 edx_flags & (1 << 5) ? "MSR" : "-",
8837 edx_flags & (1 << 22) ? "ACPI-TM" : "-", edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-");
8838 }
8839
8840 probe_platform_features(family, model);
8841
8842 if (!(edx_flags & (1 << 5)))
8843 errx(1, "CPUID: no MSR");
8844
8845 if (max_extended_level >= 0x80000007) {
8846
8847 /*
8848 * Non-Stop TSC is advertised by CPUID.EAX=0x80000007: EDX.bit8
8849 * this check is valid for both Intel and AMD
8850 */
8851 __cpuid(0x80000007, eax, ebx, ecx, edx);
8852 has_invariant_tsc = edx & (1 << 8);
8853 }
8854
8855 /*
8856 * APERF/MPERF is advertised by CPUID.EAX=0x6: ECX.bit0
8857 * this check is valid for both Intel and AMD
8858 */
8859
8860 __cpuid(0x6, eax, ebx, ecx, edx);
8861 cpuid_has_aperf_mperf = ecx & (1 << 0);
8862 do_dts = eax & (1 << 0);
8863 if (do_dts)
8864 BIC_PRESENT(BIC_CoreTmp);
8865 has_turbo = eax & (1 << 1);
8866 do_ptm = eax & (1 << 6);
8867 if (do_ptm)
8868 BIC_PRESENT(BIC_PkgTmp);
8869 has_hwp = eax & (1 << 7);
8870 has_hwp_notify = eax & (1 << 8);
8871 has_hwp_activity_window = eax & (1 << 9);
8872 has_hwp_epp = eax & (1 << 10);
8873 has_hwp_pkg = eax & (1 << 11);
8874 has_epb = ecx & (1 << 3);
8875
8876 if (!quiet)
8877 fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, "
8878 "%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n",
8879 cpuid_has_aperf_mperf ? "" : "No-",
8880 has_turbo ? "" : "No-",
8881 do_dts ? "" : "No-",
8882 do_ptm ? "" : "No-",
8883 has_hwp ? "" : "No-",
8884 has_hwp_notify ? "" : "No-",
8885 has_hwp_activity_window ? "" : "No-", has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-");
8886
8887 if (!quiet)
8888 decode_misc_enable_msr();
8889
8890 if (max_level >= 0x7 && !quiet) {
8891 int has_sgx;
8892
8893 ecx = 0;
8894
8895 __cpuid_count(0x7, 0, eax, ebx, ecx, edx);
8896
8897 has_sgx = ebx & (1 << 2);
8898
8899 is_hybrid = edx & (1 << 15);
8900
8901 fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-");
8902
8903 if (has_sgx)
8904 decode_feature_control_msr();
8905 }
8906
8907 if (max_level >= 0x15) {
8908 unsigned int eax_crystal;
8909 unsigned int ebx_tsc;
8910
8911 /*
8912 * CPUID 15H TSC/Crystal ratio, possibly Crystal Hz
8913 */
8914 eax_crystal = ebx_tsc = crystal_hz = edx = 0;
8915 __cpuid(0x15, eax_crystal, ebx_tsc, crystal_hz, edx);
8916
8917 if (ebx_tsc != 0) {
8918 if (!quiet && (ebx != 0))
8919 fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", eax_crystal, ebx_tsc, crystal_hz);
8920
8921 if (crystal_hz == 0)
8922 crystal_hz = platform->crystal_freq;
8923
8924 if (crystal_hz) {
8925 tsc_hz = (unsigned long long)crystal_hz *ebx_tsc / eax_crystal;
8926 if (!quiet)
8927 fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n",
8928 tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal);
8929 }
8930 }
8931 }
8932 if (max_level >= 0x16) {
8933 unsigned int base_mhz, max_mhz, bus_mhz, edx;
8934
8935 /*
8936 * CPUID 16H Base MHz, Max MHz, Bus MHz
8937 */
8938 base_mhz = max_mhz = bus_mhz = edx = 0;
8939
8940 __cpuid(0x16, base_mhz, max_mhz, bus_mhz, edx);
8941
8942 bclk = bus_mhz;
8943
8944 base_hz = base_mhz * 1000000;
8945 has_base_hz = 1;
8946
8947 if (platform->enable_tsc_tweak)
8948 tsc_tweak = base_hz / tsc_hz;
8949
8950 if (!quiet)
8951 fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", base_mhz, max_mhz, bus_mhz);
8952 }
8953
8954 if (cpuid_has_aperf_mperf)
8955 aperf_mperf_multiplier = platform->need_perf_multiplier ? 1024 : 1;
8956
8957 BIC_PRESENT(BIC_IRQ);
8958 BIC_PRESENT(BIC_NMI);
8959 BIC_PRESENT(BIC_TSC_MHz);
8960 }
8961
counter_info_init(void)8962 static void counter_info_init(void)
8963 {
8964 for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i) {
8965 struct cstate_counter_arch_info *const cai = &ccstate_counter_arch_infos[i];
8966
8967 if (platform->has_msr_knl_core_c6_residency && cai->msr == MSR_CORE_C6_RESIDENCY)
8968 cai->msr = MSR_KNL_CORE_C6_RESIDENCY;
8969
8970 if (!platform->has_msr_core_c1_res && cai->msr == MSR_CORE_C1_RES)
8971 cai->msr = 0;
8972
8973 if (platform->has_msr_atom_pkg_c6_residency && cai->msr == MSR_PKG_C6_RESIDENCY)
8974 cai->msr = MSR_ATOM_PKG_C6_RESIDENCY;
8975 }
8976
8977 for (int i = 0; i < NUM_MSR_COUNTERS; ++i) {
8978 msr_counter_arch_infos[i].present = false;
8979 msr_counter_arch_infos[i].needed = false;
8980 }
8981 }
8982
probe_pm_features(void)8983 void probe_pm_features(void)
8984 {
8985 probe_pstates();
8986
8987 probe_cstates();
8988
8989 probe_lpi();
8990
8991 probe_intel_uncore_frequency();
8992
8993 probe_graphics();
8994
8995 probe_rapl();
8996
8997 probe_thermal();
8998
8999 if (platform->has_nhm_msrs && !no_msr)
9000 BIC_PRESENT(BIC_SMI);
9001
9002 if (!quiet)
9003 decode_misc_feature_control();
9004 }
9005
9006 /* perf_llc_probe
9007 *
9008 * return 1 on success, else 0
9009 */
has_perf_llc_access(void)9010 int has_perf_llc_access(void)
9011 {
9012 int fd;
9013
9014 if (no_perf)
9015 return 0;
9016
9017 fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP);
9018 if (fd != -1)
9019 close(fd);
9020
9021 if (fd == -1)
9022 warnx("Failed to access %s. Some of the counters may not be available\n"
9023 "\tRun as root to enable them or use %s to disable the access explicitly", "perf LLC counters", "'--hide LLC' or '--no-perf'");
9024
9025 return (fd != -1);
9026 }
9027
perf_llc_init(void)9028 void perf_llc_init(void)
9029 {
9030 int cpu;
9031 int retval;
9032
9033 if (no_perf)
9034 return;
9035 if (!(BIC_IS_ENABLED(BIC_LLC_RPS) && BIC_IS_ENABLED(BIC_LLC_HIT)))
9036 return;
9037
9038 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
9039
9040 if (cpu_is_not_allowed(cpu))
9041 continue;
9042
9043 assert(fd_llc_percpu != 0);
9044 fd_llc_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP);
9045 if (fd_llc_percpu[cpu] == -1) {
9046 warnx("%s: perf REFS: failed to open counter on cpu%d", __func__, cpu);
9047 free_fd_llc_percpu();
9048 return;
9049 }
9050 assert(fd_llc_percpu != 0);
9051 retval = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, fd_llc_percpu[cpu], PERF_FORMAT_GROUP);
9052 if (retval == -1) {
9053 warnx("%s: perf MISS: failed to open counter on cpu%d", __func__, cpu);
9054 free_fd_llc_percpu();
9055 return;
9056 }
9057 }
9058 BIC_PRESENT(BIC_LLC_RPS);
9059 BIC_PRESENT(BIC_LLC_HIT);
9060 }
9061
9062 /*
9063 * in /dev/cpu/ return success for names that are numbers
9064 * ie. filter out ".", "..", "microcode".
9065 */
dir_filter(const struct dirent * dirp)9066 int dir_filter(const struct dirent *dirp)
9067 {
9068 if (isdigit(dirp->d_name[0]))
9069 return 1;
9070 else
9071 return 0;
9072 }
9073
9074 char *possible_file = "/sys/devices/system/cpu/possible";
9075 char possible_buf[1024];
9076
initialize_cpu_possible_set(void)9077 int initialize_cpu_possible_set(void)
9078 {
9079 FILE *fp;
9080
9081 fp = fopen(possible_file, "r");
9082 if (!fp) {
9083 warn("open %s", possible_file);
9084 return -1;
9085 }
9086 if (fread(possible_buf, sizeof(char), 1024, fp) == 0) {
9087 warn("read %s", possible_file);
9088 goto err;
9089 }
9090 if (parse_cpu_str(possible_buf, cpu_possible_set, cpu_possible_setsize)) {
9091 warnx("%s: cpu str malformat %s\n", possible_file, cpu_effective_str);
9092 goto err;
9093 }
9094 return 0;
9095
9096 err:
9097 fclose(fp);
9098 return -1;
9099 }
9100
topology_probe(bool startup)9101 void topology_probe(bool startup)
9102 {
9103 int i;
9104 int max_core_id = 0;
9105 int max_package_id = 0;
9106 int max_siblings = 0;
9107
9108 /* Initialize num_cpus, max_cpu_num */
9109 set_max_cpu_num();
9110 topo.num_cpus = 0;
9111 for_all_proc_cpus(count_cpus);
9112 if (!summary_only)
9113 BIC_PRESENT(BIC_CPU);
9114
9115 if (debug > 1)
9116 fprintf(outf, "num_cpus %d max_cpu_num %d\n", topo.num_cpus, topo.max_cpu_num);
9117
9118 cpus = calloc(1, (topo.max_cpu_num + 1) * sizeof(struct cpu_topology));
9119 if (cpus == NULL)
9120 err(1, "calloc cpus");
9121
9122 /*
9123 * Allocate and initialize cpu_present_set
9124 */
9125 cpu_present_set = CPU_ALLOC((topo.max_cpu_num + 1));
9126 if (cpu_present_set == NULL)
9127 err(3, "CPU_ALLOC");
9128 cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9129 CPU_ZERO_S(cpu_present_setsize, cpu_present_set);
9130 for_all_proc_cpus(mark_cpu_present);
9131
9132 /*
9133 * Allocate and initialize cpu_possible_set
9134 */
9135 cpu_possible_set = CPU_ALLOC((topo.max_cpu_num + 1));
9136 if (cpu_possible_set == NULL)
9137 err(3, "CPU_ALLOC");
9138 cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9139 CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set);
9140 initialize_cpu_possible_set();
9141
9142 /*
9143 * Allocate and initialize cpu_effective_set
9144 */
9145 cpu_effective_set = CPU_ALLOC((topo.max_cpu_num + 1));
9146 if (cpu_effective_set == NULL)
9147 err(3, "CPU_ALLOC");
9148 cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9149 CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set);
9150 update_effective_set(startup);
9151
9152 /*
9153 * Allocate and initialize cpu_allowed_set
9154 */
9155 cpu_allowed_set = CPU_ALLOC((topo.max_cpu_num + 1));
9156 if (cpu_allowed_set == NULL)
9157 err(3, "CPU_ALLOC");
9158 cpu_allowed_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9159 CPU_ZERO_S(cpu_allowed_setsize, cpu_allowed_set);
9160
9161 /*
9162 * Validate and update cpu_allowed_set.
9163 *
9164 * Make sure all cpus in cpu_subset are also in cpu_present_set during startup.
9165 * Give a warning when cpus in cpu_subset become unavailable at runtime.
9166 * Give a warning when cpus are not effective because of cgroup setting.
9167 *
9168 * cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpu_subset.
9169 */
9170 for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) {
9171 if (cpu_subset && !CPU_ISSET_S(i, cpu_subset_size, cpu_subset))
9172 continue;
9173
9174 if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set)) {
9175 if (cpu_subset) {
9176 /* cpus in cpu_subset must be in cpu_present_set during startup */
9177 if (startup)
9178 err(1, "cpu%d not present", i);
9179 else
9180 fprintf(stderr, "cpu%d not present\n", i);
9181 }
9182 continue;
9183 }
9184
9185 if (CPU_COUNT_S(cpu_effective_setsize, cpu_effective_set)) {
9186 if (!CPU_ISSET_S(i, cpu_effective_setsize, cpu_effective_set)) {
9187 fprintf(stderr, "cpu%d not effective\n", i);
9188 continue;
9189 }
9190 }
9191
9192 CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set);
9193 }
9194
9195 if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set))
9196 err(-ENODEV, "No valid cpus found");
9197 sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set);
9198
9199 /*
9200 * Allocate and initialize cpu_affinity_set
9201 */
9202 cpu_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1));
9203 if (cpu_affinity_set == NULL)
9204 err(3, "CPU_ALLOC");
9205 cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9206 CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
9207
9208 for_all_proc_cpus(init_thread_id);
9209
9210 for_all_proc_cpus(set_cpu_hybrid_type);
9211
9212 /*
9213 * For online cpus
9214 * find max_core_id, max_package_id
9215 */
9216 for (i = 0; i <= topo.max_cpu_num; ++i) {
9217 int siblings;
9218
9219 if (cpu_is_not_present(i)) {
9220 if (debug > 1)
9221 fprintf(outf, "cpu%d NOT PRESENT\n", i);
9222 continue;
9223 }
9224
9225 cpus[i].logical_cpu_id = i;
9226
9227 /* get package information */
9228 cpus[i].physical_package_id = get_physical_package_id(i);
9229 if (cpus[i].physical_package_id > max_package_id)
9230 max_package_id = cpus[i].physical_package_id;
9231
9232 /* get die information */
9233 cpus[i].die_id = get_die_id(i);
9234 if (cpus[i].die_id > topo.max_die_id)
9235 topo.max_die_id = cpus[i].die_id;
9236
9237 /* get l3 information */
9238 cpus[i].l3_id = get_l3_id(i);
9239 if (cpus[i].l3_id > topo.max_l3_id)
9240 topo.max_l3_id = cpus[i].l3_id;
9241
9242 /* get numa node information */
9243 cpus[i].physical_node_id = get_physical_node_id(&cpus[i]);
9244 if (cpus[i].physical_node_id > topo.max_node_num)
9245 topo.max_node_num = cpus[i].physical_node_id;
9246
9247 /* get core information */
9248 cpus[i].physical_core_id = get_core_id(i);
9249 if (cpus[i].physical_core_id > max_core_id)
9250 max_core_id = cpus[i].physical_core_id;
9251
9252 /* get thread information */
9253 siblings = get_thread_siblings(&cpus[i]);
9254 if (siblings > max_siblings)
9255 max_siblings = siblings;
9256 if (cpus[i].thread_id == 0)
9257 topo.num_cores++;
9258 }
9259 topo.max_core_id = max_core_id;
9260 topo.max_package_id = max_package_id;
9261
9262 topo.cores_per_node = max_core_id + 1;
9263 if (debug > 1)
9264 fprintf(outf, "max_core_id %d, sizing for %d cores per package\n", max_core_id, topo.cores_per_node);
9265 if (!summary_only)
9266 BIC_PRESENT(BIC_Core);
9267
9268 topo.num_die = topo.max_die_id + 1;
9269 if (debug > 1)
9270 fprintf(outf, "max_die_id %d, sizing for %d die\n", topo.max_die_id, topo.num_die);
9271 if (!summary_only && topo.num_die > 1)
9272 BIC_PRESENT(BIC_Die);
9273
9274 if (!summary_only && topo.max_l3_id > 0)
9275 BIC_PRESENT(BIC_L3);
9276
9277 topo.num_packages = max_package_id + 1;
9278 if (debug > 1)
9279 fprintf(outf, "max_package_id %d, sizing for %d packages\n", max_package_id, topo.num_packages);
9280 if (!summary_only && topo.num_packages > 1)
9281 BIC_PRESENT(BIC_Package);
9282
9283 set_node_data();
9284 if (debug > 1)
9285 fprintf(outf, "nodes_per_pkg %d\n", topo.nodes_per_pkg);
9286 if (!summary_only && topo.nodes_per_pkg > 1)
9287 BIC_PRESENT(BIC_Node);
9288
9289 topo.threads_per_core = max_siblings;
9290 if (debug > 1)
9291 fprintf(outf, "max_siblings %d\n", max_siblings);
9292
9293 if (debug < 1)
9294 return;
9295
9296 for (i = 0; i <= topo.max_cpu_num; ++i) {
9297 if (cpu_is_not_present(i))
9298 continue;
9299 fprintf(outf,
9300 "cpu %d pkg %d die %d l3 %d node %d lnode %d core %d thread %d\n",
9301 i, cpus[i].physical_package_id, cpus[i].die_id, cpus[i].l3_id,
9302 cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].physical_core_id, cpus[i].thread_id);
9303 }
9304
9305 }
9306
allocate_counters(struct thread_data ** t,struct core_data ** c,struct pkg_data ** p)9307 void allocate_counters(struct thread_data **t, struct core_data **c, struct pkg_data **p)
9308 {
9309 int i;
9310 int num_cores = topo.cores_per_node * topo.nodes_per_pkg * topo.num_packages;
9311 int num_threads = topo.threads_per_core * num_cores;
9312
9313 *t = calloc(num_threads, sizeof(struct thread_data));
9314 if (*t == NULL)
9315 goto error;
9316
9317 for (i = 0; i < num_threads; i++)
9318 (*t)[i].cpu_id = -1;
9319
9320 *c = calloc(num_cores, sizeof(struct core_data));
9321 if (*c == NULL)
9322 goto error;
9323
9324 for (i = 0; i < num_cores; i++) {
9325 (*c)[i].core_id = -1;
9326 (*c)[i].base_cpu = -1;
9327 }
9328
9329 *p = calloc(topo.num_packages, sizeof(struct pkg_data));
9330 if (*p == NULL)
9331 goto error;
9332
9333 for (i = 0; i < topo.num_packages; i++) {
9334 (*p)[i].package_id = i;
9335 (*p)[i].base_cpu = -1;
9336 }
9337
9338 return;
9339 error:
9340 err(1, "calloc counters");
9341 }
9342
9343 /*
9344 * init_counter()
9345 *
9346 * set FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE
9347 */
init_counter(struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base,int cpu_id)9348 void init_counter(struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base, int cpu_id)
9349 {
9350 int pkg_id = cpus[cpu_id].physical_package_id;
9351 int node_id = cpus[cpu_id].logical_node_id;
9352 int core_id = cpus[cpu_id].physical_core_id;
9353 int thread_id = cpus[cpu_id].thread_id;
9354 struct thread_data *t;
9355 struct core_data *c;
9356
9357 /* Workaround for systems where physical_node_id==-1
9358 * and logical_node_id==(-1 - topo.num_cpus)
9359 */
9360 if (node_id < 0)
9361 node_id = 0;
9362
9363 t = GET_THREAD(thread_base, thread_id, core_id, node_id, pkg_id);
9364 c = GET_CORE(core_base, core_id, node_id, pkg_id);
9365
9366 t->cpu_id = cpu_id;
9367 if (!cpu_is_not_allowed(cpu_id)) {
9368
9369 if (c->base_cpu < 0)
9370 c->base_cpu = t->cpu_id;
9371 if (pkg_base[pkg_id].base_cpu < 0)
9372 pkg_base[pkg_id].base_cpu = t->cpu_id;
9373 }
9374
9375 c->core_id = core_id;
9376 pkg_base[pkg_id].package_id = pkg_id;
9377 }
9378
initialize_counters(int cpu_id)9379 int initialize_counters(int cpu_id)
9380 {
9381 init_counter(EVEN_COUNTERS, cpu_id);
9382 init_counter(ODD_COUNTERS, cpu_id);
9383 return 0;
9384 }
9385
allocate_output_buffer()9386 void allocate_output_buffer()
9387 {
9388 output_buffer = calloc(1, (1 + topo.num_cpus) * 2048);
9389 outp = output_buffer;
9390 if (outp == NULL)
9391 err(-1, "calloc output buffer");
9392 }
9393
allocate_fd_percpu(void)9394 void allocate_fd_percpu(void)
9395 {
9396 fd_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9397 if (fd_percpu == NULL)
9398 err(-1, "calloc fd_percpu");
9399 }
9400
allocate_irq_buffers(void)9401 void allocate_irq_buffers(void)
9402 {
9403 irq_column_2_cpu = calloc(topo.num_cpus, sizeof(int));
9404 if (irq_column_2_cpu == NULL)
9405 err(-1, "calloc %d", topo.num_cpus);
9406
9407 irqs_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9408 if (irqs_per_cpu == NULL)
9409 err(-1, "calloc %d IRQ", topo.max_cpu_num + 1);
9410
9411 nmi_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9412 if (nmi_per_cpu == NULL)
9413 err(-1, "calloc %d NMI", topo.max_cpu_num + 1);
9414 }
9415
update_topo(PER_THREAD_PARAMS)9416 int update_topo(PER_THREAD_PARAMS)
9417 {
9418 topo.allowed_cpus++;
9419 if ((int)t->cpu_id == c->base_cpu)
9420 topo.allowed_cores++;
9421 if ((int)t->cpu_id == p->base_cpu)
9422 topo.allowed_packages++;
9423
9424 return 0;
9425 }
9426
topology_update(void)9427 void topology_update(void)
9428 {
9429 topo.allowed_cpus = 0;
9430 topo.allowed_cores = 0;
9431 topo.allowed_packages = 0;
9432 for_all_cpus(update_topo, ODD_COUNTERS);
9433 }
9434
setup_all_buffers(bool startup)9435 void setup_all_buffers(bool startup)
9436 {
9437 topology_probe(startup);
9438 allocate_irq_buffers();
9439 allocate_fd_percpu();
9440 allocate_counters(&thread_even, &core_even, &package_even);
9441 allocate_counters(&thread_odd, &core_odd, &package_odd);
9442 allocate_output_buffer();
9443 for_all_proc_cpus(initialize_counters);
9444 topology_update();
9445 }
9446
set_base_cpu(void)9447 void set_base_cpu(void)
9448 {
9449 int i;
9450
9451 for (i = 0; i < topo.max_cpu_num + 1; ++i) {
9452 if (cpu_is_not_allowed(i))
9453 continue;
9454 base_cpu = i;
9455 if (debug > 1)
9456 fprintf(outf, "base_cpu = %d\n", base_cpu);
9457 return;
9458 }
9459 err(-ENODEV, "No valid cpus found");
9460 }
9461
has_added_counters(void)9462 bool has_added_counters(void)
9463 {
9464 /*
9465 * It only makes sense to call this after the command line is parsed,
9466 * otherwise sys structure is not populated.
9467 */
9468
9469 return sys.added_core_counters | sys.added_thread_counters | sys.added_package_counters;
9470 }
9471
check_msr_access(void)9472 void check_msr_access(void)
9473 {
9474 check_msr_driver();
9475 check_msr_permission();
9476
9477 if (no_msr)
9478 bic_disable_msr_access();
9479 }
9480
check_perf_access(void)9481 void check_perf_access(void)
9482 {
9483 if (BIC_IS_ENABLED(BIC_IPC))
9484 if (!has_perf_instr_count_access())
9485 no_perf = 1;
9486
9487 if (BIC_IS_ENABLED(BIC_LLC_RPS) || BIC_IS_ENABLED(BIC_LLC_HIT))
9488 if (!has_perf_llc_access())
9489 no_perf = 1;
9490
9491 if (no_perf)
9492 bic_disable_perf_access();
9493 }
9494
perf_has_hybrid_devices(void)9495 bool perf_has_hybrid_devices(void)
9496 {
9497 /*
9498 * 0: unknown
9499 * 1: has separate perf device for p and e core
9500 * -1: doesn't have separate perf device for p and e core
9501 */
9502 static int cached;
9503
9504 if (cached > 0)
9505 return true;
9506
9507 if (cached < 0)
9508 return false;
9509
9510 if (access("/sys/bus/event_source/devices/cpu_core", F_OK)) {
9511 cached = -1;
9512 return false;
9513 }
9514
9515 if (access("/sys/bus/event_source/devices/cpu_atom", F_OK)) {
9516 cached = -1;
9517 return false;
9518 }
9519
9520 cached = 1;
9521 return true;
9522 }
9523
added_perf_counters_init_(struct perf_counter_info * pinfo)9524 int added_perf_counters_init_(struct perf_counter_info *pinfo)
9525 {
9526 size_t num_domains = 0;
9527 unsigned int next_domain;
9528 bool *domain_visited;
9529 unsigned int perf_type, perf_config;
9530 double perf_scale;
9531 int fd_perf;
9532
9533 if (!pinfo)
9534 return 0;
9535
9536 const size_t max_num_domains = MAX(topo.max_cpu_num + 1, MAX(topo.max_core_id + 1, topo.max_package_id + 1));
9537
9538 domain_visited = calloc(max_num_domains, sizeof(*domain_visited));
9539
9540 while (pinfo) {
9541 switch (pinfo->scope) {
9542 case SCOPE_CPU:
9543 num_domains = topo.max_cpu_num + 1;
9544 break;
9545
9546 case SCOPE_CORE:
9547 num_domains = topo.max_core_id + 1;
9548 break;
9549
9550 case SCOPE_PACKAGE:
9551 num_domains = topo.max_package_id + 1;
9552 break;
9553 }
9554
9555 /* Allocate buffer for file descriptor for each domain. */
9556 pinfo->fd_perf_per_domain = calloc(num_domains, sizeof(*pinfo->fd_perf_per_domain));
9557 if (!pinfo->fd_perf_per_domain)
9558 errx(1, "%s: alloc %s", __func__, "fd_perf_per_domain");
9559
9560 for (size_t i = 0; i < num_domains; ++i)
9561 pinfo->fd_perf_per_domain[i] = -1;
9562
9563 pinfo->num_domains = num_domains;
9564 pinfo->scale = 1.0;
9565
9566 memset(domain_visited, 0, max_num_domains * sizeof(*domain_visited));
9567
9568 for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
9569
9570 next_domain = cpu_to_domain(pinfo, cpu);
9571
9572 assert(next_domain < num_domains);
9573
9574 if (cpu_is_not_allowed(cpu))
9575 continue;
9576
9577 if (domain_visited[next_domain])
9578 continue;
9579
9580 /*
9581 * Intel hybrid platforms expose different perf devices for P and E cores.
9582 * Instead of one, "/sys/bus/event_source/devices/cpu" device, there are
9583 * "/sys/bus/event_source/devices/{cpu_core,cpu_atom}".
9584 *
9585 * This makes it more complicated to the user, because most of the counters
9586 * are available on both and have to be handled manually, otherwise.
9587 *
9588 * Code below, allow user to use the old "cpu" name, which is translated accordingly.
9589 */
9590 const char *perf_device = pinfo->device;
9591
9592 if (strcmp(perf_device, "cpu") == 0 && perf_has_hybrid_devices()) {
9593 switch (cpus[cpu].type) {
9594 case INTEL_PCORE_TYPE:
9595 perf_device = "cpu_core";
9596 break;
9597
9598 case INTEL_ECORE_TYPE:
9599 perf_device = "cpu_atom";
9600 break;
9601
9602 default: /* Don't change, we will probably fail and report a problem soon. */
9603 break;
9604 }
9605 }
9606
9607 perf_type = read_perf_type(perf_device);
9608 if (perf_type == (unsigned int)-1) {
9609 warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "type");
9610 continue;
9611 }
9612
9613 perf_config = read_perf_config(perf_device, pinfo->event);
9614 if (perf_config == (unsigned int)-1) {
9615 warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "config");
9616 continue;
9617 }
9618
9619 /* Scale is not required, some counters just don't have it. */
9620 perf_scale = read_perf_scale(perf_device, pinfo->event);
9621 if (perf_scale == 0.0)
9622 perf_scale = 1.0;
9623
9624 fd_perf = open_perf_counter(cpu, perf_type, perf_config, -1, 0);
9625 if (fd_perf == -1) {
9626 warnx("%s: perf/%s/%s: failed to open counter on cpu%d", __func__, perf_device, pinfo->event, cpu);
9627 continue;
9628 }
9629
9630 domain_visited[next_domain] = 1;
9631 pinfo->fd_perf_per_domain[next_domain] = fd_perf;
9632 pinfo->scale = perf_scale;
9633
9634 if (debug)
9635 fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n", perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]);
9636 }
9637
9638 pinfo = pinfo->next;
9639 }
9640
9641 free(domain_visited);
9642
9643 return 0;
9644 }
9645
added_perf_counters_init(void)9646 void added_perf_counters_init(void)
9647 {
9648 if (added_perf_counters_init_(sys.perf_tp))
9649 errx(1, "%s: %s", __func__, "thread");
9650
9651 if (added_perf_counters_init_(sys.perf_cp))
9652 errx(1, "%s: %s", __func__, "core");
9653
9654 if (added_perf_counters_init_(sys.perf_pp))
9655 errx(1, "%s: %s", __func__, "package");
9656 }
9657
parse_telem_info_file(int fd_dir,const char * info_filename,const char * format,unsigned long * output)9658 int parse_telem_info_file(int fd_dir, const char *info_filename, const char *format, unsigned long *output)
9659 {
9660 int fd_telem_info;
9661 FILE *file_telem_info;
9662 unsigned long value;
9663
9664 fd_telem_info = openat(fd_dir, info_filename, O_RDONLY);
9665 if (fd_telem_info == -1)
9666 return -1;
9667
9668 file_telem_info = fdopen(fd_telem_info, "r");
9669 if (file_telem_info == NULL) {
9670 close(fd_telem_info);
9671 return -1;
9672 }
9673
9674 if (fscanf(file_telem_info, format, &value) != 1) {
9675 fclose(file_telem_info);
9676 return -1;
9677 }
9678
9679 fclose(file_telem_info);
9680
9681 *output = value;
9682
9683 return 0;
9684 }
9685
pmt_mmio_open(unsigned int target_guid)9686 struct pmt_mmio *pmt_mmio_open(unsigned int target_guid)
9687 {
9688 struct pmt_diriter_t pmt_iter;
9689 const struct dirent *entry;
9690 struct stat st;
9691 int fd_telem_dir, fd_pmt;
9692 unsigned long guid, size, offset;
9693 size_t mmap_size;
9694 void *mmio;
9695 struct pmt_mmio *head = NULL, *last = NULL;
9696 struct pmt_mmio *new_pmt = NULL;
9697
9698 if (stat(SYSFS_TELEM_PATH, &st) == -1)
9699 return NULL;
9700
9701 pmt_diriter_init(&pmt_iter);
9702 entry = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH);
9703 if (!entry) {
9704 pmt_diriter_remove(&pmt_iter);
9705 return NULL;
9706 }
9707
9708 for (; entry != NULL; entry = pmt_diriter_next(&pmt_iter)) {
9709 if (fstatat(dirfd(pmt_iter.dir), entry->d_name, &st, 0) == -1)
9710 break;
9711
9712 if (!S_ISDIR(st.st_mode))
9713 continue;
9714
9715 fd_telem_dir = openat(dirfd(pmt_iter.dir), entry->d_name, O_RDONLY);
9716 if (fd_telem_dir == -1)
9717 break;
9718
9719 if (parse_telem_info_file(fd_telem_dir, "guid", "%lx", &guid)) {
9720 close(fd_telem_dir);
9721 break;
9722 }
9723
9724 if (parse_telem_info_file(fd_telem_dir, "size", "%lu", &size)) {
9725 close(fd_telem_dir);
9726 break;
9727 }
9728
9729 if (guid != target_guid) {
9730 close(fd_telem_dir);
9731 continue;
9732 }
9733
9734 if (parse_telem_info_file(fd_telem_dir, "offset", "%lu", &offset)) {
9735 close(fd_telem_dir);
9736 break;
9737 }
9738
9739 assert(offset == 0);
9740
9741 fd_pmt = openat(fd_telem_dir, "telem", O_RDONLY);
9742 if (fd_pmt == -1)
9743 goto loop_cleanup_and_break;
9744
9745 mmap_size = ROUND_UP_TO_PAGE_SIZE(size);
9746 mmio = mmap(0, mmap_size, PROT_READ, MAP_SHARED, fd_pmt, 0);
9747 if (mmio != MAP_FAILED) {
9748 if (debug)
9749 fprintf(stderr, "%s: 0x%lx mmaped at: %p\n", __func__, guid, mmio);
9750
9751 new_pmt = calloc(1, sizeof(*new_pmt));
9752
9753 if (!new_pmt) {
9754 fprintf(stderr, "%s: Failed to allocate pmt_mmio\n", __func__);
9755 exit(1);
9756 }
9757
9758 /*
9759 * Create linked list of mmaped regions,
9760 * but preserve the ordering from sysfs.
9761 * Ordering is important for the user to
9762 * use the seq=%u parameter when adding a counter.
9763 */
9764 new_pmt->guid = guid;
9765 new_pmt->mmio_base = mmio;
9766 new_pmt->pmt_offset = offset;
9767 new_pmt->size = size;
9768 new_pmt->next = pmt_mmios;
9769
9770 if (last)
9771 last->next = new_pmt;
9772 else
9773 head = new_pmt;
9774
9775 last = new_pmt;
9776 }
9777
9778 loop_cleanup_and_break:
9779 close(fd_pmt);
9780 close(fd_telem_dir);
9781 }
9782
9783 pmt_diriter_remove(&pmt_iter);
9784
9785 /*
9786 * If we found something, stick just
9787 * created linked list to the front.
9788 */
9789 if (head)
9790 pmt_mmios = head;
9791
9792 return head;
9793 }
9794
pmt_mmio_find(unsigned int guid)9795 struct pmt_mmio *pmt_mmio_find(unsigned int guid)
9796 {
9797 struct pmt_mmio *pmmio = pmt_mmios;
9798
9799 while (pmmio) {
9800 if (pmmio->guid == guid)
9801 return pmmio;
9802
9803 pmmio = pmmio->next;
9804 }
9805
9806 return NULL;
9807 }
9808
pmt_get_counter_pointer(struct pmt_mmio * pmmio,unsigned long counter_offset)9809 void *pmt_get_counter_pointer(struct pmt_mmio *pmmio, unsigned long counter_offset)
9810 {
9811 char *ret;
9812
9813 /* Get base of mmaped PMT file. */
9814 ret = (char *)pmmio->mmio_base;
9815
9816 /*
9817 * Apply PMT MMIO offset to obtain beginning of the mmaped telemetry data.
9818 * It's not guaranteed that the mmaped memory begins with the telemetry data
9819 * - we might have to apply the offset first.
9820 */
9821 ret += pmmio->pmt_offset;
9822
9823 /* Apply the counter offset to get the address to the mmaped counter. */
9824 ret += counter_offset;
9825
9826 return ret;
9827 }
9828
pmt_add_guid(unsigned int guid,unsigned int seq)9829 struct pmt_mmio *pmt_add_guid(unsigned int guid, unsigned int seq)
9830 {
9831 struct pmt_mmio *ret;
9832
9833 ret = pmt_mmio_find(guid);
9834 if (!ret)
9835 ret = pmt_mmio_open(guid);
9836
9837 while (ret && seq) {
9838 ret = ret->next;
9839 --seq;
9840 }
9841
9842 return ret;
9843 }
9844
9845 enum pmt_open_mode {
9846 PMT_OPEN_TRY, /* Open failure is not an error. */
9847 PMT_OPEN_REQUIRED, /* Open failure is a fatal error. */
9848 };
9849
pmt_find_counter(struct pmt_counter * pcounter,const char * name)9850 struct pmt_counter *pmt_find_counter(struct pmt_counter *pcounter, const char *name)
9851 {
9852 while (pcounter) {
9853 if (strcmp(pcounter->name, name) == 0)
9854 break;
9855
9856 pcounter = pcounter->next;
9857 }
9858
9859 return pcounter;
9860 }
9861
pmt_get_scope_root(enum counter_scope scope)9862 struct pmt_counter **pmt_get_scope_root(enum counter_scope scope)
9863 {
9864 switch (scope) {
9865 case SCOPE_CPU:
9866 return &sys.pmt_tp;
9867 case SCOPE_CORE:
9868 return &sys.pmt_cp;
9869 case SCOPE_PACKAGE:
9870 return &sys.pmt_pp;
9871 }
9872
9873 __builtin_unreachable();
9874 }
9875
pmt_counter_add_domain(struct pmt_counter * pcounter,unsigned long * pmmio,unsigned int domain_id)9876 void pmt_counter_add_domain(struct pmt_counter *pcounter, unsigned long *pmmio, unsigned int domain_id)
9877 {
9878 /* Make sure the new domain fits. */
9879 if (domain_id >= pcounter->num_domains)
9880 pmt_counter_resize(pcounter, domain_id + 1);
9881
9882 assert(pcounter->domains);
9883 assert(domain_id < pcounter->num_domains);
9884
9885 pcounter->domains[domain_id].pcounter = pmmio;
9886 }
9887
pmt_add_counter(unsigned int guid,unsigned int seq,const char * name,enum pmt_datatype type,unsigned int lsb,unsigned int msb,unsigned int offset,enum counter_scope scope,enum counter_format format,unsigned int domain_id,enum pmt_open_mode mode)9888 int pmt_add_counter(unsigned int guid, unsigned int seq, const char *name, enum pmt_datatype type,
9889 unsigned int lsb, unsigned int msb, unsigned int offset, enum counter_scope scope,
9890 enum counter_format format, unsigned int domain_id, enum pmt_open_mode mode)
9891 {
9892 struct pmt_mmio *mmio;
9893 struct pmt_counter *pcounter;
9894 struct pmt_counter **const pmt_root = pmt_get_scope_root(scope);
9895 bool new_counter = false;
9896 int conflict = 0;
9897
9898 if (lsb > msb) {
9899 fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "lsb <= msb", name);
9900 exit(1);
9901 }
9902
9903 if (msb >= 64) {
9904 fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "msb < 64", name);
9905 exit(1);
9906 }
9907
9908 mmio = pmt_add_guid(guid, seq);
9909 if (!mmio) {
9910 if (mode != PMT_OPEN_TRY) {
9911 fprintf(stderr, "%s: failed to map PMT MMIO for guid %x, seq %u\n", __func__, guid, seq);
9912 exit(1);
9913 }
9914
9915 return 1;
9916 }
9917
9918 if (offset >= mmio->size) {
9919 if (mode != PMT_OPEN_TRY) {
9920 fprintf(stderr, "%s: offset %u outside of PMT MMIO size %u\n", __func__, offset, mmio->size);
9921 exit(1);
9922 }
9923
9924 return 1;
9925 }
9926
9927 pcounter = pmt_find_counter(*pmt_root, name);
9928 if (!pcounter) {
9929 pcounter = calloc(1, sizeof(*pcounter));
9930 new_counter = true;
9931 }
9932
9933 if (new_counter) {
9934 strncpy(pcounter->name, name, ARRAY_SIZE(pcounter->name) - 1);
9935 pcounter->type = type;
9936 pcounter->scope = scope;
9937 pcounter->lsb = lsb;
9938 pcounter->msb = msb;
9939 pcounter->format = format;
9940 } else {
9941 conflict += pcounter->type != type;
9942 conflict += pcounter->scope != scope;
9943 conflict += pcounter->lsb != lsb;
9944 conflict += pcounter->msb != msb;
9945 conflict += pcounter->format != format;
9946 }
9947
9948 if (conflict) {
9949 fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n", __func__, name);
9950 exit(1);
9951 }
9952
9953 pmt_counter_add_domain(pcounter, pmt_get_counter_pointer(mmio, offset), domain_id);
9954
9955 if (new_counter) {
9956 pcounter->next = *pmt_root;
9957 *pmt_root = pcounter;
9958 }
9959
9960 return 0;
9961 }
9962
pmt_init(void)9963 void pmt_init(void)
9964 {
9965 int cpu_num;
9966 unsigned long seq, offset, mod_num;
9967
9968 if (BIC_IS_ENABLED(BIC_Diec6)) {
9969 pmt_add_counter(PMT_MTL_DC6_GUID, PMT_MTL_DC6_SEQ, "Die%c6", PMT_TYPE_XTAL_TIME,
9970 PMT_COUNTER_MTL_DC6_LSB, PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET,
9971 SCOPE_PACKAGE, FORMAT_DELTA, 0, PMT_OPEN_TRY);
9972 }
9973
9974 if (BIC_IS_ENABLED(BIC_CPU_c1e)) {
9975 seq = 0;
9976 offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE;
9977 mod_num = 0; /* Relative module number for current PMT file. */
9978
9979 /* Open the counter for each CPU. */
9980 for (cpu_num = 0; cpu_num < topo.max_cpu_num;) {
9981
9982 if (cpu_is_not_allowed(cpu_num))
9983 goto next_loop_iter;
9984
9985 /*
9986 * Set the scope to CPU, even though CWF report the counter per module.
9987 * CPUs inside the same module will read from the same location, instead of reporting zeros.
9988 *
9989 * CWF with newer firmware might require a PMT_TYPE_XTAL_TIME intead of PMT_TYPE_TCORE_CLOCK.
9990 */
9991 pmt_add_counter(PMT_CWF_MC1E_GUID, seq, "CPU%c1e", PMT_TYPE_TCORE_CLOCK,
9992 PMT_COUNTER_CWF_MC1E_LSB, PMT_COUNTER_CWF_MC1E_MSB, offset, SCOPE_CPU, FORMAT_DELTA, cpu_num, PMT_OPEN_TRY);
9993
9994 /*
9995 * Rather complex logic for each time we go to the next loop iteration,
9996 * so keep it as a label.
9997 */
9998 next_loop_iter:
9999 /*
10000 * Advance the cpu number and check if we should also advance offset to
10001 * the next counter inside the PMT file.
10002 *
10003 * On Clearwater Forest platform, the counter is reported per module,
10004 * so open the same counter for all of the CPUs inside the module.
10005 * That way, reported table show the correct value for all of the CPUs inside the module,
10006 * instead of zeros.
10007 */
10008 ++cpu_num;
10009 if (cpu_num % PMT_COUNTER_CWF_CPUS_PER_MODULE == 0) {
10010 offset += PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT;
10011 ++mod_num;
10012 }
10013
10014 /*
10015 * There are PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE in each PMT file.
10016 *
10017 * If that number is reached, seq must be incremented to advance to the next file in a sequence.
10018 * Offset inside that file and a module counter has to be reset.
10019 */
10020 if (mod_num == PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE) {
10021 ++seq;
10022 offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE;
10023 mod_num = 0;
10024 }
10025 }
10026 }
10027 }
10028
turbostat_init()10029 void turbostat_init()
10030 {
10031 setup_all_buffers(true);
10032 set_base_cpu();
10033 check_msr_access();
10034 check_perf_access();
10035 process_cpuid();
10036 counter_info_init();
10037 probe_pm_features();
10038 msr_perf_init();
10039 linux_perf_init();
10040 rapl_perf_init();
10041 cstate_perf_init();
10042 perf_llc_init();
10043 added_perf_counters_init();
10044 pmt_init();
10045
10046 for_all_cpus(get_cpu_type, ODD_COUNTERS);
10047 for_all_cpus(get_cpu_type, EVEN_COUNTERS);
10048
10049 if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(base_cpu) != -1)
10050 BIC_PRESENT(BIC_IPC);
10051
10052 /*
10053 * If TSC tweak is needed, but couldn't get it,
10054 * disable more BICs, since it can't be reported accurately.
10055 */
10056 if (platform->enable_tsc_tweak && !has_base_hz) {
10057 CLR_BIC(BIC_Busy, &bic_enabled);
10058 CLR_BIC(BIC_Bzy_MHz, &bic_enabled);
10059 }
10060 }
10061
affinitize_child(void)10062 void affinitize_child(void)
10063 {
10064 /* Prefer cpu_possible_set, if available */
10065 if (sched_setaffinity(0, cpu_possible_setsize, cpu_possible_set)) {
10066 warn("sched_setaffinity cpu_possible_set");
10067
10068 /* Otherwise, allow child to run on same cpu set as turbostat */
10069 if (sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set))
10070 warn("sched_setaffinity cpu_allowed_set");
10071 }
10072 }
10073
fork_it(char ** argv)10074 int fork_it(char **argv)
10075 {
10076 pid_t child_pid;
10077 int status;
10078
10079 snapshot_proc_sysfs_files();
10080 status = for_all_cpus(get_counters, EVEN_COUNTERS);
10081 first_counter_read = 0;
10082 if (status)
10083 exit(status);
10084 gettimeofday(&tv_even, (struct timezone *)NULL);
10085
10086 child_pid = fork();
10087 if (!child_pid) {
10088 /* child */
10089 affinitize_child();
10090 execvp(argv[0], argv);
10091 err(errno, "exec %s", argv[0]);
10092 } else {
10093
10094 /* parent */
10095 if (child_pid == -1)
10096 err(1, "fork");
10097
10098 signal(SIGINT, SIG_IGN);
10099 signal(SIGQUIT, SIG_IGN);
10100 if (waitpid(child_pid, &status, 0) == -1)
10101 err(status, "waitpid");
10102
10103 if (WIFEXITED(status))
10104 status = WEXITSTATUS(status);
10105 }
10106 /*
10107 * n.b. fork_it() does not check for errors from for_all_cpus()
10108 * because re-starting is problematic when forking
10109 */
10110 snapshot_proc_sysfs_files();
10111 for_all_cpus(get_counters, ODD_COUNTERS);
10112 gettimeofday(&tv_odd, (struct timezone *)NULL);
10113 timersub(&tv_odd, &tv_even, &tv_delta);
10114 if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS))
10115 fprintf(outf, "%s: Counter reset detected\n", progname);
10116 delta_platform(&platform_counters_odd, &platform_counters_even);
10117
10118 compute_average(EVEN_COUNTERS);
10119 format_all_counters(EVEN_COUNTERS);
10120
10121 fprintf(outf, "%.6f sec\n", tv_delta.tv_sec + tv_delta.tv_usec / 1000000.0);
10122
10123 flush_output_stderr();
10124
10125 return status;
10126 }
10127
get_and_dump_counters(void)10128 int get_and_dump_counters(void)
10129 {
10130 int status;
10131
10132 snapshot_proc_sysfs_files();
10133 status = for_all_cpus(get_counters, ODD_COUNTERS);
10134 if (status)
10135 return status;
10136
10137 status = for_all_cpus(dump_counters, ODD_COUNTERS);
10138 if (status)
10139 return status;
10140
10141 flush_output_stdout();
10142
10143 return status;
10144 }
10145
print_version()10146 void print_version()
10147 {
10148 fprintf(outf, "turbostat version 2025.12.02 - Len Brown <lenb@kernel.org>\n");
10149 }
10150
10151 #define COMMAND_LINE_SIZE 2048
10152
print_bootcmd(void)10153 void print_bootcmd(void)
10154 {
10155 char bootcmd[COMMAND_LINE_SIZE];
10156 FILE *fp;
10157 int ret;
10158
10159 memset(bootcmd, 0, COMMAND_LINE_SIZE);
10160 fp = fopen("/proc/cmdline", "r");
10161 if (!fp)
10162 return;
10163
10164 ret = fread(bootcmd, sizeof(char), COMMAND_LINE_SIZE - 1, fp);
10165 if (ret) {
10166 bootcmd[ret] = '\0';
10167 /* the last character is already '\n' */
10168 fprintf(outf, "Kernel command line: %s", bootcmd);
10169 }
10170
10171 fclose(fp);
10172 }
10173
find_msrp_by_name(struct msr_counter * head,char * name)10174 struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name)
10175 {
10176 struct msr_counter *mp;
10177
10178 for (mp = head; mp; mp = mp->next) {
10179 if (debug)
10180 fprintf(stderr, "%s: %s %s\n", __func__, name, mp->name);
10181 if (!strcmp(name, mp->name))
10182 return mp;
10183 }
10184 return NULL;
10185 }
10186
add_counter(unsigned int msr_num,char * path,char * name,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format,int flags,int id)10187 int add_counter(unsigned int msr_num, char *path, char *name,
10188 unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int id)
10189 {
10190 struct msr_counter *msrp;
10191
10192 if (no_msr && msr_num)
10193 errx(1, "Requested MSR counter 0x%x, but in --no-msr mode", msr_num);
10194
10195 if (debug)
10196 fprintf(stderr, "%s(msr%d, %s, %s, width%d, scope%d, type%d, format%d, flags%x, id%d)\n",
10197 __func__, msr_num, path, name, width, scope, type, format, flags, id);
10198
10199 switch (scope) {
10200
10201 case SCOPE_CPU:
10202 msrp = find_msrp_by_name(sys.tp, name);
10203 if (msrp) {
10204 if (debug)
10205 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10206 break;
10207 }
10208 if (sys.added_thread_counters++ >= MAX_ADDED_THREAD_COUNTERS) {
10209 warnx("ignoring thread counter %s", name);
10210 return -1;
10211 }
10212 break;
10213 case SCOPE_CORE:
10214 msrp = find_msrp_by_name(sys.cp, name);
10215 if (msrp) {
10216 if (debug)
10217 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10218 break;
10219 }
10220 if (sys.added_core_counters++ >= MAX_ADDED_CORE_COUNTERS) {
10221 warnx("ignoring core counter %s", name);
10222 return -1;
10223 }
10224 break;
10225 case SCOPE_PACKAGE:
10226 msrp = find_msrp_by_name(sys.pp, name);
10227 if (msrp) {
10228 if (debug)
10229 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10230 break;
10231 }
10232 if (sys.added_package_counters++ >= MAX_ADDED_PACKAGE_COUNTERS) {
10233 warnx("ignoring package counter %s", name);
10234 return -1;
10235 }
10236 break;
10237 default:
10238 warnx("ignoring counter %s with unknown scope", name);
10239 return -1;
10240 }
10241
10242 if (msrp == NULL) {
10243 msrp = calloc(1, sizeof(struct msr_counter));
10244 if (msrp == NULL)
10245 err(-1, "calloc msr_counter");
10246
10247 msrp->msr_num = msr_num;
10248 strncpy(msrp->name, name, NAME_BYTES - 1);
10249 msrp->width = width;
10250 msrp->type = type;
10251 msrp->format = format;
10252 msrp->flags = flags;
10253
10254 switch (scope) {
10255 case SCOPE_CPU:
10256 msrp->next = sys.tp;
10257 sys.tp = msrp;
10258 break;
10259 case SCOPE_CORE:
10260 msrp->next = sys.cp;
10261 sys.cp = msrp;
10262 break;
10263 case SCOPE_PACKAGE:
10264 msrp->next = sys.pp;
10265 sys.pp = msrp;
10266 break;
10267 }
10268 }
10269
10270 if (path) {
10271 struct sysfs_path *sp;
10272
10273 sp = calloc(1, sizeof(struct sysfs_path));
10274 if (sp == NULL) {
10275 perror("calloc");
10276 exit(1);
10277 }
10278 strncpy(sp->path, path, PATH_BYTES - 1);
10279 sp->id = id;
10280 sp->next = msrp->sp;
10281 msrp->sp = sp;
10282 }
10283
10284 return 0;
10285 }
10286
10287 /*
10288 * Initialize the fields used for identifying and opening the counter.
10289 *
10290 * Defer the initialization of any runtime buffers for actually reading
10291 * the counters for when we initialize all perf counters, so we can later
10292 * easily call re_initialize().
10293 */
make_perf_counter_info(const char * perf_device,const char * perf_event,const char * name,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format)10294 struct perf_counter_info *make_perf_counter_info(const char *perf_device,
10295 const char *perf_event,
10296 const char *name,
10297 unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format)
10298 {
10299 struct perf_counter_info *pinfo;
10300
10301 pinfo = calloc(1, sizeof(*pinfo));
10302 if (!pinfo)
10303 errx(1, "%s: Failed to allocate %s/%s\n", __func__, perf_device, perf_event);
10304
10305 strncpy(pinfo->device, perf_device, ARRAY_SIZE(pinfo->device) - 1);
10306 strncpy(pinfo->event, perf_event, ARRAY_SIZE(pinfo->event) - 1);
10307
10308 strncpy(pinfo->name, name, ARRAY_SIZE(pinfo->name) - 1);
10309 pinfo->width = width;
10310 pinfo->scope = scope;
10311 pinfo->type = type;
10312 pinfo->format = format;
10313
10314 return pinfo;
10315 }
10316
add_perf_counter(const char * perf_device,const char * perf_event,const char * name_buffer,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format)10317 int add_perf_counter(const char *perf_device, const char *perf_event, const char *name_buffer, unsigned int width,
10318 enum counter_scope scope, enum counter_type type, enum counter_format format)
10319 {
10320 struct perf_counter_info *pinfo;
10321
10322 switch (scope) {
10323 case SCOPE_CPU:
10324 if (sys.added_thread_perf_counters >= MAX_ADDED_THREAD_COUNTERS) {
10325 warnx("ignoring thread counter perf/%s/%s", perf_device, perf_event);
10326 return -1;
10327 }
10328 break;
10329
10330 case SCOPE_CORE:
10331 if (sys.added_core_perf_counters >= MAX_ADDED_CORE_COUNTERS) {
10332 warnx("ignoring core counter perf/%s/%s", perf_device, perf_event);
10333 return -1;
10334 }
10335 break;
10336
10337 case SCOPE_PACKAGE:
10338 if (sys.added_package_perf_counters >= MAX_ADDED_PACKAGE_COUNTERS) {
10339 warnx("ignoring package counter perf/%s/%s", perf_device, perf_event);
10340 return -1;
10341 }
10342 break;
10343 }
10344
10345 pinfo = make_perf_counter_info(perf_device, perf_event, name_buffer, width, scope, type, format);
10346
10347 if (!pinfo)
10348 return -1;
10349
10350 switch (scope) {
10351 case SCOPE_CPU:
10352 pinfo->next = sys.perf_tp;
10353 sys.perf_tp = pinfo;
10354 ++sys.added_thread_perf_counters;
10355 break;
10356
10357 case SCOPE_CORE:
10358 pinfo->next = sys.perf_cp;
10359 sys.perf_cp = pinfo;
10360 ++sys.added_core_perf_counters;
10361 break;
10362
10363 case SCOPE_PACKAGE:
10364 pinfo->next = sys.perf_pp;
10365 sys.perf_pp = pinfo;
10366 ++sys.added_package_perf_counters;
10367 break;
10368 }
10369
10370 // FIXME: we might not have debug here yet
10371 if (debug)
10372 fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n", __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope);
10373
10374 return 0;
10375 }
10376
parse_add_command_msr(char * add_command)10377 void parse_add_command_msr(char *add_command)
10378 {
10379 int msr_num = 0;
10380 char *path = NULL;
10381 char perf_device[PERF_DEV_NAME_BYTES] = "";
10382 char perf_event[PERF_EVT_NAME_BYTES] = "";
10383 char name_buffer[PERF_NAME_BYTES] = "";
10384 int width = 64;
10385 int fail = 0;
10386 enum counter_scope scope = SCOPE_CPU;
10387 enum counter_type type = COUNTER_CYCLES;
10388 enum counter_format format = FORMAT_DELTA;
10389
10390 while (add_command) {
10391
10392 if (sscanf(add_command, "msr0x%x", &msr_num) == 1)
10393 goto next;
10394
10395 if (sscanf(add_command, "msr%d", &msr_num) == 1)
10396 goto next;
10397
10398 BUILD_BUG_ON(ARRAY_SIZE(perf_device) <= 31);
10399 BUILD_BUG_ON(ARRAY_SIZE(perf_event) <= 31);
10400 if (sscanf(add_command, "perf/%31[^/]/%31[^,]", &perf_device[0], &perf_event[0]) == 2)
10401 goto next;
10402
10403 if (*add_command == '/') {
10404 path = add_command;
10405 goto next;
10406 }
10407
10408 if (sscanf(add_command, "u%d", &width) == 1) {
10409 if ((width == 32) || (width == 64))
10410 goto next;
10411 width = 64;
10412 }
10413 if (!strncmp(add_command, "cpu", strlen("cpu"))) {
10414 scope = SCOPE_CPU;
10415 goto next;
10416 }
10417 if (!strncmp(add_command, "core", strlen("core"))) {
10418 scope = SCOPE_CORE;
10419 goto next;
10420 }
10421 if (!strncmp(add_command, "package", strlen("package"))) {
10422 scope = SCOPE_PACKAGE;
10423 goto next;
10424 }
10425 if (!strncmp(add_command, "cycles", strlen("cycles"))) {
10426 type = COUNTER_CYCLES;
10427 goto next;
10428 }
10429 if (!strncmp(add_command, "seconds", strlen("seconds"))) {
10430 type = COUNTER_SECONDS;
10431 goto next;
10432 }
10433 if (!strncmp(add_command, "usec", strlen("usec"))) {
10434 type = COUNTER_USEC;
10435 goto next;
10436 }
10437 if (!strncmp(add_command, "raw", strlen("raw"))) {
10438 format = FORMAT_RAW;
10439 goto next;
10440 }
10441 if (!strncmp(add_command, "average", strlen("average"))) {
10442 format = FORMAT_AVERAGE;
10443 goto next;
10444 }
10445 if (!strncmp(add_command, "delta", strlen("delta"))) {
10446 format = FORMAT_DELTA;
10447 goto next;
10448 }
10449 if (!strncmp(add_command, "percent", strlen("percent"))) {
10450 format = FORMAT_PERCENT;
10451 goto next;
10452 }
10453
10454 BUILD_BUG_ON(ARRAY_SIZE(name_buffer) <= 18);
10455 if (sscanf(add_command, "%18s,%*s", name_buffer) == 1) {
10456 char *eos;
10457
10458 eos = strchr(name_buffer, ',');
10459 if (eos)
10460 *eos = '\0';
10461 goto next;
10462 }
10463
10464 next:
10465 add_command = strchr(add_command, ',');
10466 if (add_command) {
10467 *add_command = '\0';
10468 add_command++;
10469 }
10470
10471 }
10472 if ((msr_num == 0) && (path == NULL) && (perf_device[0] == '\0' || perf_event[0] == '\0')) {
10473 fprintf(stderr, "--add: (msrDDD | msr0xXXX | /path_to_counter | perf/device/event) required\n");
10474 fail++;
10475 }
10476
10477 /* Test for non-empty perf_device and perf_event */
10478 const bool is_perf_counter = perf_device[0] && perf_event[0];
10479
10480 /* generate default column header */
10481 if (*name_buffer == '\0') {
10482 if (is_perf_counter) {
10483 snprintf(name_buffer, ARRAY_SIZE(name_buffer), "perf/%s", perf_event);
10484 } else {
10485 if (width == 32)
10486 sprintf(name_buffer, "M0x%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
10487 else
10488 sprintf(name_buffer, "M0X%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
10489 }
10490 }
10491
10492 if (is_perf_counter) {
10493 if (add_perf_counter(perf_device, perf_event, name_buffer, width, scope, type, format))
10494 fail++;
10495 } else {
10496 if (add_counter(msr_num, path, name_buffer, width, scope, type, format, 0, 0))
10497 fail++;
10498 }
10499
10500 if (fail) {
10501 help();
10502 exit(1);
10503 }
10504 }
10505
starts_with(const char * str,const char * prefix)10506 bool starts_with(const char *str, const char *prefix)
10507 {
10508 return strncmp(prefix, str, strlen(prefix)) == 0;
10509 }
10510
pmt_parse_from_path(const char * target_path,unsigned int * out_guid,unsigned int * out_seq)10511 int pmt_parse_from_path(const char *target_path, unsigned int *out_guid, unsigned int *out_seq)
10512 {
10513 struct pmt_diriter_t pmt_iter;
10514 const struct dirent *dirname;
10515 struct stat stat, target_stat;
10516 int fd_telem_dir = -1;
10517 int fd_target_dir;
10518 unsigned int seq = 0;
10519 unsigned long guid, target_guid;
10520 int ret = -1;
10521
10522 fd_target_dir = open(target_path, O_RDONLY | O_DIRECTORY);
10523 if (fd_target_dir == -1) {
10524 return -1;
10525 }
10526
10527 if (fstat(fd_target_dir, &target_stat) == -1) {
10528 fprintf(stderr, "%s: Failed to stat the target: %s", __func__, strerror(errno));
10529 exit(1);
10530 }
10531
10532 if (parse_telem_info_file(fd_target_dir, "guid", "%lx", &target_guid)) {
10533 fprintf(stderr, "%s: Failed to parse the target guid file: %s", __func__, strerror(errno));
10534 exit(1);
10535 }
10536
10537 close(fd_target_dir);
10538
10539 pmt_diriter_init(&pmt_iter);
10540
10541 for (dirname = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH); dirname != NULL; dirname = pmt_diriter_next(&pmt_iter)) {
10542
10543 fd_telem_dir = openat(dirfd(pmt_iter.dir), dirname->d_name, O_RDONLY | O_DIRECTORY);
10544 if (fd_telem_dir == -1)
10545 continue;
10546
10547 if (parse_telem_info_file(fd_telem_dir, "guid", "%lx", &guid)) {
10548 fprintf(stderr, "%s: Failed to parse the guid file: %s", __func__, strerror(errno));
10549 continue;
10550 }
10551
10552 if (fstat(fd_telem_dir, &stat) == -1) {
10553 fprintf(stderr, "%s: Failed to stat %s directory: %s", __func__, dirname->d_name, strerror(errno));
10554 continue;
10555 }
10556
10557 /*
10558 * If reached the same directory as target, exit the loop.
10559 * Seq has the correct value now.
10560 */
10561 if (stat.st_dev == target_stat.st_dev && stat.st_ino == target_stat.st_ino) {
10562 ret = 0;
10563 break;
10564 }
10565
10566 /*
10567 * If reached directory with the same guid,
10568 * but it's not the target directory yet,
10569 * increment seq and continue the search.
10570 */
10571 if (guid == target_guid)
10572 ++seq;
10573
10574 close(fd_telem_dir);
10575 fd_telem_dir = -1;
10576 }
10577
10578 pmt_diriter_remove(&pmt_iter);
10579
10580 if (fd_telem_dir != -1)
10581 close(fd_telem_dir);
10582
10583 if (!ret) {
10584 *out_guid = target_guid;
10585 *out_seq = seq;
10586 }
10587
10588 return ret;
10589 }
10590
parse_add_command_pmt(char * add_command)10591 void parse_add_command_pmt(char *add_command)
10592 {
10593 char *name = NULL;
10594 char *type_name = NULL;
10595 char *format_name = NULL;
10596 char *direct_path = NULL;
10597 static const char direct_path_prefix[] = "path=";
10598 unsigned int offset;
10599 unsigned int lsb;
10600 unsigned int msb;
10601 unsigned int guid;
10602 unsigned int seq = 0; /* By default, pick first file in a sequence with a given GUID. */
10603 unsigned int domain_id;
10604 enum counter_scope scope = 0;
10605 enum pmt_datatype type = PMT_TYPE_RAW;
10606 enum counter_format format = FORMAT_RAW;
10607 bool has_offset = false;
10608 bool has_lsb = false;
10609 bool has_msb = false;
10610 bool has_format = true; /* Format has a default value. */
10611 bool has_guid = false;
10612 bool has_scope = false;
10613 bool has_type = true; /* Type has a default value. */
10614
10615 /* Consume the "pmt," prefix. */
10616 add_command = strchr(add_command, ',');
10617 if (!add_command) {
10618 help();
10619 exit(1);
10620 }
10621 ++add_command;
10622
10623 while (add_command) {
10624 if (starts_with(add_command, "name=")) {
10625 name = add_command + strlen("name=");
10626 goto next;
10627 }
10628
10629 if (starts_with(add_command, "type=")) {
10630 type_name = add_command + strlen("type=");
10631 goto next;
10632 }
10633
10634 if (starts_with(add_command, "domain=")) {
10635 const size_t prefix_len = strlen("domain=");
10636
10637 if (sscanf(add_command + prefix_len, "cpu%u", &domain_id) == 1) {
10638 scope = SCOPE_CPU;
10639 has_scope = true;
10640 } else if (sscanf(add_command + prefix_len, "core%u", &domain_id) == 1) {
10641 scope = SCOPE_CORE;
10642 has_scope = true;
10643 } else if (sscanf(add_command + prefix_len, "package%u", &domain_id) == 1) {
10644 scope = SCOPE_PACKAGE;
10645 has_scope = true;
10646 }
10647
10648 if (!has_scope) {
10649 printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n", __func__);
10650 exit(1);
10651 }
10652
10653 goto next;
10654 }
10655
10656 if (starts_with(add_command, "format=")) {
10657 format_name = add_command + strlen("format=");
10658 goto next;
10659 }
10660
10661 if (sscanf(add_command, "offset=%u", &offset) == 1) {
10662 has_offset = true;
10663 goto next;
10664 }
10665
10666 if (sscanf(add_command, "lsb=%u", &lsb) == 1) {
10667 has_lsb = true;
10668 goto next;
10669 }
10670
10671 if (sscanf(add_command, "msb=%u", &msb) == 1) {
10672 has_msb = true;
10673 goto next;
10674 }
10675
10676 if (sscanf(add_command, "guid=%x", &guid) == 1) {
10677 has_guid = true;
10678 goto next;
10679 }
10680
10681 if (sscanf(add_command, "seq=%x", &seq) == 1)
10682 goto next;
10683
10684 if (strncmp(add_command, direct_path_prefix, strlen(direct_path_prefix)) == 0) {
10685 direct_path = add_command + strlen(direct_path_prefix);
10686 goto next;
10687 }
10688 next:
10689 add_command = strchr(add_command, ',');
10690 if (add_command) {
10691 *add_command = '\0';
10692 add_command++;
10693 }
10694 }
10695
10696 if (!name) {
10697 printf("%s: missing %s\n", __func__, "name");
10698 exit(1);
10699 }
10700
10701 if (strlen(name) >= PMT_COUNTER_NAME_SIZE_BYTES) {
10702 printf("%s: name has to be at most %d characters long\n", __func__, PMT_COUNTER_NAME_SIZE_BYTES);
10703 exit(1);
10704 }
10705
10706 if (format_name) {
10707 has_format = false;
10708
10709 if (strcmp("raw", format_name) == 0) {
10710 format = FORMAT_RAW;
10711 has_format = true;
10712 }
10713
10714 if (strcmp("average", format_name) == 0) {
10715 format = FORMAT_AVERAGE;
10716 has_format = true;
10717 }
10718
10719 if (strcmp("delta", format_name) == 0) {
10720 format = FORMAT_DELTA;
10721 has_format = true;
10722 }
10723
10724 if (!has_format) {
10725 fprintf(stderr, "%s: Invalid format %s. Expected raw, average or delta\n", __func__, format_name);
10726 exit(1);
10727 }
10728 }
10729
10730 if (type_name) {
10731 has_type = false;
10732
10733 if (strcmp("raw", type_name) == 0) {
10734 type = PMT_TYPE_RAW;
10735 has_type = true;
10736 }
10737
10738 if (strcmp("txtal_time", type_name) == 0) {
10739 type = PMT_TYPE_XTAL_TIME;
10740 has_type = true;
10741 }
10742
10743 if (strcmp("tcore_clock", type_name) == 0) {
10744 type = PMT_TYPE_TCORE_CLOCK;
10745 has_type = true;
10746 }
10747
10748 if (!has_type) {
10749 printf("%s: invalid %s: %s\n", __func__, "type", type_name);
10750 exit(1);
10751 }
10752 }
10753
10754 if (!has_offset) {
10755 printf("%s : missing %s\n", __func__, "offset");
10756 exit(1);
10757 }
10758
10759 if (!has_lsb) {
10760 printf("%s: missing %s\n", __func__, "lsb");
10761 exit(1);
10762 }
10763
10764 if (!has_msb) {
10765 printf("%s: missing %s\n", __func__, "msb");
10766 exit(1);
10767 }
10768
10769 if (direct_path && has_guid) {
10770 printf("%s: path and guid+seq parameters are mutually exclusive\n"
10771 "notice: passed guid=0x%x and path=%s\n", __func__, guid, direct_path);
10772 exit(1);
10773 }
10774
10775 if (direct_path) {
10776 if (pmt_parse_from_path(direct_path, &guid, &seq)) {
10777 printf("%s: failed to parse PMT file from %s\n", __func__, direct_path);
10778 exit(1);
10779 }
10780
10781 /* GUID was just infered from the direct path. */
10782 has_guid = true;
10783 }
10784
10785 if (!has_guid) {
10786 printf("%s: missing %s\n", __func__, "guid or path");
10787 exit(1);
10788 }
10789
10790 if (!has_scope) {
10791 printf("%s: missing %s\n", __func__, "scope");
10792 exit(1);
10793 }
10794
10795 if (lsb > msb) {
10796 printf("%s: lsb > msb doesn't make sense\n", __func__);
10797 exit(1);
10798 }
10799
10800 pmt_add_counter(guid, seq, name, type, lsb, msb, offset, scope, format, domain_id, PMT_OPEN_REQUIRED);
10801 }
10802
parse_add_command(char * add_command)10803 void parse_add_command(char *add_command)
10804 {
10805 if (strncmp(add_command, "pmt", strlen("pmt")) == 0)
10806 return parse_add_command_pmt(add_command);
10807 return parse_add_command_msr(add_command);
10808 }
10809
is_deferred_add(char * name)10810 int is_deferred_add(char *name)
10811 {
10812 int i;
10813
10814 for (i = 0; i < deferred_add_index; ++i)
10815 if (!strcmp(name, deferred_add_names[i])) {
10816 deferred_add_consumed |= (1 << i);
10817 return 1;
10818 }
10819 return 0;
10820 }
10821
is_deferred_skip(char * name)10822 int is_deferred_skip(char *name)
10823 {
10824 int i;
10825
10826 for (i = 0; i < deferred_skip_index; ++i)
10827 if (!strcmp(name, deferred_skip_names[i])) {
10828 deferred_skip_consumed |= (1 << i);
10829 return 1;
10830 }
10831 return 0;
10832 }
10833
verify_deferred_consumed(void)10834 void verify_deferred_consumed(void)
10835 {
10836 int i;
10837 int fail = 0;
10838
10839 for (i = 0; i < deferred_add_index; ++i) {
10840 if (!(deferred_add_consumed & (1 << i))) {
10841 warnx("Counter '%s' can not be added.", deferred_add_names[i]);
10842 fail++;
10843 }
10844 }
10845 for (i = 0; i < deferred_skip_index; ++i) {
10846 if (!(deferred_skip_consumed & (1 << i))) {
10847 warnx("Counter '%s' can not be skipped.", deferred_skip_names[i]);
10848 fail++;
10849 }
10850 }
10851 if (fail)
10852 exit(-EINVAL);
10853 }
10854
probe_cpuidle_residency(void)10855 void probe_cpuidle_residency(void)
10856 {
10857 char path[64];
10858 char name_buf[16];
10859 FILE *input;
10860 int state;
10861 int min_state = 1024, max_state = 0;
10862 char *sp;
10863
10864 for (state = 10; state >= 0; --state) {
10865
10866 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state);
10867 input = fopen(path, "r");
10868 if (input == NULL)
10869 continue;
10870 if (!fgets(name_buf, sizeof(name_buf), input))
10871 err(1, "%s: failed to read file", path);
10872
10873 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
10874 sp = strchr(name_buf, '-');
10875 if (!sp)
10876 sp = strchrnul(name_buf, '\n');
10877 *sp = '%';
10878 *(sp + 1) = '\0';
10879
10880 remove_underbar(name_buf);
10881
10882 fclose(input);
10883
10884 sprintf(path, "cpuidle/state%d/time", state);
10885
10886 if (!DO_BIC(BIC_pct_idle) && !is_deferred_add(name_buf))
10887 continue;
10888
10889 if (is_deferred_skip(name_buf))
10890 continue;
10891
10892 add_counter(0, path, name_buf, 32, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0);
10893
10894 if (state > max_state)
10895 max_state = state;
10896 if (state < min_state)
10897 min_state = state;
10898 }
10899 }
10900
probe_cpuidle_counts(void)10901 void probe_cpuidle_counts(void)
10902 {
10903 char path[64];
10904 char name_buf[16];
10905 FILE *input;
10906 int state;
10907 int min_state = 1024, max_state = 0;
10908 char *sp;
10909
10910 if (!DO_BIC(BIC_cpuidle))
10911 return;
10912
10913 for (state = 10; state >= 0; --state) {
10914
10915 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", base_cpu, state);
10916 input = fopen(path, "r");
10917 if (input == NULL)
10918 continue;
10919 if (!fgets(name_buf, sizeof(name_buf), input))
10920 err(1, "%s: failed to read file", path);
10921 fclose(input);
10922
10923 remove_underbar(name_buf);
10924
10925 if (!DO_BIC(BIC_cpuidle) && !is_deferred_add(name_buf))
10926 continue;
10927
10928 if (is_deferred_skip(name_buf))
10929 continue;
10930
10931 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
10932 sp = strchr(name_buf, '-');
10933 if (!sp)
10934 sp = strchrnul(name_buf, '\n');
10935
10936 /*
10937 * The 'below' sysfs file always contains 0 for the deepest state (largest index),
10938 * do not add it.
10939 */
10940 if (state != max_state) {
10941 /*
10942 * Add 'C1+' for C1, and so on. The 'below' sysfs file always contains 0 for
10943 * the last state, so do not add it.
10944 */
10945
10946 *sp = '+';
10947 *(sp + 1) = '\0';
10948 sprintf(path, "cpuidle/state%d/below", state);
10949 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
10950 }
10951
10952 *sp = '\0';
10953 sprintf(path, "cpuidle/state%d/usage", state);
10954 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
10955
10956 /*
10957 * The 'above' sysfs file always contains 0 for the shallowest state (smallest
10958 * index), do not add it.
10959 */
10960 if (state != min_state) {
10961 *sp = '-';
10962 *(sp + 1) = '\0';
10963 sprintf(path, "cpuidle/state%d/above", state);
10964 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
10965 }
10966 }
10967 }
10968
10969 /*
10970 * parse cpuset with following syntax
10971 * 1,2,4..6,8-10 and set bits in cpu_subset
10972 */
parse_cpu_command(char * optarg)10973 void parse_cpu_command(char *optarg)
10974 {
10975 if (!strcmp(optarg, "core")) {
10976 if (cpu_subset)
10977 goto error;
10978 show_core_only++;
10979 return;
10980 }
10981 if (!strcmp(optarg, "package")) {
10982 if (cpu_subset)
10983 goto error;
10984 show_pkg_only++;
10985 return;
10986 }
10987 if (show_core_only || show_pkg_only)
10988 goto error;
10989
10990 cpu_subset = CPU_ALLOC(CPU_SUBSET_MAXCPUS);
10991 if (cpu_subset == NULL)
10992 err(3, "CPU_ALLOC");
10993 cpu_subset_size = CPU_ALLOC_SIZE(CPU_SUBSET_MAXCPUS);
10994
10995 CPU_ZERO_S(cpu_subset_size, cpu_subset);
10996
10997 if (parse_cpu_str(optarg, cpu_subset, cpu_subset_size))
10998 goto error;
10999
11000 return;
11001
11002 error:
11003 fprintf(stderr, "\"--cpu %s\" malformed\n", optarg);
11004 help();
11005 exit(-1);
11006 }
11007
cmdline(int argc,char ** argv)11008 void cmdline(int argc, char **argv)
11009 {
11010 int opt;
11011 int option_index = 0;
11012 static struct option long_options[] = {
11013 { "add", required_argument, 0, 'a' },
11014 { "cpu", required_argument, 0, 'c' },
11015 { "Dump", no_argument, 0, 'D' },
11016 { "debug", no_argument, 0, 'd' }, /* internal, not documented */
11017 { "enable", required_argument, 0, 'e' },
11018 { "force", no_argument, 0, 'f' },
11019 { "interval", required_argument, 0, 'i' },
11020 { "IPC", no_argument, 0, 'I' },
11021 { "num_iterations", required_argument, 0, 'n' },
11022 { "header_iterations", required_argument, 0, 'N' },
11023 { "help", no_argument, 0, 'h' },
11024 { "hide", required_argument, 0, 'H' }, // meh, -h taken by --help
11025 { "Joules", no_argument, 0, 'J' },
11026 { "list", no_argument, 0, 'l' },
11027 { "out", required_argument, 0, 'o' },
11028 { "quiet", no_argument, 0, 'q' },
11029 { "no-msr", no_argument, 0, 'M' },
11030 { "no-perf", no_argument, 0, 'P' },
11031 { "show", required_argument, 0, 's' },
11032 { "Summary", no_argument, 0, 'S' },
11033 { "TCC", required_argument, 0, 'T' },
11034 { "version", no_argument, 0, 'v' },
11035 { 0, 0, 0, 0 }
11036 };
11037
11038 progname = argv[0];
11039
11040 /*
11041 * Parse some options early, because they may make other options invalid,
11042 * like adding the MSR counter with --add and at the same time using --no-msr.
11043 */
11044 while ((opt = getopt_long_only(argc, argv, "+MPn:", long_options, &option_index)) != -1) {
11045 switch (opt) {
11046 case 'M':
11047 no_msr = 1;
11048 break;
11049 case 'P':
11050 no_perf = 1;
11051 break;
11052 default:
11053 break;
11054 }
11055 }
11056 optind = 0;
11057
11058 while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:o:qMST:v", long_options, &option_index)) != -1) {
11059 switch (opt) {
11060 case 'a':
11061 parse_add_command(optarg);
11062 break;
11063 case 'c':
11064 parse_cpu_command(optarg);
11065 break;
11066 case 'D':
11067 dump_only++;
11068 /*
11069 * Force the no_perf early to prevent using it as a source.
11070 * User asks for raw values, but perf returns them relative
11071 * to the opening of the file descriptor.
11072 */
11073 no_perf = 1;
11074 break;
11075 case 'e':
11076 /* --enable specified counter, without clearning existing list */
11077 bic_lookup(&bic_enabled, optarg, SHOW_LIST);
11078 break;
11079 case 'f':
11080 force_load++;
11081 break;
11082 case 'd':
11083 debug++;
11084 bic_set_all(&bic_enabled);
11085 break;
11086 case 'H':
11087 /*
11088 * --hide: do not show those specified
11089 * multiple invocations simply clear more bits in enabled mask
11090 */
11091 {
11092 cpu_set_t bic_group_hide;
11093
11094 BIC_INIT(&bic_group_hide);
11095
11096 bic_lookup(&bic_group_hide, optarg, HIDE_LIST);
11097 bic_clear_bits(&bic_enabled, &bic_group_hide);
11098 }
11099 break;
11100 case 'h':
11101 default:
11102 help();
11103 exit(1);
11104 case 'i':
11105 {
11106 double interval = strtod(optarg, NULL);
11107
11108 if (interval < 0.001) {
11109 fprintf(outf, "interval %f seconds is too small\n", interval);
11110 exit(2);
11111 }
11112
11113 interval_tv.tv_sec = interval_ts.tv_sec = interval;
11114 interval_tv.tv_usec = (interval - interval_tv.tv_sec) * 1000000;
11115 interval_ts.tv_nsec = (interval - interval_ts.tv_sec) * 1000000000;
11116 }
11117 break;
11118 case 'J':
11119 rapl_joules++;
11120 break;
11121 case 'l':
11122 bic_set_all(&bic_enabled);
11123 list_header_only++;
11124 quiet++;
11125 break;
11126 case 'o':
11127 outf = fopen_or_die(optarg, "w");
11128 break;
11129 case 'q':
11130 quiet = 1;
11131 break;
11132 case 'M':
11133 case 'P':
11134 /* Parsed earlier */
11135 break;
11136 case 'n':
11137 num_iterations = strtod(optarg, NULL);
11138
11139 if (num_iterations <= 0) {
11140 fprintf(outf, "iterations %d should be positive number\n", num_iterations);
11141 exit(2);
11142 }
11143 break;
11144 case 'N':
11145 header_iterations = strtod(optarg, NULL);
11146
11147 if (header_iterations <= 0) {
11148 fprintf(outf, "iterations %d should be positive number\n", header_iterations);
11149 exit(2);
11150 }
11151 break;
11152 case 's':
11153 /*
11154 * --show: show only those specified
11155 * The 1st invocation will clear and replace the enabled mask
11156 * subsequent invocations can add to it.
11157 */
11158 if (shown == 0)
11159 BIC_INIT(&bic_enabled);
11160 bic_lookup(&bic_enabled, optarg, SHOW_LIST);
11161 shown = 1;
11162 break;
11163 case 'S':
11164 summary_only++;
11165 break;
11166 case 'T':
11167 tj_max_override = atoi(optarg);
11168 break;
11169 case 'v':
11170 print_version();
11171 exit(0);
11172 break;
11173 }
11174 }
11175 }
11176
set_rlimit(void)11177 void set_rlimit(void)
11178 {
11179 struct rlimit limit;
11180
11181 if (getrlimit(RLIMIT_NOFILE, &limit) < 0)
11182 err(1, "Failed to get rlimit");
11183
11184 if (limit.rlim_max < MAX_NOFILE)
11185 limit.rlim_max = MAX_NOFILE;
11186 if (limit.rlim_cur < MAX_NOFILE)
11187 limit.rlim_cur = MAX_NOFILE;
11188
11189 if (setrlimit(RLIMIT_NOFILE, &limit) < 0)
11190 err(1, "Failed to set rlimit");
11191 }
11192
main(int argc,char ** argv)11193 int main(int argc, char **argv)
11194 {
11195 int fd, ret;
11196
11197 bic_groups_init();
11198
11199 fd = open("/sys/fs/cgroup/cgroup.procs", O_WRONLY);
11200 if (fd < 0)
11201 goto skip_cgroup_setting;
11202
11203 ret = write(fd, "0\n", 2);
11204 if (ret == -1)
11205 perror("Can't update cgroup\n");
11206
11207 close(fd);
11208
11209 skip_cgroup_setting:
11210 outf = stderr;
11211 cmdline(argc, argv);
11212
11213 if (!quiet) {
11214 print_version();
11215 print_bootcmd();
11216 }
11217
11218 probe_cpuidle_residency();
11219 probe_cpuidle_counts();
11220
11221 verify_deferred_consumed();
11222
11223 if (!getuid())
11224 set_rlimit();
11225
11226 turbostat_init();
11227
11228 if (!no_msr)
11229 msr_sum_record();
11230
11231 /* dump counters and exit */
11232 if (dump_only)
11233 return get_and_dump_counters();
11234
11235 /* list header and exit */
11236 if (list_header_only) {
11237 print_header(",");
11238 flush_output_stdout();
11239 return 0;
11240 }
11241
11242 /*
11243 * if any params left, it must be a command to fork
11244 */
11245 if (argc - optind)
11246 return fork_it(argv + optind);
11247 else
11248 turbostat_loop();
11249
11250 return 0;
11251 }
11252