1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Intel Uncore Frequency Setting
4 * Copyright (c) 2022, Intel Corporation.
5 * All rights reserved.
6 *
7 * Provide interface to set MSR 620 at a granularity of per die. On CPU online,
8 * one control CPU is identified per die to read/write limit. This control CPU
9 * is changed, if the CPU state is changed to offline. When the last CPU is
10 * offline in a die then remove the sysfs object for that die.
11 * The majority of actual code is related to sysfs create and read/write
12 * attributes.
13 *
14 * Author: Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>
15 */
16
17 #include <linux/bitfield.h>
18 #include <linux/cpu.h>
19 #include <linux/module.h>
20 #include <linux/slab.h>
21 #include <linux/suspend.h>
22 #include <asm/cpu_device_id.h>
23 #include <asm/intel-family.h>
24 #include <asm/msr.h>
25
26 #include "uncore-frequency-common.h"
27
28 /* Max instances for uncore data, one for each die */
29 static int uncore_max_entries __read_mostly;
30 /* Storage for uncore data for all instances */
31 static struct uncore_data *uncore_instances;
32 /* Stores the CPU mask of the target CPUs to use during uncore read/write */
33 static cpumask_t uncore_cpu_mask;
34 /* CPU online callback register instance */
35 static enum cpuhp_state uncore_hp_state __read_mostly;
36
37 #define MSR_UNCORE_RATIO_LIMIT 0x620
38 #define MSR_UNCORE_PERF_STATUS 0x621
39 #define UNCORE_FREQ_KHZ_MULTIPLIER 100000
40
41 #define UNCORE_MAX_RATIO_MASK GENMASK_ULL(6, 0)
42 #define UNCORE_MIN_RATIO_MASK GENMASK_ULL(14, 8)
43
44 #define UNCORE_CURRENT_RATIO_MASK GENMASK_ULL(6, 0)
45
uncore_read_control_freq(struct uncore_data * data,unsigned int * value,enum uncore_index index)46 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *value,
47 enum uncore_index index)
48 {
49 u64 cap;
50 int ret;
51
52 if (data->control_cpu < 0)
53 return -ENXIO;
54
55 ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
56 if (ret)
57 return ret;
58
59 if (index == UNCORE_INDEX_MAX_FREQ)
60 *value = FIELD_GET(UNCORE_MAX_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER;
61 else
62 *value = FIELD_GET(UNCORE_MIN_RATIO_MASK, cap) * UNCORE_FREQ_KHZ_MULTIPLIER;
63
64 return 0;
65 }
66
uncore_write_control_freq(struct uncore_data * data,unsigned int input,enum uncore_index index)67 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
68 enum uncore_index index)
69 {
70 int ret;
71 u64 cap;
72
73 input /= UNCORE_FREQ_KHZ_MULTIPLIER;
74 if (!input || input > FIELD_MAX(UNCORE_MAX_RATIO_MASK))
75 return -EINVAL;
76
77 if (data->control_cpu < 0)
78 return -ENXIO;
79
80 ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, &cap);
81 if (ret)
82 return ret;
83
84 if (index == UNCORE_INDEX_MAX_FREQ) {
85 cap &= ~UNCORE_MAX_RATIO_MASK;
86 cap |= FIELD_PREP(UNCORE_MAX_RATIO_MASK, input);
87 } else {
88 cap &= ~UNCORE_MIN_RATIO_MASK;
89 cap |= FIELD_PREP(UNCORE_MIN_RATIO_MASK, input);
90 }
91
92 ret = wrmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT, cap);
93 if (ret)
94 return ret;
95
96 data->stored_uncore_data = cap;
97
98 return 0;
99 }
100
uncore_read_freq(struct uncore_data * data,unsigned int * freq)101 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
102 {
103 u64 ratio;
104 int ret;
105
106 if (data->control_cpu < 0)
107 return -ENXIO;
108
109 ret = rdmsrq_on_cpu(data->control_cpu, MSR_UNCORE_PERF_STATUS, &ratio);
110 if (ret)
111 return ret;
112
113 *freq = FIELD_GET(UNCORE_CURRENT_RATIO_MASK, ratio) * UNCORE_FREQ_KHZ_MULTIPLIER;
114
115 return 0;
116 }
117
uncore_read(struct uncore_data * data,unsigned int * value,enum uncore_index index)118 static int uncore_read(struct uncore_data *data, unsigned int *value, enum uncore_index index)
119 {
120 switch (index) {
121 case UNCORE_INDEX_MIN_FREQ:
122 case UNCORE_INDEX_MAX_FREQ:
123 return uncore_read_control_freq(data, value, index);
124
125 case UNCORE_INDEX_CURRENT_FREQ:
126 return uncore_read_freq(data, value);
127
128 default:
129 break;
130 }
131
132 return -EOPNOTSUPP;
133 }
134
135 /* Caller provides protection */
uncore_get_instance(unsigned int cpu)136 static struct uncore_data *uncore_get_instance(unsigned int cpu)
137 {
138 int id = topology_logical_die_id(cpu);
139
140 if (id >= 0 && id < uncore_max_entries)
141 return &uncore_instances[id];
142
143 return NULL;
144 }
145
uncore_event_cpu_online(unsigned int cpu)146 static int uncore_event_cpu_online(unsigned int cpu)
147 {
148 struct uncore_data *data;
149 int target;
150 int ret;
151
152 /* Check if there is an online cpu in the package for uncore MSR */
153 target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu));
154 if (target < nr_cpu_ids)
155 return 0;
156
157 data = uncore_get_instance(cpu);
158 if (!data)
159 return 0;
160
161 data->package_id = topology_physical_package_id(cpu);
162 data->die_id = topology_die_id(cpu);
163 data->domain_id = UNCORE_DOMAIN_ID_INVALID;
164
165 ret = uncore_freq_add_entry(data, cpu);
166 if (ret)
167 return ret;
168
169 /* Use this CPU on this die as a control CPU */
170 cpumask_set_cpu(cpu, &uncore_cpu_mask);
171
172 return 0;
173 }
174
uncore_event_cpu_offline(unsigned int cpu)175 static int uncore_event_cpu_offline(unsigned int cpu)
176 {
177 struct uncore_data *data;
178 int target;
179
180 data = uncore_get_instance(cpu);
181 if (!data)
182 return 0;
183
184 /* Check if existing cpu is used for uncore MSRs */
185 if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask))
186 return 0;
187
188 /* Find a new cpu to set uncore MSR */
189 target = cpumask_any_but(topology_die_cpumask(cpu), cpu);
190
191 if (target < nr_cpu_ids) {
192 cpumask_set_cpu(target, &uncore_cpu_mask);
193 uncore_freq_add_entry(data, target);
194 } else {
195 uncore_freq_remove_die_entry(data);
196 }
197
198 return 0;
199 }
200
uncore_pm_notify(struct notifier_block * nb,unsigned long mode,void * _unused)201 static int uncore_pm_notify(struct notifier_block *nb, unsigned long mode,
202 void *_unused)
203 {
204 int i;
205
206 switch (mode) {
207 case PM_POST_HIBERNATION:
208 case PM_POST_RESTORE:
209 case PM_POST_SUSPEND:
210 for (i = 0; i < uncore_max_entries; ++i) {
211 struct uncore_data *data = &uncore_instances[i];
212
213 if (!data || !data->valid || !data->stored_uncore_data)
214 return 0;
215
216 wrmsrq_on_cpu(data->control_cpu, MSR_UNCORE_RATIO_LIMIT,
217 data->stored_uncore_data);
218 }
219 break;
220 default:
221 break;
222 }
223 return 0;
224 }
225
226 static struct notifier_block uncore_pm_nb = {
227 .notifier_call = uncore_pm_notify,
228 };
229
230 static const struct x86_cpu_id intel_uncore_cpu_ids[] = {
231 X86_MATCH_VFM(INTEL_BROADWELL_G, NULL),
232 X86_MATCH_VFM(INTEL_BROADWELL_X, NULL),
233 X86_MATCH_VFM(INTEL_BROADWELL_D, NULL),
234 X86_MATCH_VFM(INTEL_SKYLAKE_X, NULL),
235 X86_MATCH_VFM(INTEL_ICELAKE_X, NULL),
236 X86_MATCH_VFM(INTEL_ICELAKE_D, NULL),
237 X86_MATCH_VFM(INTEL_SAPPHIRERAPIDS_X, NULL),
238 X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, NULL),
239 X86_MATCH_VFM(INTEL_KABYLAKE, NULL),
240 X86_MATCH_VFM(INTEL_KABYLAKE_L, NULL),
241 X86_MATCH_VFM(INTEL_COMETLAKE, NULL),
242 X86_MATCH_VFM(INTEL_COMETLAKE_L, NULL),
243 X86_MATCH_VFM(INTEL_CANNONLAKE_L, NULL),
244 X86_MATCH_VFM(INTEL_ICELAKE, NULL),
245 X86_MATCH_VFM(INTEL_ICELAKE_L, NULL),
246 X86_MATCH_VFM(INTEL_ROCKETLAKE, NULL),
247 X86_MATCH_VFM(INTEL_TIGERLAKE, NULL),
248 X86_MATCH_VFM(INTEL_TIGERLAKE_L, NULL),
249 X86_MATCH_VFM(INTEL_ALDERLAKE, NULL),
250 X86_MATCH_VFM(INTEL_ALDERLAKE_L, NULL),
251 X86_MATCH_VFM(INTEL_RAPTORLAKE, NULL),
252 X86_MATCH_VFM(INTEL_RAPTORLAKE_P, NULL),
253 X86_MATCH_VFM(INTEL_RAPTORLAKE_S, NULL),
254 X86_MATCH_VFM(INTEL_METEORLAKE, NULL),
255 X86_MATCH_VFM(INTEL_METEORLAKE_L, NULL),
256 X86_MATCH_VFM(INTEL_ARROWLAKE, NULL),
257 X86_MATCH_VFM(INTEL_ARROWLAKE_H, NULL),
258 X86_MATCH_VFM(INTEL_LUNARLAKE_M, NULL),
259 {}
260 };
261 MODULE_DEVICE_TABLE(x86cpu, intel_uncore_cpu_ids);
262
intel_uncore_init(void)263 static int __init intel_uncore_init(void)
264 {
265 const struct x86_cpu_id *id;
266 int ret;
267
268 if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
269 return -ENODEV;
270
271 id = x86_match_cpu(intel_uncore_cpu_ids);
272 if (!id)
273 return -ENODEV;
274
275 uncore_max_entries = topology_max_packages() *
276 topology_max_dies_per_package();
277 uncore_instances = kcalloc(uncore_max_entries,
278 sizeof(*uncore_instances), GFP_KERNEL);
279 if (!uncore_instances)
280 return -ENOMEM;
281
282 ret = uncore_freq_common_init(uncore_read, uncore_write_control_freq);
283 if (ret)
284 goto err_free;
285
286 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
287 "platform/x86/uncore-freq:online",
288 uncore_event_cpu_online,
289 uncore_event_cpu_offline);
290 if (ret < 0)
291 goto err_rem_kobj;
292
293 uncore_hp_state = ret;
294
295 ret = register_pm_notifier(&uncore_pm_nb);
296 if (ret)
297 goto err_rem_state;
298
299 return 0;
300
301 err_rem_state:
302 cpuhp_remove_state(uncore_hp_state);
303 err_rem_kobj:
304 uncore_freq_common_exit();
305 err_free:
306 kfree(uncore_instances);
307
308 return ret;
309 }
module_init(intel_uncore_init)310 module_init(intel_uncore_init)
311
312 static void __exit intel_uncore_exit(void)
313 {
314 int i;
315
316 unregister_pm_notifier(&uncore_pm_nb);
317 cpuhp_remove_state(uncore_hp_state);
318 for (i = 0; i < uncore_max_entries; ++i)
319 uncore_freq_remove_die_entry(&uncore_instances[i]);
320 uncore_freq_common_exit();
321 kfree(uncore_instances);
322 }
323 module_exit(intel_uncore_exit)
324
325 MODULE_IMPORT_NS("INTEL_UNCORE_FREQUENCY");
326 MODULE_LICENSE("GPL v2");
327 MODULE_DESCRIPTION("Intel Uncore Frequency Limits Driver");
328