1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * x86_pkg_temp_thermal driver
4 * Copyright (c) 2013, Intel Corporation.
5 */
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8 #include <linux/module.h>
9 #include <linux/init.h>
10 #include <linux/intel_tcc.h>
11 #include <linux/err.h>
12 #include <linux/param.h>
13 #include <linux/device.h>
14 #include <linux/platform_device.h>
15 #include <linux/cpu.h>
16 #include <linux/smp.h>
17 #include <linux/slab.h>
18 #include <linux/pm.h>
19 #include <linux/thermal.h>
20 #include <linux/debugfs.h>
21
22 #include <asm/cpu_device_id.h>
23 #include <asm/msr.h>
24
25 #include "thermal_interrupt.h"
26
27 /*
28 * Rate control delay: Idea is to introduce denounce effect
29 * This should be long enough to avoid reduce events, when
30 * threshold is set to a temperature, which is constantly
31 * violated, but at the short enough to take any action.
32 * The action can be remove threshold or change it to next
33 * interesting setting. Based on experiments, in around
34 * every 5 seconds under load will give us a significant
35 * temperature change.
36 */
37 #define PKG_TEMP_THERMAL_NOTIFY_DELAY 5000
38 static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
39 module_param(notify_delay_ms, int, 0644);
40 MODULE_PARM_DESC(notify_delay_ms,
41 "User space notification delay in milli seconds.");
42
43 /* Number of trip points in thermal zone. Currently it can't
44 * be more than 2. MSR can allow setting and getting notifications
45 * for only 2 thresholds. This define enforces this, if there
46 * is some wrong values returned by cpuid for number of thresholds.
47 */
48 #define MAX_NUMBER_OF_TRIPS 2
49
50 struct zone_device {
51 int cpu;
52 bool work_scheduled;
53 u32 msr_pkg_therm_low;
54 u32 msr_pkg_therm_high;
55 struct delayed_work work;
56 struct thermal_zone_device *tzone;
57 struct cpumask cpumask;
58 };
59
60 static struct thermal_zone_params pkg_temp_tz_params = {
61 .no_hwmon = true,
62 };
63
64 /* Keep track of how many zone pointers we allocated in init() */
65 static int max_id __read_mostly;
66 /* Array of zone pointers */
67 static struct zone_device **zones;
68 /* Serializes interrupt notification, work and hotplug */
69 static DEFINE_RAW_SPINLOCK(pkg_temp_lock);
70 /* Protects zone operation in the work function against hotplug removal */
71 static DEFINE_MUTEX(thermal_zone_mutex);
72
73 /* The dynamically assigned cpu hotplug state for module_exit() */
74 static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
75
76 /* Debug counters to show using debugfs */
77 static struct dentry *debugfs;
78 static unsigned int pkg_interrupt_cnt;
79 static unsigned int pkg_work_cnt;
80
pkg_temp_debugfs_init(void)81 static void pkg_temp_debugfs_init(void)
82 {
83 debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
84
85 debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
86 &pkg_interrupt_cnt);
87 debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
88 &pkg_work_cnt);
89 }
90
91 /*
92 * Protection:
93 *
94 * - cpu hotplug: Read serialized by cpu hotplug lock
95 * Write must hold pkg_temp_lock
96 *
97 * - Other callsites: Must hold pkg_temp_lock
98 */
pkg_temp_thermal_get_dev(unsigned int cpu)99 static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
100 {
101 int id = topology_logical_die_id(cpu);
102
103 if (id >= 0 && id < max_id)
104 return zones[id];
105 return NULL;
106 }
107
sys_get_curr_temp(struct thermal_zone_device * tzd,int * temp)108 static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
109 {
110 struct zone_device *zonedev = thermal_zone_device_priv(tzd);
111 int val, ret;
112
113 ret = intel_tcc_get_temp(zonedev->cpu, &val, true);
114 if (ret < 0)
115 return ret;
116
117 *temp = val * 1000;
118 pr_debug("sys_get_curr_temp %d\n", *temp);
119 return 0;
120 }
121
122 static int
sys_set_trip_temp(struct thermal_zone_device * tzd,const struct thermal_trip * trip,int temp)123 sys_set_trip_temp(struct thermal_zone_device *tzd,
124 const struct thermal_trip *trip, int temp)
125 {
126 struct zone_device *zonedev = thermal_zone_device_priv(tzd);
127 unsigned int trip_index = THERMAL_TRIP_PRIV_TO_INT(trip->priv);
128 u32 l, h, mask, shift, intr;
129 int tj_max, val, ret;
130
131 if (temp == THERMAL_TEMP_INVALID)
132 temp = 0;
133
134 tj_max = intel_tcc_get_tjmax(zonedev->cpu);
135 if (tj_max < 0)
136 return tj_max;
137 tj_max *= 1000;
138
139 val = (tj_max - temp)/1000;
140
141 if (trip_index >= MAX_NUMBER_OF_TRIPS || val < 0 || val > 0x7f)
142 return -EINVAL;
143
144 ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
145 &l, &h);
146 if (ret < 0)
147 return ret;
148
149 if (trip_index) {
150 mask = THERM_MASK_THRESHOLD1;
151 shift = THERM_SHIFT_THRESHOLD1;
152 intr = THERM_INT_THRESHOLD1_ENABLE;
153 } else {
154 mask = THERM_MASK_THRESHOLD0;
155 shift = THERM_SHIFT_THRESHOLD0;
156 intr = THERM_INT_THRESHOLD0_ENABLE;
157 }
158 l &= ~mask;
159 /*
160 * When users space sets a trip temperature == 0, which is indication
161 * that, it is no longer interested in receiving notifications.
162 */
163 if (!temp) {
164 l &= ~intr;
165 } else {
166 l |= val << shift;
167 l |= intr;
168 }
169
170 return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
171 l, h);
172 }
173
174 /* Thermal zone callback registry */
175 static const struct thermal_zone_device_ops tzone_ops = {
176 .get_temp = sys_get_curr_temp,
177 .set_trip_temp = sys_set_trip_temp,
178 };
179
pkg_thermal_rate_control(void)180 static bool pkg_thermal_rate_control(void)
181 {
182 return true;
183 }
184
185 /* Enable threshold interrupt on local package/cpu */
enable_pkg_thres_interrupt(void)186 static inline void enable_pkg_thres_interrupt(void)
187 {
188 u8 thres_0, thres_1;
189 u32 l, h;
190
191 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
192 /* only enable/disable if it had valid threshold value */
193 thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
194 thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
195 if (thres_0)
196 l |= THERM_INT_THRESHOLD0_ENABLE;
197 if (thres_1)
198 l |= THERM_INT_THRESHOLD1_ENABLE;
199 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
200 }
201
202 /* Disable threshold interrupt on local package/cpu */
disable_pkg_thres_interrupt(void)203 static inline void disable_pkg_thres_interrupt(void)
204 {
205 u32 l, h;
206
207 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
208
209 l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
210 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
211 }
212
pkg_temp_thermal_threshold_work_fn(struct work_struct * work)213 static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
214 {
215 struct thermal_zone_device *tzone = NULL;
216 int cpu = smp_processor_id();
217 struct zone_device *zonedev;
218
219 mutex_lock(&thermal_zone_mutex);
220 raw_spin_lock_irq(&pkg_temp_lock);
221 ++pkg_work_cnt;
222
223 zonedev = pkg_temp_thermal_get_dev(cpu);
224 if (!zonedev) {
225 raw_spin_unlock_irq(&pkg_temp_lock);
226 mutex_unlock(&thermal_zone_mutex);
227 return;
228 }
229 zonedev->work_scheduled = false;
230
231 thermal_clear_package_intr_status(PACKAGE_LEVEL, THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
232 tzone = zonedev->tzone;
233
234 enable_pkg_thres_interrupt();
235 raw_spin_unlock_irq(&pkg_temp_lock);
236
237 /*
238 * If tzone is not NULL, then thermal_zone_mutex will prevent the
239 * concurrent removal in the cpu offline callback.
240 */
241 if (tzone)
242 thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
243
244 mutex_unlock(&thermal_zone_mutex);
245 }
246
pkg_thermal_schedule_work(int cpu,struct delayed_work * work)247 static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
248 {
249 unsigned long ms = msecs_to_jiffies(notify_delay_ms);
250
251 schedule_delayed_work_on(cpu, work, ms);
252 }
253
pkg_thermal_notify(u64 msr_val)254 static int pkg_thermal_notify(u64 msr_val)
255 {
256 int cpu = smp_processor_id();
257 struct zone_device *zonedev;
258 unsigned long flags;
259
260 raw_spin_lock_irqsave(&pkg_temp_lock, flags);
261 ++pkg_interrupt_cnt;
262
263 disable_pkg_thres_interrupt();
264
265 /* Work is per package, so scheduling it once is enough. */
266 zonedev = pkg_temp_thermal_get_dev(cpu);
267 if (zonedev && !zonedev->work_scheduled) {
268 zonedev->work_scheduled = true;
269 pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work);
270 }
271
272 raw_spin_unlock_irqrestore(&pkg_temp_lock, flags);
273 return 0;
274 }
275
pkg_temp_thermal_trips_init(int cpu,int tj_max,struct thermal_trip * trips,int num_trips)276 static int pkg_temp_thermal_trips_init(int cpu, int tj_max,
277 struct thermal_trip *trips, int num_trips)
278 {
279 unsigned long thres_reg_value;
280 u32 mask, shift, eax, edx;
281 int ret, i;
282
283 for (i = 0; i < num_trips; i++) {
284
285 if (i) {
286 mask = THERM_MASK_THRESHOLD1;
287 shift = THERM_SHIFT_THRESHOLD1;
288 } else {
289 mask = THERM_MASK_THRESHOLD0;
290 shift = THERM_SHIFT_THRESHOLD0;
291 }
292
293 ret = rdmsr_on_cpu(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
294 &eax, &edx);
295 if (ret < 0)
296 return ret;
297
298 thres_reg_value = (eax & mask) >> shift;
299
300 trips[i].temperature = thres_reg_value ?
301 tj_max - thres_reg_value * 1000 : THERMAL_TEMP_INVALID;
302
303 trips[i].type = THERMAL_TRIP_PASSIVE;
304 trips[i].flags |= THERMAL_TRIP_FLAG_RW_TEMP;
305 trips[i].priv = THERMAL_INT_TO_TRIP_PRIV(i);
306
307 pr_debug("%s: cpu=%d, trip=%d, temp=%d\n",
308 __func__, cpu, i, trips[i].temperature);
309 }
310
311 return 0;
312 }
313
pkg_temp_thermal_device_add(unsigned int cpu)314 static int pkg_temp_thermal_device_add(unsigned int cpu)
315 {
316 struct thermal_trip trips[MAX_NUMBER_OF_TRIPS] = { 0 };
317 int id = topology_logical_die_id(cpu);
318 u32 eax, ebx, ecx, edx;
319 struct zone_device *zonedev;
320 int thres_count, err;
321 int tj_max;
322
323 if (id >= max_id)
324 return -ENOMEM;
325
326 cpuid(6, &eax, &ebx, &ecx, &edx);
327 thres_count = ebx & 0x07;
328 if (!thres_count)
329 return -ENODEV;
330
331 thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
332
333 tj_max = intel_tcc_get_tjmax(cpu);
334 if (tj_max < 0)
335 return tj_max;
336 tj_max *= 1000;
337
338 zonedev = kzalloc_obj(*zonedev);
339 if (!zonedev)
340 return -ENOMEM;
341
342 err = pkg_temp_thermal_trips_init(cpu, tj_max, trips, thres_count);
343 if (err)
344 goto out_kfree_zonedev;
345
346 INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn);
347 zonedev->cpu = cpu;
348 zonedev->tzone = thermal_zone_device_register_with_trips("x86_pkg_temp",
349 trips, thres_count,
350 zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
351 if (IS_ERR(zonedev->tzone)) {
352 err = PTR_ERR(zonedev->tzone);
353 goto out_kfree_zonedev;
354 }
355 err = thermal_zone_device_enable(zonedev->tzone);
356 if (err)
357 goto out_unregister_tz;
358
359 /* Store MSR value for package thermal interrupt, to restore at exit */
360 rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
361 zonedev->msr_pkg_therm_high);
362
363 cpumask_set_cpu(cpu, &zonedev->cpumask);
364 raw_spin_lock_irq(&pkg_temp_lock);
365 zones[id] = zonedev;
366 raw_spin_unlock_irq(&pkg_temp_lock);
367
368 return 0;
369
370 out_unregister_tz:
371 thermal_zone_device_unregister(zonedev->tzone);
372 out_kfree_zonedev:
373 kfree(zonedev);
374 return err;
375 }
376
pkg_thermal_cpu_offline(unsigned int cpu)377 static int pkg_thermal_cpu_offline(unsigned int cpu)
378 {
379 struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
380 bool lastcpu, was_target;
381 int target;
382
383 if (!zonedev)
384 return 0;
385
386 target = cpumask_any_but(&zonedev->cpumask, cpu);
387 cpumask_clear_cpu(cpu, &zonedev->cpumask);
388 lastcpu = target >= nr_cpu_ids;
389 /*
390 * Remove the sysfs files, if this is the last cpu in the package
391 * before doing further cleanups.
392 */
393 if (lastcpu) {
394 struct thermal_zone_device *tzone = zonedev->tzone;
395
396 /*
397 * We must protect against a work function calling
398 * thermal_zone_update, after/while unregister. We null out
399 * the pointer under the zone mutex, so the worker function
400 * won't try to call.
401 */
402 mutex_lock(&thermal_zone_mutex);
403 zonedev->tzone = NULL;
404 mutex_unlock(&thermal_zone_mutex);
405
406 thermal_zone_device_unregister(tzone);
407 }
408
409 /* Protect against work and interrupts */
410 raw_spin_lock_irq(&pkg_temp_lock);
411
412 /*
413 * Check whether this cpu was the current target and store the new
414 * one. When we drop the lock, then the interrupt notify function
415 * will see the new target.
416 */
417 was_target = zonedev->cpu == cpu;
418 zonedev->cpu = target;
419
420 /*
421 * If this is the last CPU in the package remove the package
422 * reference from the array and restore the interrupt MSR. When we
423 * drop the lock neither the interrupt notify function nor the
424 * worker will see the package anymore.
425 */
426 if (lastcpu) {
427 zones[topology_logical_die_id(cpu)] = NULL;
428 /* After this point nothing touches the MSR anymore. */
429 wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
430 zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
431 }
432
433 /*
434 * Check whether there is work scheduled and whether the work is
435 * targeted at the outgoing CPU.
436 */
437 if (zonedev->work_scheduled && was_target) {
438 /*
439 * To cancel the work we need to drop the lock, otherwise
440 * we might deadlock if the work needs to be flushed.
441 */
442 raw_spin_unlock_irq(&pkg_temp_lock);
443 cancel_delayed_work_sync(&zonedev->work);
444 raw_spin_lock_irq(&pkg_temp_lock);
445 /*
446 * If this is not the last cpu in the package and the work
447 * did not run after we dropped the lock above, then we
448 * need to reschedule the work, otherwise the interrupt
449 * stays disabled forever.
450 */
451 if (!lastcpu && zonedev->work_scheduled)
452 pkg_thermal_schedule_work(target, &zonedev->work);
453 }
454
455 raw_spin_unlock_irq(&pkg_temp_lock);
456
457 /* Final cleanup if this is the last cpu */
458 if (lastcpu)
459 kfree(zonedev);
460
461 return 0;
462 }
463
pkg_thermal_cpu_online(unsigned int cpu)464 static int pkg_thermal_cpu_online(unsigned int cpu)
465 {
466 struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
467 struct cpuinfo_x86 *c = &cpu_data(cpu);
468
469 /* Paranoia check */
470 if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
471 return -ENODEV;
472
473 /* If the package exists, nothing to do */
474 if (zonedev) {
475 cpumask_set_cpu(cpu, &zonedev->cpumask);
476 return 0;
477 }
478 return pkg_temp_thermal_device_add(cpu);
479 }
480
481 static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
482 X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_PTS, NULL),
483 {}
484 };
485 MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
486
pkg_temp_thermal_init(void)487 static int __init pkg_temp_thermal_init(void)
488 {
489 int ret;
490
491 if (!x86_match_cpu(pkg_temp_thermal_ids))
492 return -ENODEV;
493
494 max_id = topology_max_packages() * topology_max_dies_per_package();
495 zones = kzalloc_objs(struct zone_device *, max_id);
496 if (!zones)
497 return -ENOMEM;
498
499 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
500 pkg_thermal_cpu_online, pkg_thermal_cpu_offline);
501 if (ret < 0)
502 goto err;
503
504 /* Store the state for module exit */
505 pkg_thermal_hp_state = ret;
506
507 platform_thermal_package_notify = pkg_thermal_notify;
508 platform_thermal_package_rate_control = pkg_thermal_rate_control;
509
510 /* Don't care if it fails */
511 pkg_temp_debugfs_init();
512 return 0;
513
514 err:
515 kfree(zones);
516 return ret;
517 }
module_init(pkg_temp_thermal_init)518 module_init(pkg_temp_thermal_init)
519
520 static void __exit pkg_temp_thermal_exit(void)
521 {
522 platform_thermal_package_notify = NULL;
523 platform_thermal_package_rate_control = NULL;
524
525 cpuhp_remove_state(pkg_thermal_hp_state);
526 debugfs_remove_recursive(debugfs);
527 kfree(zones);
528 }
529 module_exit(pkg_temp_thermal_exit)
530
531 MODULE_IMPORT_NS("INTEL_TCC");
532 MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
533 MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
534 MODULE_LICENSE("GPL v2");
535