1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * itmt.c: Support Intel Turbo Boost Max Technology 3.0 4 * 5 * (C) Copyright 2016 Intel Corporation 6 * Author: Tim Chen <tim.c.chen@linux.intel.com> 7 * 8 * On platforms supporting Intel Turbo Boost Max Technology 3.0, (ITMT), 9 * the maximum turbo frequencies of some cores in a CPU package may be 10 * higher than for the other cores in the same package. In that case, 11 * better performance can be achieved by making the scheduler prefer 12 * to run tasks on the CPUs with higher max turbo frequencies. 13 * 14 * This file provides functions and data structures for enabling the 15 * scheduler to favor scheduling on cores can be boosted to a higher 16 * frequency under ITMT. 17 */ 18 19 #include <linux/sched.h> 20 #include <linux/cpumask.h> 21 #include <linux/cpuset.h> 22 #include <linux/debugfs.h> 23 #include <linux/mutex.h> 24 #include <linux/sysctl.h> 25 #include <linux/nodemask.h> 26 27 static DEFINE_MUTEX(itmt_update_mutex); 28 DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority); 29 30 /* Boolean to track if system has ITMT capabilities */ 31 static bool __read_mostly sched_itmt_capable; 32 33 /* 34 * Boolean to control whether we want to move processes to cpu capable 35 * of higher turbo frequency for cpus supporting Intel Turbo Boost Max 36 * Technology 3.0. 37 * 38 * It can be set via /sys/kernel/debug/x86/sched_itmt_enabled 39 */ 40 bool __read_mostly sysctl_sched_itmt_enabled; 41 42 static ssize_t sched_itmt_enabled_write(struct file *filp, 43 const char __user *ubuf, 44 size_t cnt, loff_t *ppos) 45 { 46 ssize_t result; 47 bool orig; 48 49 guard(mutex)(&itmt_update_mutex); 50 51 orig = sysctl_sched_itmt_enabled; 52 result = debugfs_write_file_bool(filp, ubuf, cnt, ppos); 53 54 if (sysctl_sched_itmt_enabled != orig) { 55 x86_topology_update = true; 56 rebuild_sched_domains(); 57 } 58 59 return result; 60 } 61 62 static const struct file_operations dfs_sched_itmt_fops = { 63 .read = debugfs_read_file_bool, 64 .write = sched_itmt_enabled_write, 65 .open = simple_open, 66 .llseek = default_llseek, 67 }; 68 69 static struct dentry *dfs_sched_itmt; 70 71 /** 72 * sched_set_itmt_support() - Indicate platform supports ITMT 73 * 74 * This function is used by the OS to indicate to scheduler that the platform 75 * is capable of supporting the ITMT feature. 76 * 77 * The current scheme has the pstate driver detects if the system 78 * is ITMT capable and call sched_set_itmt_support. 79 * 80 * This must be done only after sched_set_itmt_core_prio 81 * has been called to set the cpus' priorities. 82 * It must not be called with cpu hot plug lock 83 * held as we need to acquire the lock to rebuild sched domains 84 * later. 85 * 86 * Return: 0 on success 87 */ 88 int sched_set_itmt_support(void) 89 { 90 guard(mutex)(&itmt_update_mutex); 91 92 if (sched_itmt_capable) 93 return 0; 94 95 dfs_sched_itmt = debugfs_create_file_unsafe("sched_itmt_enabled", 96 0644, 97 arch_debugfs_dir, 98 &sysctl_sched_itmt_enabled, 99 &dfs_sched_itmt_fops); 100 if (IS_ERR_OR_NULL(dfs_sched_itmt)) { 101 dfs_sched_itmt = NULL; 102 return -ENOMEM; 103 } 104 105 sched_itmt_capable = true; 106 107 sysctl_sched_itmt_enabled = 1; 108 109 x86_topology_update = true; 110 rebuild_sched_domains(); 111 112 return 0; 113 } 114 115 /** 116 * sched_clear_itmt_support() - Revoke platform's support of ITMT 117 * 118 * This function is used by the OS to indicate that it has 119 * revoked the platform's support of ITMT feature. 120 * 121 * It must not be called with cpu hot plug lock 122 * held as we need to acquire the lock to rebuild sched domains 123 * later. 124 */ 125 void sched_clear_itmt_support(void) 126 { 127 guard(mutex)(&itmt_update_mutex); 128 129 if (!sched_itmt_capable) 130 return; 131 132 sched_itmt_capable = false; 133 134 debugfs_remove(dfs_sched_itmt); 135 dfs_sched_itmt = NULL; 136 137 if (sysctl_sched_itmt_enabled) { 138 /* disable sched_itmt if we are no longer ITMT capable */ 139 sysctl_sched_itmt_enabled = 0; 140 x86_topology_update = true; 141 rebuild_sched_domains(); 142 } 143 } 144 145 int arch_asym_cpu_priority(int cpu) 146 { 147 return per_cpu(sched_core_priority, cpu); 148 } 149 150 /** 151 * sched_set_itmt_core_prio() - Set CPU priority based on ITMT 152 * @prio: Priority of @cpu 153 * @cpu: The CPU number 154 * 155 * The pstate driver will find out the max boost frequency 156 * and call this function to set a priority proportional 157 * to the max boost frequency. CPUs with higher boost 158 * frequency will receive higher priority. 159 * 160 * No need to rebuild sched domain after updating 161 * the CPU priorities. The sched domains have no 162 * dependency on CPU priorities. 163 */ 164 void sched_set_itmt_core_prio(int prio, int cpu) 165 { 166 per_cpu(sched_core_priority, cpu) = prio; 167 } 168