1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 3 #ifndef __CPUSET_INTERNAL_H 4 #define __CPUSET_INTERNAL_H 5 6 #include <linux/cgroup.h> 7 #include <linux/cpu.h> 8 #include <linux/cpumask.h> 9 #include <linux/cpuset.h> 10 #include <linux/spinlock.h> 11 #include <linux/union_find.h> 12 #include <linux/sched/isolation.h> 13 14 /* See "Frequency meter" comments, below. */ 15 16 struct fmeter { 17 int cnt; /* unprocessed events count */ 18 int val; /* most recent output value */ 19 time64_t time; /* clock (secs) when val computed */ 20 spinlock_t lock; /* guards read or write of above */ 21 }; 22 23 /* 24 * Invalid partition error code 25 */ 26 enum prs_errcode { 27 PERR_NONE = 0, 28 PERR_INVCPUS, 29 PERR_INVPARENT, 30 PERR_NOTPART, 31 PERR_NOTEXCL, 32 PERR_NOCPUS, 33 PERR_HOTPLUG, 34 PERR_CPUSEMPTY, 35 PERR_HKEEPING, 36 PERR_ACCESS, 37 PERR_REMOTE, 38 PERR_ISOPARENT, 39 }; 40 41 /* bits in struct cpuset flags field */ 42 typedef enum { 43 CS_CPU_EXCLUSIVE, 44 CS_MEM_EXCLUSIVE, 45 CS_MEM_HARDWALL, 46 CS_MEMORY_MIGRATE, 47 CS_SCHED_LOAD_BALANCE, 48 CS_SPREAD_PAGE, 49 CS_SPREAD_SLAB, 50 } cpuset_flagbits_t; 51 52 /* The various types of files and directories in a cpuset file system */ 53 54 typedef enum { 55 FILE_MEMORY_MIGRATE, 56 FILE_CPULIST, 57 FILE_MEMLIST, 58 FILE_EFFECTIVE_CPULIST, 59 FILE_EFFECTIVE_MEMLIST, 60 FILE_SUBPARTS_CPULIST, 61 FILE_EXCLUSIVE_CPULIST, 62 FILE_EFFECTIVE_XCPULIST, 63 FILE_ISOLATED_CPULIST, 64 FILE_CPU_EXCLUSIVE, 65 FILE_MEM_EXCLUSIVE, 66 FILE_MEM_HARDWALL, 67 FILE_SCHED_LOAD_BALANCE, 68 FILE_PARTITION_ROOT, 69 FILE_SCHED_RELAX_DOMAIN_LEVEL, 70 FILE_MEMORY_PRESSURE_ENABLED, 71 FILE_MEMORY_PRESSURE, 72 FILE_SPREAD_PAGE, 73 FILE_SPREAD_SLAB, 74 } cpuset_filetype_t; 75 76 struct cpuset { 77 struct cgroup_subsys_state css; 78 79 unsigned long flags; /* "unsigned long" so bitops work */ 80 81 /* 82 * On default hierarchy: 83 * 84 * The user-configured masks can only be changed by writing to 85 * cpuset.cpus and cpuset.mems, and won't be limited by the 86 * parent masks. 87 * 88 * The effective masks is the real masks that apply to the tasks 89 * in the cpuset. They may be changed if the configured masks are 90 * changed or hotplug happens. 91 * 92 * effective_mask == configured_mask & parent's effective_mask, 93 * and if it ends up empty, it will inherit the parent's mask. 94 * 95 * 96 * On legacy hierarchy: 97 * 98 * The user-configured masks are always the same with effective masks. 99 */ 100 101 /* user-configured CPUs and Memory Nodes allow to tasks */ 102 cpumask_var_t cpus_allowed; 103 nodemask_t mems_allowed; 104 105 /* effective CPUs and Memory Nodes allow to tasks */ 106 cpumask_var_t effective_cpus; 107 nodemask_t effective_mems; 108 109 /* 110 * Exclusive CPUs dedicated to current cgroup (default hierarchy only) 111 * 112 * The effective_cpus of a valid partition root comes solely from its 113 * effective_xcpus and some of the effective_xcpus may be distributed 114 * to sub-partitions below & hence excluded from its effective_cpus. 115 * For a valid partition root, its effective_cpus have no relationship 116 * with cpus_allowed unless its exclusive_cpus isn't set. 117 * 118 * This value will only be set if either exclusive_cpus is set or 119 * when this cpuset becomes a local partition root. 120 */ 121 cpumask_var_t effective_xcpus; 122 123 /* 124 * Exclusive CPUs as requested by the user (default hierarchy only) 125 * 126 * Its value is independent of cpus_allowed and designates the set of 127 * CPUs that can be granted to the current cpuset or its children when 128 * it becomes a valid partition root. The effective set of exclusive 129 * CPUs granted (effective_xcpus) depends on whether those exclusive 130 * CPUs are passed down by its ancestors and not yet taken up by 131 * another sibling partition root along the way. 132 * 133 * If its value isn't set, it defaults to cpus_allowed. 134 */ 135 cpumask_var_t exclusive_cpus; 136 137 /* 138 * This is old Memory Nodes tasks took on. 139 * 140 * - top_cpuset.old_mems_allowed is initialized to mems_allowed. 141 * - A new cpuset's old_mems_allowed is initialized when some 142 * task is moved into it. 143 * - old_mems_allowed is used in cpuset_migrate_mm() when we change 144 * cpuset.mems_allowed and have tasks' nodemask updated, and 145 * then old_mems_allowed is updated to mems_allowed. 146 */ 147 nodemask_t old_mems_allowed; 148 149 /* 150 * For linking impacted cpusets during an attach operation. 151 */ 152 struct llist_node attach_node; 153 154 /* partition root state */ 155 int partition_root_state; 156 157 /* 158 * Whether cpuset is a remote partition. 159 * It used to be a list anchoring all remote partitions — we can switch back 160 * to a list if we need to iterate over the remote partitions. 161 */ 162 bool remote_partition; 163 164 /* 165 * Set to true when a remote partition is undergoing update. 166 */ 167 bool remote_partition_updating; 168 169 /* 170 * number of SCHED_DEADLINE tasks attached to this cpuset, so that we 171 * know when to rebuild associated root domain bandwidth information. 172 */ 173 atomic_t nr_deadline_tasks; 174 int nr_migrate_dl_tasks; 175 /* DL bandwidth that needs destination reservation for this attach. */ 176 u64 sum_migrate_dl_bw; 177 /* 178 * CPU used for temporary DL bandwidth allocation during attach; 179 * -1 if no DL bandwidth was allocated in the current attach. 180 */ 181 int dl_bw_cpu; 182 183 /* Invalid partition error code, not lock protected */ 184 enum prs_errcode prs_err; 185 186 /* Handle for cpuset.cpus.partition */ 187 struct cgroup_file partition_file; 188 189 #ifdef CONFIG_CPUSETS_V1 190 struct fmeter fmeter; /* memory_pressure filter */ 191 192 /* for custom sched domain */ 193 int relax_domain_level; 194 195 /* Used to merge intersecting subsets for generate_sched_domains */ 196 struct uf_node node; 197 #endif 198 }; 199 200 extern struct cpuset top_cpuset; 201 202 static inline struct cpuset *css_cs(struct cgroup_subsys_state *css) 203 { 204 return css ? container_of(css, struct cpuset, css) : NULL; 205 } 206 207 /* Retrieve the cpuset for a task */ 208 static inline struct cpuset *task_cs(struct task_struct *task) 209 { 210 return css_cs(task_css(task, cpuset_cgrp_id)); 211 } 212 213 static inline struct cpuset *parent_cs(struct cpuset *cs) 214 { 215 return css_cs(cs->css.parent); 216 } 217 218 /* convenient tests for these bits */ 219 static inline bool is_cpuset_online(struct cpuset *cs) 220 { 221 return css_is_online(&cs->css) && !css_is_dying(&cs->css); 222 } 223 224 static inline int is_cpu_exclusive(const struct cpuset *cs) 225 { 226 return test_bit(CS_CPU_EXCLUSIVE, &cs->flags); 227 } 228 229 static inline int is_mem_exclusive(const struct cpuset *cs) 230 { 231 return test_bit(CS_MEM_EXCLUSIVE, &cs->flags); 232 } 233 234 static inline int is_mem_hardwall(const struct cpuset *cs) 235 { 236 return test_bit(CS_MEM_HARDWALL, &cs->flags); 237 } 238 239 static inline int is_sched_load_balance(const struct cpuset *cs) 240 { 241 return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); 242 } 243 244 static inline int is_memory_migrate(const struct cpuset *cs) 245 { 246 return test_bit(CS_MEMORY_MIGRATE, &cs->flags); 247 } 248 249 static inline int is_spread_page(const struct cpuset *cs) 250 { 251 return test_bit(CS_SPREAD_PAGE, &cs->flags); 252 } 253 254 static inline int is_spread_slab(const struct cpuset *cs) 255 { 256 return test_bit(CS_SPREAD_SLAB, &cs->flags); 257 } 258 259 /* 260 * Helper routine for generate_sched_domains(). 261 * Do cpusets a, b have overlapping effective cpus_allowed masks? 262 */ 263 static inline int cpusets_overlap(struct cpuset *a, struct cpuset *b) 264 { 265 return cpumask_intersects(a->effective_cpus, b->effective_cpus); 266 } 267 268 static inline int nr_cpusets(void) 269 { 270 /* jump label reference count + the top-level cpuset */ 271 return static_key_count(&cpusets_enabled_key.key) + 1; 272 } 273 274 static inline bool cpuset_is_populated(struct cpuset *cs) 275 { 276 lockdep_assert_cpuset_lock_held(); 277 return cgroup_is_populated(cs->css.cgroup); 278 } 279 280 /** 281 * cpuset_for_each_child - traverse online children of a cpuset 282 * @child_cs: loop cursor pointing to the current child 283 * @pos_css: used for iteration 284 * @parent_cs: target cpuset to walk children of 285 * 286 * Walk @child_cs through the online children of @parent_cs. Must be used 287 * with RCU read locked. 288 */ 289 #define cpuset_for_each_child(child_cs, pos_css, parent_cs) \ 290 css_for_each_child((pos_css), &(parent_cs)->css) \ 291 if (is_cpuset_online(((child_cs) = css_cs((pos_css))))) 292 293 /** 294 * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants 295 * @des_cs: loop cursor pointing to the current descendant 296 * @pos_css: used for iteration 297 * @root_cs: target cpuset to walk ancestor of 298 * 299 * Walk @des_cs through the online descendants of @root_cs. Must be used 300 * with RCU read locked. The caller may modify @pos_css by calling 301 * css_rightmost_descendant() to skip subtree. @root_cs is included in the 302 * iteration and the first node to be visited. 303 */ 304 #define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs) \ 305 css_for_each_descendant_pre((pos_css), &(root_cs)->css) \ 306 if (is_cpuset_online(((des_cs) = css_cs((pos_css))))) 307 308 void rebuild_sched_domains_locked(void); 309 void cpuset_callback_lock_irq(void); 310 void cpuset_callback_unlock_irq(void); 311 void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus); 312 void cpuset_update_tasks_nodemask(struct cpuset *cs); 313 struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs); 314 void free_cpuset(struct cpuset *cs); 315 int validate_change(struct cpuset *cur, struct cpuset *trial); 316 ssize_t cpuset_write_resmask(struct kernfs_open_file *of, 317 char *buf, size_t nbytes, loff_t off); 318 int cpuset_common_seq_show(struct seq_file *sf, void *v); 319 void cpuset_full_lock(void); 320 void cpuset_full_unlock(void); 321 322 /* 323 * cpuset-v1.c 324 */ 325 #ifdef CONFIG_CPUSETS_V1 326 extern struct cftype cpuset1_files[]; 327 void cpuset1_update_task_spread_flags(struct cpuset *cs, 328 struct task_struct *tsk); 329 void cpuset1_hotplug_update_tasks(struct cpuset *cs, 330 struct cpumask *new_cpus, nodemask_t *new_mems, 331 bool cpus_updated, bool mems_updated); 332 int cpuset1_validate_change(struct cpuset *cur, struct cpuset *trial); 333 bool cpuset1_cpus_excl_conflict(struct cpuset *cs1, struct cpuset *cs2); 334 void cpuset1_init(struct cpuset *cs); 335 void cpuset1_online_css(struct cgroup_subsys_state *css); 336 int cpuset1_generate_sched_domains(cpumask_var_t **domains, 337 struct sched_domain_attr **attributes); 338 void cpuset1_offline_css(struct cpuset *cs); 339 340 #else 341 static inline void cpuset1_update_task_spread_flags(struct cpuset *cs, 342 struct task_struct *tsk) {} 343 static inline void cpuset1_hotplug_update_tasks(struct cpuset *cs, 344 struct cpumask *new_cpus, nodemask_t *new_mems, 345 bool cpus_updated, bool mems_updated) {} 346 static inline int cpuset1_validate_change(struct cpuset *cur, 347 struct cpuset *trial) { return 0; } 348 static inline bool cpuset1_cpus_excl_conflict(struct cpuset *cs1, 349 struct cpuset *cs2) { return false; } 350 static inline void cpuset1_init(struct cpuset *cs) {} 351 static inline void cpuset1_online_css(struct cgroup_subsys_state *css) {} 352 static inline int cpuset1_generate_sched_domains(cpumask_var_t **domains, 353 struct sched_domain_attr **attributes) { return 0; }; 354 static inline void cpuset1_offline_css(struct cpuset *cs) {} 355 356 #endif /* CONFIG_CPUSETS_V1 */ 357 358 #endif /* __CPUSET_INTERNAL_H */ 359