Lines Matching +full:cpu +full:- +full:capacity
1 // SPDX-License-Identifier: GPL-2.0
41 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level,
44 struct sched_group *group = sd->groups;
45 unsigned long flags = sd->flags;
50 printk(KERN_DEBUG "%*s domain-%d: ", level, "", level);
52 cpumask_pr_args(sched_domain_span(sd)), sd->name);
54 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) {
55 printk(KERN_ERR "ERROR: domain->span does not contain CPU%d\n", cpu);
57 if (group && !cpumask_test_cpu(cpu, sched_group_span(group))) {
58 printk(KERN_ERR "ERROR: domain->groups does not contain CPU%d\n", cpu);
65 if ((meta_flags & SDF_SHARED_CHILD) && sd->child &&
66 !(sd->child->flags & flag))
70 if ((meta_flags & SDF_SHARED_PARENT) && sd->parent &&
71 !(sd->parent->flags & flag))
90 if (!(sd->flags & SD_OVERLAP) &&
100 group->sgc->id,
103 if ((sd->flags & SD_OVERLAP) &&
109 if (group->sgc->capacity != SCHED_CAPACITY_SCALE)
110 printk(KERN_CONT " cap=%lu", group->sgc->capacity);
112 if (group == sd->groups && sd->child &&
113 !cpumask_equal(sched_domain_span(sd->child),
115 printk(KERN_ERR "ERROR: domain->groups does not match domain->child\n");
120 group = group->next;
122 if (group != sd->groups)
125 } while (group != sd->groups);
129 printk(KERN_ERR "ERROR: groups don't span domain->span\n");
131 if (sd->parent &&
132 !cpumask_subset(groupmask, sched_domain_span(sd->parent)))
133 printk(KERN_ERR "ERROR: parent span is not a superset of domain->span\n");
137 static void sched_domain_debug(struct sched_domain *sd, int cpu)
145 printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu);
149 printk(KERN_DEBUG "CPU%d attaching sched-domain(s):\n", cpu);
152 if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask))
155 sd = sd->parent;
174 if ((sd->flags & SD_DEGENERATE_GROUPS_MASK) &&
175 (sd->groups != sd->groups->next))
179 if (sd->flags & (SD_WAKE_AFFINE))
188 unsigned long cflags = sd->flags, pflags = parent->flags;
197 if (parent->groups == parent->groups->next)
219 /* EAS is enabled for asymmetric CPU capacity topologies. */
245 pr_info("rd %*pbl: Checking EAS: frequency-invariant load tracking not yet supported",
256 pr_info("rd %*pbl: Checking EAS, cpufreq policy not set for CPU: %d",
261 gov = policy->governor;
291 return -EPERM;
295 return -EOPNOTSUPP;
338 tmp = pd->next;
344 static struct perf_domain *find_pd(struct perf_domain *pd, int cpu)
347 if (cpumask_test_cpu(cpu, perf_domain_span(pd)))
349 pd = pd->next;
355 static struct perf_domain *pd_init(int cpu)
357 struct em_perf_domain *obj = em_cpu_get(cpu);
362 pr_info("%s: no EM found for CPU%d\n", __func__, cpu);
369 pd->em_pd = obj;
386 em_pd_nr_perf_states(pd->em_pd));
387 pd = pd->next;
426 int cpu = cpumask_first(cpu_map);
427 struct root_domain *rd = cpu_rq(cpu)->rd;
444 tmp->next = pd;
451 tmp = rd->pd;
452 rcu_assign_pointer(rd->pd, pd);
454 call_rcu(&tmp->rcu, destroy_perf_domain_rcu);
460 tmp = rd->pd;
461 rcu_assign_pointer(rd->pd, NULL);
463 call_rcu(&tmp->rcu, destroy_perf_domain_rcu);
475 cpupri_cleanup(&rd->cpupri);
476 cpudl_cleanup(&rd->cpudl);
477 free_cpumask_var(rd->dlo_mask);
478 free_cpumask_var(rd->rto_mask);
479 free_cpumask_var(rd->online);
480 free_cpumask_var(rd->span);
481 free_pd(rd->pd);
492 if (rq->rd) {
493 old_rd = rq->rd;
495 if (cpumask_test_cpu(rq->cpu, old_rd->online))
498 cpumask_clear_cpu(rq->cpu, old_rd->span);
505 if (!atomic_dec_and_test(&old_rd->refcount))
509 atomic_inc(&rd->refcount);
510 rq->rd = rd;
512 cpumask_set_cpu(rq->cpu, rd->span);
513 if (cpumask_test_cpu(rq->cpu, cpu_active_mask))
521 if (rq->fair_server.dl_server)
522 __dl_server_attach_root(&rq->fair_server, rq);
527 call_rcu(&old_rd->rcu, free_rootdomain);
532 atomic_inc(&rd->refcount);
537 if (!atomic_dec_and_test(&rd->refcount))
540 call_rcu(&rd->rcu, free_rootdomain);
545 if (!zalloc_cpumask_var(&rd->span, GFP_KERNEL))
547 if (!zalloc_cpumask_var(&rd->online, GFP_KERNEL))
549 if (!zalloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL))
551 if (!zalloc_cpumask_var(&rd->rto_mask, GFP_KERNEL))
555 rd->rto_cpu = -1;
556 raw_spin_lock_init(&rd->rto_lock);
557 rd->rto_push_work = IRQ_WORK_INIT_HARD(rto_push_irq_work_func);
560 rd->visit_cookie = 0;
561 init_dl_bw(&rd->dl_bw);
562 if (cpudl_init(&rd->cpudl) != 0)
565 if (cpupri_init(&rd->cpupri) != 0)
570 cpudl_cleanup(&rd->cpudl);
572 free_cpumask_var(rd->rto_mask);
574 free_cpumask_var(rd->dlo_mask);
576 free_cpumask_var(rd->online);
578 free_cpumask_var(rd->span);
580 return -ENOMEM;
584 * By default the system creates a single root-domain with all CPUs as
621 tmp = sg->next;
623 if (free_sgc && atomic_dec_and_test(&sg->sgc->ref))
624 kfree(sg->sgc);
626 if (atomic_dec_and_test(&sg->ref))
637 * dropping group/capacity references, freeing where none remain.
639 free_sched_groups(sd->groups, 1);
641 if (sd->shared && atomic_dec_and_test(&sd->shared->ref))
642 kfree(sd->shared);
651 struct sched_domain *parent = sd->parent;
660 call_rcu(&sd->rcu, destroy_sched_domains_rcu);
668 * Also keep a unique ID per domain (we use the first CPU number in the cpumask
684 static void update_top_cache_domain(int cpu)
688 int id = cpu;
691 sd = highest_flag_domain(cpu, SD_SHARE_LLC);
695 sds = sd->shared;
698 rcu_assign_pointer(per_cpu(sd_llc, cpu), sd);
699 per_cpu(sd_llc_size, cpu) = size;
700 per_cpu(sd_llc_id, cpu) = id;
701 rcu_assign_pointer(per_cpu(sd_llc_shared, cpu), sds);
703 sd = lowest_flag_domain(cpu, SD_CLUSTER);
710 * but equals to LLC id on non-Cluster machines.
712 per_cpu(sd_share_id, cpu) = id;
714 sd = lowest_flag_domain(cpu, SD_NUMA);
715 rcu_assign_pointer(per_cpu(sd_numa, cpu), sd);
717 sd = highest_flag_domain(cpu, SD_ASYM_PACKING);
718 rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd);
720 sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL);
721 rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd);
725 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
729 cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
731 struct rq *rq = cpu_rq(cpu);
736 struct sched_domain *parent = tmp->parent;
741 tmp->parent = parent->parent;
743 if (parent->parent) {
744 parent->parent->child = tmp;
745 parent->parent->groups->flags = tmp->flags;
753 if (parent->flags & SD_PREFER_SIBLING)
754 tmp->flags |= SD_PREFER_SIBLING;
757 tmp = tmp->parent;
762 sd = sd->parent;
765 struct sched_group *sg = sd->groups;
773 sg->flags = 0;
774 } while (sg != sd->groups);
776 sd->child = NULL;
780 sched_domain_debug(sd, cpu);
783 tmp = rq->sd;
784 rcu_assign_pointer(rq->sd, sd);
785 dirty_sched_domain_sysctl(cpu);
788 update_top_cache_domain(cpu);
804 * Return the canonical balance CPU for this group, this is the first CPU
821 * Given a node-distance table, for example:
831 * 0 ----- 1
835 * 3 ----- 2
843 * NUMA-2 0-3 0-3 0-3 0-3
844 * groups: {0-1,3},{1-3} {0-2},{0,2-3} {1-3},{0-1,3} {0,2-3},{0-2}
846 * NUMA-1 0-1,3 0-2 1-3 0,2-3
849 * NUMA-0 0 1 2 3
854 * represented multiple times -- hence the "overlap" naming for this part of
858 * domain. For instance Node-0 NUMA-2 would only get groups: 0-1,3 and 1-3.
862 * - the first group of each domain is its child domain; this
863 * gets us the first 0-1,3
864 * - the only uncovered node is 2, who's child domain is 1-3.
866 * However, because of the overlap, computing a unique CPU for each group is
867 * more complicated. Consider for instance the groups of NODE-1 NUMA-2, both
868 * groups include the CPUs of Node-0, while those CPUs would not in fact ever
869 * end up at those groups (they would end up in group: 0-1,3).
892 * 0 ----- 1
896 * 2 ----- 3
902 * not of the same number for each CPU. Consider:
904 * NUMA-2 0-3 0-3
905 * groups: {0-2},{1-3} {1-3},{0-2}
907 * NUMA-1 0-2 0-3 0-3 1-3
909 * NUMA-0 0 1 2 3
927 struct sd_data *sdd = sd->private;
934 sibling = *per_cpu_ptr(sdd->sd, i);
941 if (!sibling->child)
945 if (!cpumask_equal(sg_span, sched_domain_span(sibling->child)))
956 * XXX: This creates per-node group entries; since the load-balancer will
957 * immediately access remote memory to construct this group's load-balance
961 build_group_from_child_sched_domain(struct sched_domain *sd, int cpu)
967 GFP_KERNEL, cpu_to_node(cpu));
973 if (sd->child) {
974 cpumask_copy(sg_span, sched_domain_span(sd->child));
975 sg->flags = sd->child->flags;
980 atomic_inc(&sg->ref);
988 struct sd_data *sdd = sd->private;
990 int cpu;
993 cpu = cpumask_first(mask);
995 sg->sgc = *per_cpu_ptr(sdd->sgc, cpu);
996 if (atomic_inc_return(&sg->sgc->ref) == 1)
1002 * Initialize sgc->capacity such that even if we mess up the
1007 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span);
1008 sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;
1009 sg->sgc->max_capacity = SCHED_CAPACITY_SCALE;
1019 while (sibling->child &&
1020 !cpumask_subset(sched_domain_span(sibling->child),
1022 sibling = sibling->child;
1029 while (sibling->child &&
1030 cpumask_equal(sched_domain_span(sibling->child),
1032 sibling = sibling->child;
1038 build_overlap_sched_groups(struct sched_domain *sd, int cpu)
1043 struct sd_data *sdd = sd->private;
1049 for_each_cpu_wrap(i, span, cpu) {
1055 sibling = *per_cpu_ptr(sdd->sd, i);
1064 * Domains should always include the CPU they're built on, so
1085 * 0 --- 1 --- 2 --- 3
1087 * NUMA-3 0-3 N/A N/A 0-3
1088 * groups: {0-2},{1-3} {1-3},{0-2}
1090 * NUMA-2 0-2 0-3 0-3 1-3
1091 * groups: {0-1},{1-3} {0-2},{2-3} {1-3},{0-1} {2-3},{0-2}
1093 * NUMA-1 0-1 0-2 1-3 2-3
1096 * NUMA-0 0 1 2 3
1098 * The NUMA-2 groups for nodes 0 and 3 are obviously buggered, as the
1101 if (sibling->child &&
1102 !cpumask_subset(sched_domain_span(sibling->child), span))
1105 sg = build_group_from_child_sched_domain(sibling, cpu);
1117 last->next = sg;
1119 last->next = first;
1121 sd->groups = first;
1128 return -ENOMEM;
1133 * Package topology (also see the load-balance blurb in fair.c)
1138 * - Simultaneous multithreading (SMT)
1139 * - Multi-Core Cache (MC)
1140 * - Package (PKG)
1146 * sched_domain -> sched_group -> sched_group_capacity
1148 * `-' `-'
1150 * The sched_domains are per-CPU and have a two way link (parent & child) and
1156 * CPU of that sched_domain [*].
1160 * CPU 0 1 2 3 4 5 6 7
1166 * - or -
1168 * PKG 0-7 0-7 0-7 0-7 0-7 0-7 0-7 0-7
1169 * MC 0-3 0-3 0-3 0-3 4-7 4-7 4-7 4-7
1170 * SMT 0-1 0-1 2-3 2-3 4-5 4-5 6-7 6-7
1172 * CPU 0 1 2 3 4 5 6 7
1180 * There are two related construction problems, both require a CPU that
1183 * - The first is the balance_cpu (see should_we_balance() and the
1184 * load-balance blurb in fair.c); for each group we only want 1 CPU to
1187 * - The second is the sched_group_capacity; we want all identical groups
1193 * for each CPU in the hierarchy.
1195 * Therefore computing a unique CPU for each group is trivial (the iteration
1197 * group), we can simply pick the first CPU in each group.
1203 static struct sched_group *get_group(int cpu, struct sd_data *sdd)
1205 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
1206 struct sched_domain *child = sd->child;
1211 cpu = cpumask_first(sched_domain_span(child));
1213 sg = *per_cpu_ptr(sdd->sg, cpu);
1214 sg->sgc = *per_cpu_ptr(sdd->sgc, cpu);
1217 already_visited = atomic_inc_return(&sg->ref) > 1;
1219 WARN_ON(already_visited != (atomic_inc_return(&sg->sgc->ref) > 1));
1228 sg->flags = child->flags;
1230 cpumask_set_cpu(cpu, sched_group_span(sg));
1231 cpumask_set_cpu(cpu, group_balance_mask(sg));
1234 sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sched_group_span(sg));
1235 sg->sgc->min_capacity = SCHED_CAPACITY_SCALE;
1236 sg->sgc->max_capacity = SCHED_CAPACITY_SCALE;
1243 * covered by the given span, will set each group's ->cpumask correctly,
1244 * and will initialize their ->sgc.
1249 build_sched_groups(struct sched_domain *sd, int cpu)
1252 struct sd_data *sdd = sd->private;
1262 for_each_cpu_wrap(i, span, cpu) {
1275 last->next = sg;
1278 last->next = first;
1279 sd->groups = first;
1287 * cpu_capacity indicates the capacity of sched group, which is used while
1294 static void init_sched_groups_capacity(int cpu, struct sched_domain *sd)
1296 struct sched_group *sg = sd->groups;
1302 int cpu, cores = 0, max_cpu = -1;
1304 sg->group_weight = cpumask_weight(sched_group_span(sg));
1307 for_each_cpu(cpu, mask) {
1310 cpumask_andnot(mask, mask, cpu_smt_mask(cpu));
1313 sg->cores = cores;
1315 if (!(sd->flags & SD_ASYM_PACKING))
1318 for_each_cpu(cpu, sched_group_span(sg)) {
1320 max_cpu = cpu;
1321 else if (sched_asym_prefer(cpu, max_cpu))
1322 max_cpu = cpu;
1324 sg->asym_prefer_cpu = max_cpu;
1327 sg = sg->next;
1328 } while (sg != sd->groups);
1330 if (cpu != group_balance_cpu(sg))
1333 update_group_capacity(sd, cpu);
1338 * Each list entry contains a CPU mask reflecting CPUs that share the same
1339 * capacity.
1345 * Verify whether there is any CPU capacity asymmetry in a given sched domain.
1356 * Count how many unique CPU capacities this domain spans across
1373 /* Some of the available CPU capacity values have not been detected */
1388 static inline void asym_cpu_capacity_update_data(int cpu)
1390 unsigned long capacity = arch_scale_cpu_capacity(cpu);
1395 * Search if capacity already exits. If not, track which the entry
1399 if (capacity == entry->capacity)
1401 else if (!insert_entry && capacity > entry->capacity)
1408 entry->capacity = capacity;
1410 /* If NULL then the new capacity is the smallest, add last. */
1412 list_add_tail_rcu(&entry->link, &asym_cap_list);
1414 list_add_rcu(&entry->link, &insert_entry->link);
1416 __cpumask_set_cpu(cpu, cpu_capacity_span(entry));
1420 * Build-up/update list of CPUs grouped by their capacities
1422 * with state indicating CPU topology changes.
1427 int cpu;
1432 for_each_cpu_and(cpu, cpu_possible_mask, housekeeping_cpumask(HK_TYPE_DOMAIN))
1433 asym_cpu_capacity_update_data(cpu);
1437 list_del_rcu(&entry->link);
1438 call_rcu(&entry->rcu, free_asym_cap_entry);
1443 * Only one capacity value has been detected i.e. this system is symmetric.
1448 list_del_rcu(&entry->link);
1449 call_rcu(&entry->rcu, free_asym_cap_entry);
1455 * Non-inlined to reduce accumulated stack pressure in build_sched_domains()
1458 static int default_relax_domain_level = -1;
1475 if (!attr || attr->relax_domain_level < 0) {
1480 request = attr->relax_domain_level;
1482 if (sd->level >= request) {
1484 sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE);
1496 if (!atomic_read(&d->rd->refcount))
1497 free_rootdomain(&d->rd->rcu);
1500 free_percpu(d->sd);
1517 d->sd = alloc_percpu(struct sched_domain *);
1518 if (!d->sd)
1520 d->rd = alloc_rootdomain();
1521 if (!d->rd)
1532 static void claim_allocations(int cpu, struct sched_domain *sd)
1534 struct sd_data *sdd = sd->private;
1536 WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd);
1537 *per_cpu_ptr(sdd->sd, cpu) = NULL;
1539 if (atomic_read(&(*per_cpu_ptr(sdd->sds, cpu))->ref))
1540 *per_cpu_ptr(sdd->sds, cpu) = NULL;
1542 if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref))
1543 *per_cpu_ptr(sdd->sg, cpu) = NULL;
1545 if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref))
1546 *per_cpu_ptr(sdd->sgc, cpu) = NULL;
1575 * SD_ASYM_PACKING - describes SMT quirks
1587 struct sched_domain *child, int cpu)
1589 struct sd_data *sdd = &tl->data;
1590 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu);
1598 sched_domains_curr_level = tl->numa_level;
1601 sd_weight = cpumask_weight(tl->mask(cpu));
1603 if (tl->sd_flags)
1604 sd_flags = (*tl->sd_flags)();
1635 .name = tl->name,
1639 cpumask_and(sd_span, cpu_map, tl->mask(cpu));
1642 sd->flags |= asym_cpu_capacity_classify(sd_span, cpu_map);
1644 WARN_ONCE((sd->flags & (SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY)) ==
1646 "CPU capacity asymmetry not supported on SMT\n");
1652 if ((sd->flags & SD_ASYM_CPUCAPACITY) && sd->child)
1653 sd->child->flags &= ~SD_PREFER_SIBLING;
1655 if (sd->flags & SD_SHARE_CPUCAPACITY) {
1656 sd->imbalance_pct = 110;
1658 } else if (sd->flags & SD_SHARE_LLC) {
1659 sd->imbalance_pct = 117;
1660 sd->cache_nice_tries = 1;
1663 } else if (sd->flags & SD_NUMA) {
1664 sd->cache_nice_tries = 2;
1666 sd->flags &= ~SD_PREFER_SIBLING;
1667 sd->flags |= SD_SERIALIZE;
1668 if (sched_domains_numa_distance[tl->numa_level] > node_reclaim_distance) {
1669 sd->flags &= ~(SD_BALANCE_EXEC |
1676 sd->cache_nice_tries = 1;
1683 if (sd->flags & SD_SHARE_LLC) {
1684 sd->shared = *per_cpu_ptr(sdd->sds, sd_id);
1685 atomic_inc(&sd->shared->ref);
1686 atomic_set(&sd->shared->nr_busy_cpus, sd_weight);
1689 sd->private = sdd;
1695 * Topology list, bottom-up.
1718 for (tl = sched_domain_topology; tl->mask; tl++)
1731 static const struct cpumask *sd_numa_mask(int cpu)
1733 return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)];
1804 * - If the maximum distance between any nodes is 1 hop, the system
1806 * - If for two nodes A and B, located N > 1 hops away from each other,
1859 * O(nr_nodes^2) de-duplicating selection sort -- in order to find the
1942 sched_numa_warn("Node-distance not symmetric");
1993 WRITE_ONCE(sched_max_numa_distance, sched_domains_numa_distance[nr_levels - 1]);
2036 void sched_update_numa(int cpu, bool online)
2040 node = cpu_to_node(cpu);
2042 * Scheduler NUMA topology is updated when the first CPU of a
2043 * node is onlined or the last CPU of a node is offlined.
2052 void sched_domains_numa_masks_set(unsigned int cpu)
2054 int node = cpu_to_node(cpu);
2064 cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]);
2069 void sched_domains_numa_masks_clear(unsigned int cpu)
2076 cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]);
2082 * sched_numa_find_closest() - given the NUMA topology, find the cpu
2083 * closest to @cpu from @cpumask.
2084 * cpumask: cpumask to find a cpu from
2085 * cpu: cpu to be close to
2087 * returns: cpu, or nr_cpu_ids when nothing found.
2089 int sched_numa_find_closest(const struct cpumask *cpus, int cpu)
2091 int i, j = cpu_to_node(cpu), found = nr_cpu_ids;
2101 cpu = cpumask_any_and(cpus, masks[i][j]);
2102 if (cpu < nr_cpu_ids) {
2103 found = cpu;
2117 int cpu;
2126 if (cpumask_weight_and(k->cpus, cur_hop[k->node]) <= k->cpu)
2129 if (b == k->masks) {
2130 k->w = 0;
2134 prev_hop = *((struct cpumask ***)b - 1);
2135 k->w = cpumask_weight_and(k->cpus, prev_hop[k->node]);
2136 if (k->w <= k->cpu)
2139 return -1;
2143 * sched_numa_find_nth_cpu() - given the NUMA topology, find the Nth closest CPU
2144 * from @cpus to @cpu, taking into account distance
2146 * @cpus: cpumask to find a cpu from
2147 * @cpu: CPU to start searching
2150 * Return: cpu, or nr_cpu_ids when nothing found.
2152 int sched_numa_find_nth_cpu(const struct cpumask *cpus, int cpu, int node)
2154 struct __cmp_key k = { .cpus = cpus, .cpu = cpu };
2159 return cpumask_nth_and(cpu, cpus, cpu_online_mask);
2163 /* CPU-less node entries are uninitialized in sched_domains_numa_masks */
2172 hop = hop_masks - k.masks;
2175 cpumask_nth_and_andnot(cpu - k.w, cpus, k.masks[hop][node], k.masks[hop-1][node]) :
2176 cpumask_nth_and(cpu, cpus, k.masks[0][node]);
2184 * sched_numa_hop_mask() - Get the cpumask of CPUs at most @hops hops away from
2193 * read-side section, copy it if required beyond that.
2205 return ERR_PTR(-EINVAL);
2209 return ERR_PTR(-EBUSY);
2223 struct sd_data *sdd = &tl->data;
2225 sdd->sd = alloc_percpu(struct sched_domain *);
2226 if (!sdd->sd)
2227 return -ENOMEM;
2229 sdd->sds = alloc_percpu(struct sched_domain_shared *);
2230 if (!sdd->sds)
2231 return -ENOMEM;
2233 sdd->sg = alloc_percpu(struct sched_group *);
2234 if (!sdd->sg)
2235 return -ENOMEM;
2237 sdd->sgc = alloc_percpu(struct sched_group_capacity *);
2238 if (!sdd->sgc)
2239 return -ENOMEM;
2250 return -ENOMEM;
2252 *per_cpu_ptr(sdd->sd, j) = sd;
2257 return -ENOMEM;
2259 *per_cpu_ptr(sdd->sds, j) = sds;
2264 return -ENOMEM;
2266 sg->next = sg;
2268 *per_cpu_ptr(sdd->sg, j) = sg;
2273 return -ENOMEM;
2275 sgc->id = j;
2277 *per_cpu_ptr(sdd->sgc, j) = sgc;
2290 struct sd_data *sdd = &tl->data;
2295 if (sdd->sd) {
2296 sd = *per_cpu_ptr(sdd->sd, j);
2297 if (sd && (sd->flags & SD_OVERLAP))
2298 free_sched_groups(sd->groups, 0);
2299 kfree(*per_cpu_ptr(sdd->sd, j));
2302 if (sdd->sds)
2303 kfree(*per_cpu_ptr(sdd->sds, j));
2304 if (sdd->sg)
2305 kfree(*per_cpu_ptr(sdd->sg, j));
2306 if (sdd->sgc)
2307 kfree(*per_cpu_ptr(sdd->sgc, j));
2309 free_percpu(sdd->sd);
2310 sdd->sd = NULL;
2311 free_percpu(sdd->sds);
2312 sdd->sds = NULL;
2313 free_percpu(sdd->sg);
2314 sdd->sg = NULL;
2315 free_percpu(sdd->sgc);
2316 sdd->sgc = NULL;
2322 struct sched_domain *child, int cpu)
2324 struct sched_domain *sd = sd_init(tl, cpu_map, child, cpu);
2327 sd->level = child->level + 1;
2328 sched_domain_level_max = max(sched_domain_level_max, sd->level);
2329 child->parent = sd;
2335 child->name, sd->name);
2350 * any two given CPUs at this (non-NUMA) topology level.
2353 const struct cpumask *cpu_map, int cpu)
2355 int i = cpu + 1;
2358 if (tl->flags & SDTL_OVERLAP)
2362 * Non-NUMA levels cannot partially overlap - they must be either
2364 * breaking the sched_group lists - i.e. a later get_group() pass
2374 if (!cpumask_equal(tl->mask(cpu), tl->mask(i)) &&
2375 cpumask_intersects(tl->mask(cpu), tl->mask(i)))
2393 int i, ret = -ENOMEM;
2416 has_asym |= sd->flags & SD_ASYM_CPUCAPACITY;
2420 if (tl->flags & SDTL_OVERLAP)
2421 sd->flags |= SD_OVERLAP;
2429 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
2430 sd->span_weight = cpumask_weight(sched_domain_span(sd));
2431 if (sd->flags & SD_OVERLAP) {
2449 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
2450 struct sched_domain *child = sd->child;
2452 if (!(sd->flags & SD_SHARE_LLC) && child &&
2453 (child->flags & SD_SHARE_LLC)) {
2460 * arbitrary cutoff based two factors -- SMT and
2461 * memory channels. For SMT-2, the intent is to
2463 * SMT-4 or SMT-8 *may* benefit from a different
2477 nr_llcs = sd->span_weight / child->span_weight;
2479 imb = sd->span_weight >> 3;
2483 sd->imb_numa_nr = imb;
2486 top_p = sd->parent;
2487 while (top_p && !(top_p->flags & SD_NUMA)) {
2488 top_p = top_p->parent;
2490 imb_span = top_p ? top_p->span_weight : sd->span_weight;
2492 int factor = max(1U, (sd->span_weight / imb_span));
2494 sd->imb_numa_nr = imb * factor;
2499 /* Calculate CPU capacity for physical packages and nodes */
2500 for (i = nr_cpumask_bits-1; i >= 0; i--) {
2504 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) {
2557 * CPU core maps. It is supposed to return 1 if the topology changed
2620 unsigned int cpu = cpumask_any(cpu_map);
2623 if (rcu_access_pointer(per_cpu(sd_asym_cpucapacity, cpu)))
2687 /* Let the architecture update CPU core mappings: */
2689 /* Trigger rebuilding CPU capacity asymmetry data */
2713 /* No match - a current sched domain not in new doms_new[] */
2734 /* No match - add a new doms_new */
2745 cpu_rq(cpumask_first(doms_cur[j]))->rd->pd) {
2750 /* No match - add perf domains for a new rd */