Lines Matching full:sd
46 static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, in sched_domain_debug_one() argument
49 struct sched_group *group = sd->groups; in sched_domain_debug_one()
50 unsigned long flags = sd->flags; in sched_domain_debug_one()
57 cpumask_pr_args(sched_domain_span(sd)), sd->name); in sched_domain_debug_one()
59 if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) { in sched_domain_debug_one()
70 if ((meta_flags & SDF_SHARED_CHILD) && sd->child && in sched_domain_debug_one()
71 !(sd->child->flags & flag)) in sched_domain_debug_one()
75 if ((meta_flags & SDF_SHARED_PARENT) && sd->parent && in sched_domain_debug_one()
76 !(sd->parent->flags & flag)) in sched_domain_debug_one()
95 if (!(sd->flags & SD_NUMA) && in sched_domain_debug_one()
108 if ((sd->flags & SD_NUMA) && in sched_domain_debug_one()
117 if (group == sd->groups && sd->child && in sched_domain_debug_one()
118 !cpumask_equal(sched_domain_span(sd->child), in sched_domain_debug_one()
127 if (group != sd->groups) in sched_domain_debug_one()
130 } while (group != sd->groups); in sched_domain_debug_one()
133 if (!cpumask_equal(sched_domain_span(sd), groupmask)) in sched_domain_debug_one()
136 if (sd->parent && in sched_domain_debug_one()
137 !cpumask_subset(groupmask, sched_domain_span(sd->parent))) in sched_domain_debug_one()
142 static void sched_domain_debug(struct sched_domain *sd, int cpu) in sched_domain_debug() argument
149 if (!sd) { in sched_domain_debug()
157 if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask)) in sched_domain_debug()
160 sd = sd->parent; in sched_domain_debug()
161 if (!sd) in sched_domain_debug()
166 /* Generate a mask of SD flags with the SDF_NEEDS_GROUPS metaflag */
173 static int sd_degenerate(struct sched_domain *sd) in sd_degenerate() argument
175 if (cpumask_weight(sched_domain_span(sd)) == 1) in sd_degenerate()
179 if ((sd->flags & SD_DEGENERATE_GROUPS_MASK) && in sd_degenerate()
180 (sd->groups != sd->groups->next)) in sd_degenerate()
184 if (sd->flags & (SD_WAKE_AFFINE)) in sd_degenerate()
191 sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) in sd_parent_degenerate() argument
193 unsigned long cflags = sd->flags, pflags = parent->flags; in sd_parent_degenerate()
198 if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent))) in sd_parent_degenerate()
632 static void destroy_sched_domain(struct sched_domain *sd) in destroy_sched_domain() argument
639 free_sched_groups(sd->groups, 1); in destroy_sched_domain()
640 free_sched_domain_shared(sd->shared); in destroy_sched_domain()
643 /* only the bottom sd has llc_counts array */ in destroy_sched_domain()
644 kfree(sd->llc_counts); in destroy_sched_domain()
646 kfree(sd); in destroy_sched_domain()
651 struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu); in destroy_sched_domains_rcu() local
653 while (sd) { in destroy_sched_domains_rcu()
654 struct sched_domain *parent = sd->parent; in destroy_sched_domains_rcu()
655 destroy_sched_domain(sd); in destroy_sched_domains_rcu()
656 sd = parent; in destroy_sched_domains_rcu()
660 static void destroy_sched_domains(struct sched_domain *sd) in destroy_sched_domains() argument
662 if (sd) in destroy_sched_domains()
663 call_rcu(&sd->rcu, destroy_sched_domains_rcu); in destroy_sched_domains()
691 struct sched_domain *sd; in update_top_cache_domain() local
695 sd = highest_flag_domain(cpu, SD_SHARE_LLC); in update_top_cache_domain()
696 if (sd) { in update_top_cache_domain()
697 id = cpumask_first(sched_domain_span(sd)); in update_top_cache_domain()
698 size = cpumask_weight(sched_domain_span(sd)); in update_top_cache_domain()
701 WARN_ON_ONCE(!sd->shared); in update_top_cache_domain()
702 sds = sd->shared; in update_top_cache_domain()
705 rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); in update_top_cache_domain()
709 sd = lowest_flag_domain(cpu, SD_CLUSTER); in update_top_cache_domain()
710 if (sd) in update_top_cache_domain()
711 id = cpumask_first(sched_domain_span(sd)); in update_top_cache_domain()
720 sd = lowest_flag_domain(cpu, SD_NUMA); in update_top_cache_domain()
721 rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); in update_top_cache_domain()
723 sd = highest_flag_domain(cpu, SD_ASYM_PACKING); in update_top_cache_domain()
724 rcu_assign_pointer(per_cpu(sd_asym_packing, cpu), sd); in update_top_cache_domain()
726 sd = lowest_flag_domain(cpu, SD_ASYM_CPUCAPACITY_FULL); in update_top_cache_domain()
731 * SD_SHARE_LLC path own the shared object, so sd->shared may be NULL in update_top_cache_domain()
734 if (sd && sd->shared) in update_top_cache_domain()
735 sds = sd->shared; in update_top_cache_domain()
737 rcu_assign_pointer(per_cpu(sd_asym_cpucapacity, cpu), sd); in update_top_cache_domain()
742 * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
746 cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) in cpu_attach_domain() argument
752 for (tmp = sd; tmp; ) { in cpu_attach_domain()
763 * It is safe to free a sd->shared that in cpu_attach_domain()
765 * sd->shared was published, the refcount in cpu_attach_domain()
791 if (sd && sd_degenerate(sd)) { in cpu_attach_domain()
792 tmp = sd; in cpu_attach_domain()
793 sd = sd->parent; in cpu_attach_domain()
795 if (sd) { in cpu_attach_domain()
796 struct sched_group *sg = sd->groups; in cpu_attach_domain()
800 sd->llc_counts = tmp->llc_counts; in cpu_attach_domain()
801 sd->llc_max = tmp->llc_max; in cpu_attach_domain()
802 sd->llc_bytes = tmp->llc_bytes; in cpu_attach_domain()
815 } while (sg != sd->groups); in cpu_attach_domain()
817 sd->child = NULL; in cpu_attach_domain()
823 sched_domain_debug(sd, cpu); in cpu_attach_domain()
826 tmp = rq->sd; in cpu_attach_domain()
827 rcu_assign_pointer(rq->sd, sd); in cpu_attach_domain()
836 struct sched_domain * __percpu *sd; member
863 * weight to the hardware LLC sharing weight. @sd should be the
875 struct sched_domain *sd) in get_effective_llc_bytes() argument
888 return div_u64((u64)ci->size * sd->span_weight, hw_weight); in get_effective_llc_bytes()
894 struct sched_domain *sd, *top_llc, *parent; in alloc_sd_llc() local
899 sd = *per_cpu_ptr(d->sd, i); in alloc_sd_llc()
900 if (!sd) in alloc_sd_llc()
908 top_llc = sd; in alloc_sd_llc()
919 sd->llc_max = max_lid + 1; in alloc_sd_llc()
920 sd->llc_counts = p; in alloc_sd_llc()
921 sd->llc_bytes = get_effective_llc_bytes(i, top_llc); in alloc_sd_llc()
931 sd = *per_cpu_ptr(d->sd, i); in alloc_sd_llc()
932 if (sd) { in alloc_sd_llc()
933 kfree(sd->llc_counts); in alloc_sd_llc()
934 sd->llc_counts = NULL; in alloc_sd_llc()
935 sd->llc_max = 0; in alloc_sd_llc()
936 sd->llc_bytes = 0; in alloc_sd_llc()
1001 struct sched_domain *sd, *sdp; in sched_update_llc_bytes() local
1019 sd = rcu_dereference_sched_domain(cpu_rq(i)->sd); in sched_update_llc_bytes()
1020 if (sd) in sched_update_llc_bytes()
1021 sd->llc_bytes = get_effective_llc_bytes(i, sdp); in sched_update_llc_bytes()
1051 * Return true if @sd belongs to an LLC group whose enclosing
1052 * partition spans more than one LLC. @sd must be the topmost
1055 * Any duplicated parent domains with the same span as @sd are
1067 static bool sd_in_multi_llcs(struct sched_domain *sd) in sd_in_multi_llcs() argument
1069 struct sched_domain *sdp = sd->parent; in sd_in_multi_llcs()
1072 if (sd->span_weight == 1) in sd_in_multi_llcs()
1075 while (sdp && sdp->span_weight == sd->span_weight) in sd_in_multi_llcs()
1202 build_balance_mask(struct sched_domain *sd, struct sched_group *sg, struct cpumask *mask) in build_balance_mask() argument
1205 struct sd_data *sdd = sd->private; in build_balance_mask()
1212 sibling = *per_cpu_ptr(sdd->sd, i); in build_balance_mask()
1239 build_group_from_child_sched_domain(struct sched_domain *sd, int cpu) in build_group_from_child_sched_domain() argument
1251 if (sd->child) { in build_group_from_child_sched_domain()
1252 cpumask_copy(sg_span, sched_domain_span(sd->child)); in build_group_from_child_sched_domain()
1253 sg->flags = sd->child->flags; in build_group_from_child_sched_domain()
1255 cpumask_copy(sg_span, sched_domain_span(sd)); in build_group_from_child_sched_domain()
1262 static void init_overlap_sched_group(struct sched_domain *sd, in init_overlap_sched_group() argument
1266 struct sd_data *sdd = sd->private; in init_overlap_sched_group()
1270 build_balance_mask(sd, sg, mask); in init_overlap_sched_group()
1291 find_descended_sibling(struct sched_domain *sd, struct sched_domain *sibling) in find_descended_sibling() argument
1295 * of sd in find_descended_sibling()
1299 sched_domain_span(sd))) in find_descended_sibling()
1316 build_overlap_sched_groups(struct sched_domain *sd, int cpu) in build_overlap_sched_groups() argument
1319 const struct cpumask *span = sched_domain_span(sd); in build_overlap_sched_groups()
1321 struct sd_data *sdd = sd->private; in build_overlap_sched_groups()
1333 sibling = *per_cpu_ptr(sdd->sd, i); in build_overlap_sched_groups()
1341 * iteration early and our sibling sd spans will be empty. in build_overlap_sched_groups()
1381 sibling = find_descended_sibling(sd, sibling); in build_overlap_sched_groups()
1399 sd->groups = first; in build_overlap_sched_groups()
1483 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); in get_group() local
1484 struct sched_domain *child = sd->child; in get_group()
1527 build_sched_groups(struct sched_domain *sd, int cpu) in build_sched_groups() argument
1530 struct sd_data *sdd = sd->private; in build_sched_groups()
1531 const struct cpumask *span = sched_domain_span(sd); in build_sched_groups()
1557 sd->groups = first; in build_sched_groups()
1572 static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) in init_sched_groups_capacity() argument
1574 struct sched_group *sg = sd->groups; in init_sched_groups_capacity()
1591 if (!(sd->flags & SD_ASYM_PACKING)) in init_sched_groups_capacity()
1604 } while (sg != sd->groups); in init_sched_groups_capacity()
1609 update_group_capacity(sd, cpu); in init_sched_groups_capacity()
1616 struct sched_domain *sd; in sched_update_asym_prefer_cpu() local
1620 for_each_domain(cpu, sd) { in sched_update_asym_prefer_cpu()
1624 if (!(sd->flags & SD_ASYM_PACKING)) in sched_update_asym_prefer_cpu()
1636 WARN_ON_ONCE(sd->flags & SD_NUMA); in sched_update_asym_prefer_cpu()
1638 sg = sd->groups; in sched_update_asym_prefer_cpu()
1800 static void set_domain_attribute(struct sched_domain *sd, in set_domain_attribute() argument
1812 if (sd->level >= request) { in set_domain_attribute()
1814 sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); in set_domain_attribute()
1833 free_percpu(d->sd); in __free_domain_allocs()
1855 d->sd = alloc_percpu(struct sched_domain *); in __visit_domain_allocation_hell()
1856 if (!d->sd) in __visit_domain_allocation_hell()
1872 struct sched_domain *sd; in claim_allocations() local
1877 for (sd = *per_cpu_ptr(d->sd, cpu); sd; sd = sd->parent) { in claim_allocations()
1878 struct sd_data *sdd = sd->private; in claim_allocations()
1880 WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd); in claim_allocations()
1881 *per_cpu_ptr(sdd->sd, cpu) = NULL; in claim_allocations()
1938 struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); in sd_init() local
1943 sd_span = sched_domain_span(sd); in sd_init()
1955 *sd = (struct sched_domain){ in sd_init()
1991 WARN_ONCE((sd->flags & (SD_SHARE_CPUCAPACITY | SD_ASYM_CPUCAPACITY)) == in sd_init()
1998 if (sd->flags & SD_SHARE_CPUCAPACITY) { in sd_init()
1999 sd->imbalance_pct = 110; in sd_init()
2001 } else if (sd->flags & SD_SHARE_LLC) { in sd_init()
2002 sd->imbalance_pct = 117; in sd_init()
2003 sd->cache_nice_tries = 1; in sd_init()
2006 } else if (sd->flags & SD_NUMA) { in sd_init()
2007 sd->cache_nice_tries = 2; in sd_init()
2009 sd->flags &= ~SD_PREFER_SIBLING; in sd_init()
2010 sd->flags |= SD_SERIALIZE; in sd_init()
2012 sd->flags &= ~(SD_BALANCE_EXEC | in sd_init()
2019 sd->cache_nice_tries = 1; in sd_init()
2022 sd->private = sdd; in sd_init()
2024 return sd; in sd_init()
2667 sdd->sd = alloc_percpu(struct sched_domain *); in __sdt_alloc()
2668 if (!sdd->sd) in __sdt_alloc()
2680 struct sched_domain *sd; in __sdt_alloc() local
2684 sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), in __sdt_alloc()
2686 if (!sd) in __sdt_alloc()
2689 *per_cpu_ptr(sdd->sd, j) = sd; in __sdt_alloc()
2723 struct sched_domain *sd; in __sdt_free() local
2725 if (sdd->sd) { in __sdt_free()
2726 sd = *per_cpu_ptr(sdd->sd, j); in __sdt_free()
2727 if (sd && (sd->flags & SD_NUMA)) in __sdt_free()
2728 free_sched_groups(sd->groups, 0); in __sdt_free()
2729 kfree(*per_cpu_ptr(sdd->sd, j)); in __sdt_free()
2737 free_percpu(sdd->sd); in __sdt_free()
2738 sdd->sd = NULL; in __sdt_free()
2786 struct sched_domain *sd = sd_init(tl, cpu_map, child, cpu); in build_sched_domain() local
2789 sd->level = child->level + 1; in build_sched_domain()
2790 sched_domain_level_max = max(sched_domain_level_max, sd->level); in build_sched_domain()
2791 child->parent = sd; in build_sched_domain()
2794 sched_domain_span(sd))) { in build_sched_domain()
2797 child->name, sd->name); in build_sched_domain()
2798 /* Fixup, ensure @sd has at least @child CPUs. */ in build_sched_domain()
2799 cpumask_or(sched_domain_span(sd), in build_sched_domain()
2800 sched_domain_span(sd), in build_sched_domain()
2805 set_domain_attribute(sd, attr); in build_sched_domain()
2807 return sd; in build_sched_domain()
2936 init_sched_domain_shared(struct s_data *d, struct sched_domain *sd, int flags) in init_sched_domain_shared() argument
2944 * same cpumask_first(sched_domain_span(sd)) CPU and can in init_sched_domain_shared()
2948 * Find the first CPU in sched_domain_span(sd) with an in init_sched_domain_shared()
2955 for_each_cpu(cpu, sched_domain_span(sd)) { in init_sched_domain_shared()
2959 sd->span_weight == 1 || in init_sched_domain_shared()
2962 sd->shared = sds; in init_sched_domain_shared()
2971 if (WARN_ON_ONCE(!sd->shared)) in init_sched_domain_shared()
2972 sd->shared = sds; in init_sched_domain_shared()
2979 atomic_set(&sd->shared->nr_busy_cpus, sd->span_weight); in init_sched_domain_shared()
2980 atomic_inc(&sd->shared->ref); in init_sched_domain_shared()
2996 * Return true if this CPU's asym path claimed sd->shared, false otherwise.
3000 struct sched_domain *sd = *per_cpu_ptr(d->sd, cpu); in claim_asym_sched_domain_shared() local
3003 if (!sd) in claim_asym_sched_domain_shared()
3006 sd_asym = sd; in claim_asym_sched_domain_shared()
3082 struct sched_domain *sd; in build_sched_domains() local
3101 sd = NULL; in build_sched_domains()
3104 sd = build_sched_domain(tl, cpu_map, attr, sd, i); in build_sched_domains()
3106 has_asym |= sd->flags & SD_ASYM_CPUCAPACITY; in build_sched_domains()
3109 *per_cpu_ptr(d.sd, i) = sd; in build_sched_domains()
3110 if (cpumask_equal(cpu_map, sched_domain_span(sd))) in build_sched_domains()
3143 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { in build_sched_domains()
3144 sd->span_weight = cpumask_weight(sched_domain_span(sd)); in build_sched_domains()
3145 if (sd->flags & SD_NUMA) { in build_sched_domains()
3146 if (build_overlap_sched_groups(sd, i)) in build_sched_domains()
3149 if (build_sched_groups(sd, i)) in build_sched_domains()
3156 sd = *per_cpu_ptr(d.sd, i); in build_sched_domains()
3157 if (!sd) in build_sched_domains()
3164 while (sd->parent && (sd->parent->flags & SD_SHARE_LLC)) in build_sched_domains()
3165 sd = sd->parent; in build_sched_domains()
3167 if (sd->flags & SD_SHARE_LLC) { in build_sched_domains()
3168 init_sched_domain_shared(&d, sd, SD_SHARE_LLC); in build_sched_domains()
3174 if (sd->parent) { in build_sched_domains()
3176 adjust_numa_imbalance(sd); in build_sched_domains()
3178 if (sd_in_multi_llcs(sd)) in build_sched_domains()
3191 for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) in build_sched_domains()
3192 init_sched_groups_capacity(i, sd); in build_sched_domains()
3201 sd = *per_cpu_ptr(d.sd, i); in build_sched_domains()
3203 cpu_attach_domain(sd, d.rd, i); in build_sched_domains()
3438 struct sched_domain *sd; in partition_sched_domains_locked() local
3442 sd = rcu_dereference(cpu_rq(cpu)->sd); in partition_sched_domains_locked()
3443 while (sd && sd->parent && (sd->parent->flags & SD_SHARE_LLC)) in partition_sched_domains_locked()
3444 sd = sd->parent; in partition_sched_domains_locked()
3445 if (sd && (sd->flags & SD_SHARE_LLC) && sd->parent && in partition_sched_domains_locked()
3446 sd_in_multi_llcs(sd)) in partition_sched_domains_locked()