121 static void free_pshard(struct scx_pshard *pshard)  in free_pshard()  argument
125 	if (!pshard)  in free_pshard()
127 	cu = &pshard->caps_updated;  in free_pshard()
129 		scx_arena_free(pshard->sch, cu->cmask_arena_out,  in free_pshard()
131 					     SCX_CMASK_NR_WORDS(pshard->nr_cids)));  in free_pshard()
132 	kfree(pshard);  in free_pshard()
139 	if (!sch->pshard)  in scx_free_pshards()
142 		free_pshard(sch->pshard[si]);  in scx_free_pshards()
143 	kfree(sch->pshard);  in scx_free_pshards()
153 	struct scx_pshard *pshard;  in alloc_pshard()  local
157 	pshard = kzalloc_node(sizeof(*pshard), GFP_KERNEL, node);  in alloc_pshard()
158 	if (!pshard)  in alloc_pshard()
161 	raw_spin_lock_init(&pshard->lock);  in alloc_pshard()
162 	pshard->sch = sch;  in alloc_pshard()
163 	pshard->base = shard->base_cid;  in alloc_pshard()
164 	pshard->nr_cids = shard->nr_cids;  in alloc_pshard()
167 		scx_cmask_init(&pshard->caps[i].cmask, shard->base_cid, shard->nr_cids);  in alloc_pshard()
169 	cu = &pshard->caps_updated;  in alloc_pshard()
176 		free_pshard(pshard);  in alloc_pshard()
182 	return pshard;  in alloc_pshard()
187 	struct scx_pshard **pshard;  in scx_alloc_pshards()  local
197 	pshard = kzalloc_objs(pshard[0], scx_nr_cid_shards, GFP_KERNEL);  in scx_alloc_pshards()
198 	if (!pshard)  in scx_alloc_pshards()
202 		pshard[si] = alloc_pshard(sch, si, shard_node[si]);  in scx_alloc_pshards()
203 		if (!pshard[si]) {  in scx_alloc_pshards()
205 				free_pshard(pshard[si]);  in scx_alloc_pshards()
206 			kfree(pshard);  in scx_alloc_pshards()
219 	WRITE_ONCE(sch->pshard, pshard);  in scx_alloc_pshards()
232 		struct scx_pshard *ps = sch->pshard[si];  in scx_init_root_caps()
891 		struct scx_pshard *ps = sch->pshard[si];  in scx_sub_seed_caps()
997 		struct scx_pshard *ps = pcpu->sch->pshard[shard];  in scx_process_sync_ecaps()
1089 	ps = sch->pshard[rcu_dereference_all(scx_cid_to_shard)[cid]];  in scx_unbypass_replay_ecaps()
1127 		ps = pos->pshard[shard];  in scx_online_ecaps()
2345 		struct scx_pshard *pps = parent->pshard[si];  in scx_bpf_sub_grant()
2346 		struct scx_pshard *cps = child->pshard[si];  in scx_bpf_sub_grant()
2464 		guard(raw_spinlock)(&parent->pshard[si]->lock);  in scx_bpf_sub_revoke()
2467 			struct scx_pshard *ps = pos->pshard[si];  in scx_bpf_sub_revoke()
2530 	struct scx_pshard **pshard;  in scx_bpf_sub_caps()  local
2569 	pshard = smp_load_acquire(&target->pshard);  in scx_bpf_sub_caps()
2570 	if (unlikely(!pshard)) {  in scx_bpf_sub_caps()
2588 			scx_cmask_or(local_out, &pshard[si]->caps[cap_bit].cmask);  in scx_bpf_sub_caps()