78 static void scx_cid_tables_free(struct scx_cid_tables *tbls)  in scx_cid_tables_free()  argument
80 	if (!tbls)  in scx_cid_tables_free()
82 	kvfree(tbls->cid_to_cpu);  in scx_cid_tables_free()
83 	kvfree(tbls->cpu_to_cid);  in scx_cid_tables_free()
84 	kvfree(tbls->cid_to_shard);  in scx_cid_tables_free()
85 	kvfree(tbls->shard_node);  in scx_cid_tables_free()
86 	kvfree(tbls->shard_ranges);  in scx_cid_tables_free()
87 	kvfree(tbls->topo);  in scx_cid_tables_free()
88 	kfree(tbls);  in scx_cid_tables_free()
99 	struct scx_cid_tables *tbls;  in scx_cid_alloc_tables()  local
101 	tbls = kzalloc_obj(*tbls, GFP_KERNEL);  in scx_cid_alloc_tables()
102 	if (!tbls)  in scx_cid_alloc_tables()
105 	tbls->cid_to_cpu = kvcalloc(npossible, sizeof(*tbls->cid_to_cpu), GFP_KERNEL);  in scx_cid_alloc_tables()
106 	tbls->cpu_to_cid = kvcalloc(nr_cpu_ids, sizeof(*tbls->cpu_to_cid), GFP_KERNEL);  in scx_cid_alloc_tables()
107 	tbls->cid_to_shard = kvcalloc(npossible, sizeof(*tbls->cid_to_shard), GFP_KERNEL);  in scx_cid_alloc_tables()
108 	tbls->shard_node = kvcalloc(npossible, sizeof(*tbls->shard_node), GFP_KERNEL);  in scx_cid_alloc_tables()
109 	tbls->shard_ranges = kvcalloc(npossible, sizeof(*tbls->shard_ranges), GFP_KERNEL);  in scx_cid_alloc_tables()
110 	tbls->topo = kvcalloc(npossible, sizeof(*tbls->topo), GFP_KERNEL);  in scx_cid_alloc_tables()
112 	if (!tbls->cid_to_cpu || !tbls->cpu_to_cid || !tbls->cid_to_shard ||  in scx_cid_alloc_tables()
113 	    !tbls->shard_node || !tbls->shard_ranges || !tbls->topo) {  in scx_cid_alloc_tables()
114 		scx_cid_tables_free(tbls);  in scx_cid_alloc_tables()
118 	return tbls;  in scx_cid_alloc_tables()
128 	struct scx_cid_tables *tbls = scx_cid_tables;  in scx_cid_publish_tables()  local
132 	scx_nr_cid_shards = tbls->nr_shards;  in scx_cid_publish_tables()
133 	rcu_assign_pointer(scx_cid_to_cpu_tbl, tbls->cid_to_cpu);  in scx_cid_publish_tables()
134 	rcu_assign_pointer(scx_cpu_to_cid_tbl, tbls->cpu_to_cid);  in scx_cid_publish_tables()
135 	rcu_assign_pointer(scx_cid_to_shard, tbls->cid_to_shard);  in scx_cid_publish_tables()
136 	rcu_assign_pointer(scx_shard_node, tbls->shard_node);  in scx_cid_publish_tables()
137 	rcu_assign_pointer(scx_cid_shard_ranges, tbls->shard_ranges);  in scx_cid_publish_tables()
138 	rcu_assign_pointer(scx_cid_topo, tbls->topo);  in scx_cid_publish_tables()
149 	struct scx_cid_tables *tbls = scx_cid_tables;  in scx_cid_retire_tables()  local
154 	if (!tbls)  in scx_cid_retire_tables()
164 	call_rcu(&tbls->rcu, scx_cid_tables_free_rcufn);  in scx_cid_retire_tables()
188 	struct scx_cid_tables *tbls;  in scx_cid_init()  local
206 	tbls = scx_cid_alloc_tables();  in scx_cid_init()
207 	if (!tbls)  in scx_cid_init()
210 	scx_cid_tables = tbls;  in scx_cid_init()
213 		tbls->shard_node[si] = NUMA_NO_NODE;  in scx_cid_init()
225 		tbls->cpu_to_cid[cpu] = -1;  in scx_cid_init()
268 			tbls->shard_node[shard_idx] = nid;  in scx_cid_init()
299 					tbls->shard_node[shard_idx] = nid;  in scx_cid_init()
307 					tbls->cid_to_cpu[cid] = ccpu;  in scx_cid_init()
308 					tbls->cpu_to_cid[ccpu] = cid;  in scx_cid_init()
309 					tbls->cid_to_shard[cid] = shard_idx;  in scx_cid_init()
310 					tbls->topo[cid] = (struct scx_cid_topo){  in scx_cid_init()
342 		if (tbls->cpu_to_cid[cpu] != -1)  in scx_cid_init()
348 		tbls->cid_to_cpu[cid] = cpu;  in scx_cid_init()
349 		tbls->cpu_to_cid[cpu] = cid;  in scx_cid_init()
358 		tbls->cid_to_shard[cid] = notopo_shard_idx;  in scx_cid_init()
359 		tbls->topo[cid] = SCX_CID_TOPO_NEG;  in scx_cid_init()
360 		tbls->topo[cid].shard_cid = notopo_shard_cid;  in scx_cid_init()
361 		tbls->topo[cid].shard_idx = notopo_shard_idx;  in scx_cid_init()
377 		s32 sidx = tbls->cid_to_shard[cid];  in scx_cid_init()
379 		if (tbls->shard_ranges[sidx].nr_cids == 0)  in scx_cid_init()
380 			tbls->shard_ranges[sidx].base_cid = cid;  in scx_cid_init()
381 		tbls->shard_ranges[sidx].nr_cids++;  in scx_cid_init()
384 	tbls->nr_shards = next_shard_idx;  in scx_cid_init()
476 	struct scx_cid_tables *tbls;  in scx_bpf_cid_override()  local
509 	tbls = scx_cid_tables;  in scx_bpf_cid_override()
574 		tbls->cpu_to_cid[cpu] = c;  in scx_bpf_cid_override()
575 		tbls->cid_to_cpu[c] = cpu;  in scx_bpf_cid_override()
587 			s32 node = cpu_to_node(tbls->cid_to_cpu[cid]);  in scx_bpf_cid_override()
592 		tbls->shard_node[si] = pick_max_node(node_counts, nr_node_ids);  in scx_bpf_cid_override()
603 		tbls->cid_to_shard[cid] = si;  in scx_bpf_cid_override()
604 		tbls->topo[cid] = SCX_CID_TOPO_NEG;  in scx_bpf_cid_override()
605 		tbls->topo[cid].shard_cid = shard_start[si];  in scx_bpf_cid_override()
606 		tbls->topo[cid].shard_idx = si;  in scx_bpf_cid_override()
610 	memset(tbls->shard_ranges, 0, npossible * sizeof(*tbls->shard_ranges));  in scx_bpf_cid_override()
614 		tbls->shard_ranges[si].base_cid = shard_start[si];  in scx_bpf_cid_override()
615 		tbls->shard_ranges[si].nr_cids = end - shard_start[si];  in scx_bpf_cid_override()
618 	tbls->nr_shards = nr_shards;  in scx_bpf_cid_override()