1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Oracle Corporation
4 *
5 * Module Author: Mike Christie
6 */
7 #include "dm-path-selector.h"
8
9 #include <linux/device-mapper.h>
10 #include <linux/module.h>
11
12 #define DM_MSG_PREFIX "multipath io-affinity"
13
14 struct path_info {
15 struct dm_path *path;
16 cpumask_var_t cpumask;
17 refcount_t refcount;
18 bool failed;
19 };
20
21 struct selector {
22 struct path_info **path_map;
23 cpumask_var_t path_mask;
24 atomic_t map_misses;
25 };
26
ioa_free_path(struct selector * s,unsigned int cpu)27 static void ioa_free_path(struct selector *s, unsigned int cpu)
28 {
29 struct path_info *pi = s->path_map[cpu];
30
31 if (!pi)
32 return;
33
34 if (refcount_dec_and_test(&pi->refcount)) {
35 cpumask_clear_cpu(cpu, s->path_mask);
36 free_cpumask_var(pi->cpumask);
37 kfree(pi);
38
39 s->path_map[cpu] = NULL;
40 }
41 }
42
ioa_add_path(struct path_selector * ps,struct dm_path * path,int argc,char ** argv,char ** error)43 static int ioa_add_path(struct path_selector *ps, struct dm_path *path,
44 int argc, char **argv, char **error)
45 {
46 struct selector *s = ps->context;
47 struct path_info *pi = NULL;
48 unsigned int cpu;
49 int ret;
50
51 if (argc != 1) {
52 *error = "io-affinity ps: invalid number of arguments";
53 return -EINVAL;
54 }
55
56 pi = kzalloc_obj(*pi);
57 if (!pi) {
58 *error = "io-affinity ps: Error allocating path context";
59 return -ENOMEM;
60 }
61
62 pi->path = path;
63 path->pscontext = pi;
64 refcount_set(&pi->refcount, 1);
65
66 if (!zalloc_cpumask_var(&pi->cpumask, GFP_KERNEL)) {
67 *error = "io-affinity ps: Error allocating cpumask context";
68 ret = -ENOMEM;
69 goto free_pi;
70 }
71
72 ret = cpumask_parse(argv[0], pi->cpumask);
73 if (ret) {
74 *error = "io-affinity ps: invalid cpumask";
75 ret = -EINVAL;
76 goto free_mask;
77 }
78
79 for_each_cpu(cpu, pi->cpumask) {
80 if (cpu >= nr_cpu_ids) {
81 DMWARN_LIMIT("Ignoring mapping for CPU %u. Max CPU is %u",
82 cpu, nr_cpu_ids);
83 break;
84 }
85
86 if (s->path_map[cpu]) {
87 DMWARN("CPU mapping for %u exists. Ignoring.", cpu);
88 continue;
89 }
90
91 cpumask_set_cpu(cpu, s->path_mask);
92 s->path_map[cpu] = pi;
93 refcount_inc(&pi->refcount);
94 }
95
96 if (refcount_dec_and_test(&pi->refcount)) {
97 *error = "io-affinity ps: No new/valid CPU mapping found";
98 ret = -EINVAL;
99 goto free_mask;
100 }
101
102 return 0;
103
104 free_mask:
105 free_cpumask_var(pi->cpumask);
106 free_pi:
107 kfree(pi);
108 return ret;
109 }
110
ioa_create(struct path_selector * ps,unsigned int argc,char ** argv)111 static int ioa_create(struct path_selector *ps, unsigned int argc, char **argv)
112 {
113 struct selector *s;
114
115 s = kmalloc_obj(*s);
116 if (!s)
117 return -ENOMEM;
118
119 s->path_map = kzalloc_objs(struct path_info *, nr_cpu_ids);
120 if (!s->path_map)
121 goto free_selector;
122
123 if (!zalloc_cpumask_var(&s->path_mask, GFP_KERNEL))
124 goto free_map;
125
126 atomic_set(&s->map_misses, 0);
127 ps->context = s;
128 return 0;
129
130 free_map:
131 kfree(s->path_map);
132 free_selector:
133 kfree(s);
134 return -ENOMEM;
135 }
136
ioa_destroy(struct path_selector * ps)137 static void ioa_destroy(struct path_selector *ps)
138 {
139 struct selector *s = ps->context;
140 unsigned int cpu;
141
142 for_each_cpu(cpu, s->path_mask)
143 ioa_free_path(s, cpu);
144
145 free_cpumask_var(s->path_mask);
146 kfree(s->path_map);
147 kfree(s);
148
149 ps->context = NULL;
150 }
151
ioa_status(struct path_selector * ps,struct dm_path * path,status_type_t type,char * result,unsigned int maxlen)152 static int ioa_status(struct path_selector *ps, struct dm_path *path,
153 status_type_t type, char *result, unsigned int maxlen)
154 {
155 struct selector *s = ps->context;
156 struct path_info *pi;
157 int sz = 0;
158
159 if (!path) {
160 DMEMIT("0 ");
161 return sz;
162 }
163
164 switch (type) {
165 case STATUSTYPE_INFO:
166 DMEMIT("%d ", atomic_read(&s->map_misses));
167 break;
168 case STATUSTYPE_TABLE:
169 pi = path->pscontext;
170 DMEMIT("%*pb ", cpumask_pr_args(pi->cpumask));
171 break;
172 case STATUSTYPE_IMA:
173 *result = '\0';
174 break;
175 }
176
177 return sz;
178 }
179
ioa_fail_path(struct path_selector * ps,struct dm_path * p)180 static void ioa_fail_path(struct path_selector *ps, struct dm_path *p)
181 {
182 struct path_info *pi = p->pscontext;
183
184 pi->failed = true;
185 }
186
ioa_reinstate_path(struct path_selector * ps,struct dm_path * p)187 static int ioa_reinstate_path(struct path_selector *ps, struct dm_path *p)
188 {
189 struct path_info *pi = p->pscontext;
190
191 pi->failed = false;
192 return 0;
193 }
194
ioa_select_path(struct path_selector * ps,size_t nr_bytes)195 static struct dm_path *ioa_select_path(struct path_selector *ps,
196 size_t nr_bytes)
197 {
198 unsigned int cpu, node;
199 struct selector *s = ps->context;
200 const struct cpumask *cpumask;
201 struct path_info *pi;
202 int i;
203
204 cpu = get_cpu();
205
206 pi = s->path_map[cpu];
207 if (pi && !pi->failed)
208 goto done;
209
210 /*
211 * Perf is not optimal, but we at least try the local node then just
212 * try not to fail.
213 */
214 if (!pi)
215 atomic_inc(&s->map_misses);
216
217 node = cpu_to_node(cpu);
218 cpumask = cpumask_of_node(node);
219 for_each_cpu(i, cpumask) {
220 pi = s->path_map[i];
221 if (pi && !pi->failed)
222 goto done;
223 }
224
225 for_each_cpu(i, s->path_mask) {
226 pi = s->path_map[i];
227 if (pi && !pi->failed)
228 goto done;
229 }
230 pi = NULL;
231
232 done:
233 put_cpu();
234 return pi ? pi->path : NULL;
235 }
236
237 static struct path_selector_type ioa_ps = {
238 .name = "io-affinity",
239 .module = THIS_MODULE,
240 .table_args = 1,
241 .info_args = 1,
242 .create = ioa_create,
243 .destroy = ioa_destroy,
244 .status = ioa_status,
245 .add_path = ioa_add_path,
246 .fail_path = ioa_fail_path,
247 .reinstate_path = ioa_reinstate_path,
248 .select_path = ioa_select_path,
249 };
250
dm_ioa_init(void)251 static int __init dm_ioa_init(void)
252 {
253 int ret = dm_register_path_selector(&ioa_ps);
254
255 if (ret < 0)
256 DMERR("register failed %d", ret);
257 return ret;
258 }
259
dm_ioa_exit(void)260 static void __exit dm_ioa_exit(void)
261 {
262 dm_unregister_path_selector(&ioa_ps);
263 }
264
265 module_init(dm_ioa_init);
266 module_exit(dm_ioa_exit);
267
268 MODULE_DESCRIPTION(DM_NAME " multipath path selector that selects paths based on the CPU IO is being executed on");
269 MODULE_AUTHOR("Mike Christie <michael.christie@oracle.com>");
270 MODULE_LICENSE("GPL");
271