1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Oracle Corporation 4 * 5 * Module Author: Mike Christie 6 */ 7 #include "dm-path-selector.h" 8 9 #include <linux/device-mapper.h> 10 #include <linux/module.h> 11 12 #define DM_MSG_PREFIX "multipath io-affinity" 13 14 struct path_info { 15 struct dm_path *path; 16 cpumask_var_t cpumask; 17 refcount_t refcount; 18 bool failed; 19 }; 20 21 struct selector { 22 struct path_info **path_map; 23 cpumask_var_t path_mask; 24 atomic_t map_misses; 25 }; 26 27 static void ioa_free_path(struct selector *s, unsigned int cpu) 28 { 29 struct path_info *pi = s->path_map[cpu]; 30 31 if (!pi) 32 return; 33 34 if (refcount_dec_and_test(&pi->refcount)) { 35 cpumask_clear_cpu(cpu, s->path_mask); 36 free_cpumask_var(pi->cpumask); 37 kfree(pi); 38 39 s->path_map[cpu] = NULL; 40 } 41 } 42 43 static int ioa_add_path(struct path_selector *ps, struct dm_path *path, 44 int argc, char **argv, char **error) 45 { 46 struct selector *s = ps->context; 47 struct path_info *pi = NULL; 48 unsigned int cpu; 49 int ret; 50 51 if (argc != 1) { 52 *error = "io-affinity ps: invalid number of arguments"; 53 return -EINVAL; 54 } 55 56 pi = kzalloc_obj(*pi); 57 if (!pi) { 58 *error = "io-affinity ps: Error allocating path context"; 59 return -ENOMEM; 60 } 61 62 pi->path = path; 63 path->pscontext = pi; 64 refcount_set(&pi->refcount, 1); 65 66 if (!zalloc_cpumask_var(&pi->cpumask, GFP_KERNEL)) { 67 *error = "io-affinity ps: Error allocating cpumask context"; 68 ret = -ENOMEM; 69 goto free_pi; 70 } 71 72 ret = cpumask_parse(argv[0], pi->cpumask); 73 if (ret) { 74 *error = "io-affinity ps: invalid cpumask"; 75 ret = -EINVAL; 76 goto free_mask; 77 } 78 79 for_each_cpu(cpu, pi->cpumask) { 80 if (cpu >= nr_cpu_ids) { 81 DMWARN_LIMIT("Ignoring mapping for CPU %u. Max CPU is %u", 82 cpu, nr_cpu_ids); 83 break; 84 } 85 86 if (s->path_map[cpu]) { 87 DMWARN("CPU mapping for %u exists. Ignoring.", cpu); 88 continue; 89 } 90 91 cpumask_set_cpu(cpu, s->path_mask); 92 s->path_map[cpu] = pi; 93 refcount_inc(&pi->refcount); 94 } 95 96 if (refcount_dec_and_test(&pi->refcount)) { 97 *error = "io-affinity ps: No new/valid CPU mapping found"; 98 ret = -EINVAL; 99 goto free_mask; 100 } 101 102 return 0; 103 104 free_mask: 105 free_cpumask_var(pi->cpumask); 106 free_pi: 107 kfree(pi); 108 return ret; 109 } 110 111 static int ioa_create(struct path_selector *ps, unsigned int argc, char **argv) 112 { 113 struct selector *s; 114 115 s = kmalloc_obj(*s); 116 if (!s) 117 return -ENOMEM; 118 119 s->path_map = kzalloc_objs(struct path_info *, nr_cpu_ids); 120 if (!s->path_map) 121 goto free_selector; 122 123 if (!zalloc_cpumask_var(&s->path_mask, GFP_KERNEL)) 124 goto free_map; 125 126 atomic_set(&s->map_misses, 0); 127 ps->context = s; 128 return 0; 129 130 free_map: 131 kfree(s->path_map); 132 free_selector: 133 kfree(s); 134 return -ENOMEM; 135 } 136 137 static void ioa_destroy(struct path_selector *ps) 138 { 139 struct selector *s = ps->context; 140 unsigned int cpu; 141 142 for_each_cpu(cpu, s->path_mask) 143 ioa_free_path(s, cpu); 144 145 free_cpumask_var(s->path_mask); 146 kfree(s->path_map); 147 kfree(s); 148 149 ps->context = NULL; 150 } 151 152 static int ioa_status(struct path_selector *ps, struct dm_path *path, 153 status_type_t type, char *result, unsigned int maxlen) 154 { 155 struct selector *s = ps->context; 156 struct path_info *pi; 157 int sz = 0; 158 159 if (!path) { 160 DMEMIT("0 "); 161 return sz; 162 } 163 164 switch (type) { 165 case STATUSTYPE_INFO: 166 DMEMIT("%d ", atomic_read(&s->map_misses)); 167 break; 168 case STATUSTYPE_TABLE: 169 pi = path->pscontext; 170 DMEMIT("%*pb ", cpumask_pr_args(pi->cpumask)); 171 break; 172 case STATUSTYPE_IMA: 173 *result = '\0'; 174 break; 175 } 176 177 return sz; 178 } 179 180 static void ioa_fail_path(struct path_selector *ps, struct dm_path *p) 181 { 182 struct path_info *pi = p->pscontext; 183 184 pi->failed = true; 185 } 186 187 static int ioa_reinstate_path(struct path_selector *ps, struct dm_path *p) 188 { 189 struct path_info *pi = p->pscontext; 190 191 pi->failed = false; 192 return 0; 193 } 194 195 static struct dm_path *ioa_select_path(struct path_selector *ps, 196 size_t nr_bytes) 197 { 198 unsigned int cpu, node; 199 struct selector *s = ps->context; 200 const struct cpumask *cpumask; 201 struct path_info *pi; 202 int i; 203 204 cpu = get_cpu(); 205 206 pi = s->path_map[cpu]; 207 if (pi && !pi->failed) 208 goto done; 209 210 /* 211 * Perf is not optimal, but we at least try the local node then just 212 * try not to fail. 213 */ 214 if (!pi) 215 atomic_inc(&s->map_misses); 216 217 node = cpu_to_node(cpu); 218 cpumask = cpumask_of_node(node); 219 for_each_cpu(i, cpumask) { 220 pi = s->path_map[i]; 221 if (pi && !pi->failed) 222 goto done; 223 } 224 225 for_each_cpu(i, s->path_mask) { 226 pi = s->path_map[i]; 227 if (pi && !pi->failed) 228 goto done; 229 } 230 pi = NULL; 231 232 done: 233 put_cpu(); 234 return pi ? pi->path : NULL; 235 } 236 237 static struct path_selector_type ioa_ps = { 238 .name = "io-affinity", 239 .module = THIS_MODULE, 240 .table_args = 1, 241 .info_args = 1, 242 .create = ioa_create, 243 .destroy = ioa_destroy, 244 .status = ioa_status, 245 .add_path = ioa_add_path, 246 .fail_path = ioa_fail_path, 247 .reinstate_path = ioa_reinstate_path, 248 .select_path = ioa_select_path, 249 }; 250 251 static int __init dm_ioa_init(void) 252 { 253 int ret = dm_register_path_selector(&ioa_ps); 254 255 if (ret < 0) 256 DMERR("register failed %d", ret); 257 return ret; 258 } 259 260 static void __exit dm_ioa_exit(void) 261 { 262 dm_unregister_path_selector(&ioa_ps); 263 } 264 265 module_init(dm_ioa_init); 266 module_exit(dm_ioa_exit); 267 268 MODULE_DESCRIPTION(DM_NAME " multipath path selector that selects paths based on the CPU IO is being executed on"); 269 MODULE_AUTHOR("Mike Christie <michael.christie@oracle.com>"); 270 MODULE_LICENSE("GPL"); 271