1 // SPDX-License-Identifier: GPL-2.0 2 // 3 // Register cache access API - maple tree based cache 4 // 5 // Copyright 2023 Arm, Ltd 6 // 7 // Author: Mark Brown <broonie@kernel.org> 8 9 #include <linux/debugfs.h> 10 #include <linux/device.h> 11 #include <linux/maple_tree.h> 12 #include <linux/slab.h> 13 14 #include "internal.h" 15 16 static int regcache_maple_read(struct regmap *map, 17 unsigned int reg, unsigned int *value) 18 { 19 struct maple_tree *mt = map->cache; 20 MA_STATE(mas, mt, reg, reg); 21 unsigned long *entry; 22 23 rcu_read_lock(); 24 25 entry = mas_walk(&mas); 26 if (!entry) { 27 rcu_read_unlock(); 28 return -ENOENT; 29 } 30 31 *value = entry[reg - mas.index]; 32 33 rcu_read_unlock(); 34 35 return 0; 36 } 37 38 static int regcache_maple_write(struct regmap *map, unsigned int reg, 39 unsigned int val) 40 { 41 struct maple_tree *mt = map->cache; 42 MA_STATE(mas, mt, reg, reg); 43 unsigned long *entry, *upper, *lower; 44 unsigned long index, last; 45 size_t lower_sz, upper_sz; 46 int ret; 47 48 rcu_read_lock(); 49 50 entry = mas_walk(&mas); 51 if (entry) { 52 entry[reg - mas.index] = val; 53 rcu_read_unlock(); 54 return 0; 55 } 56 57 /* Any adjacent entries to extend/merge? */ 58 mas_set_range(&mas, reg - 1, reg + 1); 59 index = reg; 60 last = reg; 61 62 lower = mas_find(&mas, reg - 1); 63 if (lower) { 64 index = mas.index; 65 lower_sz = (mas.last - mas.index + 1) * sizeof(unsigned long); 66 } 67 68 upper = mas_find(&mas, reg + 1); 69 if (upper) { 70 last = mas.last; 71 upper_sz = (mas.last - mas.index + 1) * sizeof(unsigned long); 72 } 73 74 rcu_read_unlock(); 75 76 entry = kmalloc_array(last - index + 1, sizeof(*entry), map->alloc_flags); 77 if (!entry) 78 return -ENOMEM; 79 80 if (lower) 81 memcpy(entry, lower, lower_sz); 82 entry[reg - index] = val; 83 if (upper) 84 memcpy(&entry[reg - index + 1], upper, upper_sz); 85 86 /* 87 * This is safe because the regmap lock means the Maple lock 88 * is redundant, but we need to take it due to lockdep asserts 89 * in the maple tree code. 90 */ 91 mas_lock(&mas); 92 93 mas_set_range(&mas, index, last); 94 ret = mas_store_gfp(&mas, entry, map->alloc_flags); 95 96 mas_unlock(&mas); 97 98 if (ret) { 99 kfree(entry); 100 return ret; 101 } 102 kfree(lower); 103 kfree(upper); 104 return 0; 105 } 106 107 static int regcache_maple_drop(struct regmap *map, unsigned int min, 108 unsigned int max) 109 { 110 struct maple_tree *mt = map->cache; 111 MA_STATE(mas, mt, min, max); 112 unsigned long *entry, *lower, *upper; 113 /* initialized to work around false-positive -Wuninitialized warning */ 114 unsigned long lower_index = 0, lower_last = 0; 115 unsigned long upper_index = 0, upper_last = 0; 116 int ret = 0; 117 118 lower = NULL; 119 upper = NULL; 120 121 mas_lock(&mas); 122 123 mas_for_each(&mas, entry, max) { 124 /* 125 * This is safe because the regmap lock means the 126 * Maple lock is redundant, but we need to take it due 127 * to lockdep asserts in the maple tree code. 128 */ 129 mas_unlock(&mas); 130 131 /* Do we need to save any of this entry? */ 132 if (mas.index < min) { 133 lower_index = mas.index; 134 lower_last = min -1; 135 136 lower = kmemdup_array(entry, 137 min - mas.index, sizeof(*lower), 138 map->alloc_flags); 139 if (!lower) { 140 ret = -ENOMEM; 141 goto out_unlocked; 142 } 143 } 144 145 if (mas.last > max) { 146 upper_index = max + 1; 147 upper_last = mas.last; 148 149 upper = kmemdup_array(&entry[max - mas.index + 1], 150 mas.last - max, sizeof(*upper), 151 map->alloc_flags); 152 if (!upper) { 153 ret = -ENOMEM; 154 goto out_unlocked; 155 } 156 } 157 158 kfree(entry); 159 mas_lock(&mas); 160 mas_erase(&mas); 161 162 /* Insert new nodes with the saved data */ 163 if (lower) { 164 mas_set_range(&mas, lower_index, lower_last); 165 ret = mas_store_gfp(&mas, lower, map->alloc_flags); 166 if (ret != 0) 167 goto out; 168 lower = NULL; 169 } 170 171 if (upper) { 172 mas_set_range(&mas, upper_index, upper_last); 173 ret = mas_store_gfp(&mas, upper, map->alloc_flags); 174 if (ret != 0) 175 goto out; 176 upper = NULL; 177 } 178 } 179 180 out: 181 mas_unlock(&mas); 182 out_unlocked: 183 kfree(lower); 184 kfree(upper); 185 186 return ret; 187 } 188 189 static int regcache_maple_sync_block(struct regmap *map, unsigned long *entry, 190 struct ma_state *mas, 191 unsigned int min, unsigned int max) 192 { 193 void *buf; 194 unsigned long r; 195 size_t val_bytes = map->format.val_bytes; 196 int ret = 0; 197 198 mas_pause(mas); 199 rcu_read_unlock(); 200 201 /* 202 * Use a raw write if writing more than one register to a 203 * device that supports raw writes to reduce transaction 204 * overheads. 205 */ 206 if (max - min > 1 && regmap_can_raw_write(map)) { 207 buf = kmalloc_array(max - min, val_bytes, map->alloc_flags); 208 if (!buf) { 209 ret = -ENOMEM; 210 goto out; 211 } 212 213 /* Render the data for a raw write */ 214 for (r = min; r < max; r++) { 215 regcache_set_val(map, buf, r - min, 216 entry[r - mas->index]); 217 } 218 219 ret = _regmap_raw_write(map, min, buf, (max - min) * val_bytes, 220 false); 221 222 kfree(buf); 223 } else { 224 for (r = min; r < max; r++) { 225 ret = _regmap_write(map, r, 226 entry[r - mas->index]); 227 if (ret != 0) 228 goto out; 229 } 230 } 231 232 out: 233 rcu_read_lock(); 234 235 return ret; 236 } 237 238 static int regcache_maple_sync(struct regmap *map, unsigned int min, 239 unsigned int max) 240 { 241 struct maple_tree *mt = map->cache; 242 unsigned long *entry; 243 MA_STATE(mas, mt, min, max); 244 unsigned long lmin = min; 245 unsigned long lmax = max; 246 unsigned int r, v, sync_start; 247 int ret = 0; 248 bool sync_needed = false; 249 250 map->cache_bypass = true; 251 252 rcu_read_lock(); 253 254 mas_for_each(&mas, entry, max) { 255 for (r = max(mas.index, lmin); r <= min(mas.last, lmax); r++) { 256 v = entry[r - mas.index]; 257 258 if (regcache_reg_needs_sync(map, r, v)) { 259 if (!sync_needed) { 260 sync_start = r; 261 sync_needed = true; 262 } 263 continue; 264 } 265 266 if (!sync_needed) 267 continue; 268 269 ret = regcache_maple_sync_block(map, entry, &mas, 270 sync_start, r); 271 if (ret != 0) 272 goto out; 273 sync_needed = false; 274 } 275 276 if (sync_needed) { 277 ret = regcache_maple_sync_block(map, entry, &mas, 278 sync_start, r); 279 if (ret != 0) 280 goto out; 281 sync_needed = false; 282 } 283 } 284 285 out: 286 rcu_read_unlock(); 287 288 map->cache_bypass = false; 289 290 return ret; 291 } 292 293 static int regcache_maple_init(struct regmap *map) 294 { 295 struct maple_tree *mt; 296 297 mt = kmalloc_obj(*mt, map->alloc_flags); 298 if (!mt) 299 return -ENOMEM; 300 map->cache = mt; 301 302 mt_init(mt); 303 304 if (!mt_external_lock(mt) && map->lock_key) 305 lockdep_set_class_and_subclass(&mt->ma_lock, map->lock_key, 1); 306 307 return 0; 308 } 309 310 static void regcache_maple_exit(struct regmap *map) 311 { 312 struct maple_tree *mt = map->cache; 313 MA_STATE(mas, mt, 0, UINT_MAX); 314 unsigned int *entry; 315 unsigned long flags; 316 317 /* if we've already been called then just return */ 318 if (!mt) 319 return; 320 321 /* 322 * Register access may take the Maple lock from hard IRQ context, so 323 * preserve its IRQ-safe lockdep state during teardown. 324 */ 325 mas_lock_irqsave(&mas, flags); 326 mas_for_each(&mas, entry, UINT_MAX) 327 kfree(entry); 328 __mt_destroy(mt); 329 mas_unlock_irqrestore(&mas, flags); 330 331 kfree(mt); 332 map->cache = NULL; 333 } 334 335 static int regcache_maple_insert_block(struct regmap *map, int first, 336 int last) 337 { 338 struct maple_tree *mt = map->cache; 339 MA_STATE(mas, mt, first, last); 340 unsigned long *entry; 341 int i, ret; 342 343 entry = kmalloc_array(last - first + 1, sizeof(*entry), map->alloc_flags); 344 if (!entry) 345 return -ENOMEM; 346 347 for (i = 0; i < last - first + 1; i++) 348 entry[i] = map->reg_defaults[first + i].def; 349 350 mas_lock(&mas); 351 352 mas_set_range(&mas, map->reg_defaults[first].reg, 353 map->reg_defaults[last].reg); 354 ret = mas_store_gfp(&mas, entry, map->alloc_flags); 355 356 mas_unlock(&mas); 357 358 if (ret) 359 kfree(entry); 360 361 return ret; 362 } 363 364 static int regcache_maple_populate(struct regmap *map) 365 { 366 int i; 367 int ret; 368 int range_start; 369 370 range_start = 0; 371 372 /* Scan for ranges of contiguous registers */ 373 for (i = 1; i < map->num_reg_defaults; i++) { 374 if (map->reg_defaults[i].reg != 375 map->reg_defaults[i - 1].reg + 1) { 376 ret = regcache_maple_insert_block(map, range_start, 377 i - 1); 378 if (ret != 0) 379 return ret; 380 381 range_start = i; 382 } 383 } 384 385 /* Add the last block */ 386 return regcache_maple_insert_block(map, range_start, map->num_reg_defaults - 1); 387 } 388 389 struct regcache_ops regcache_maple_ops = { 390 .type = REGCACHE_MAPLE, 391 .name = "maple", 392 .init = regcache_maple_init, 393 .exit = regcache_maple_exit, 394 .populate = regcache_maple_populate, 395 .read = regcache_maple_read, 396 .write = regcache_maple_write, 397 .drop = regcache_maple_drop, 398 .sync = regcache_maple_sync, 399 }; 400