1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2002 H. Peter Anvin - All Rights Reserved 4 * 5 * Algorithm list and algorithm selection for RAID-6 6 */ 7 8 #include <linux/module.h> 9 #include <linux/gfp.h> 10 #include <linux/raid/pq.h> 11 #include <linux/slab.h> 12 #include <linux/static_call.h> 13 #include <kunit/visibility.h> 14 #include "algos.h" 15 16 #define RAID6_MAX_ALGOS 16 17 static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS]; 18 static unsigned int raid6_nr_algos; 19 static const struct raid6_recov_calls *raid6_recov_algo; 20 21 /* Selected algorithm */ 22 DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome); 23 DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome); 24 DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2); 25 DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap); 26 27 /** 28 * raid6_gen_syndrome - generate RAID6 P/Q parity 29 * @disks: number of "disks" to operate on including parity 30 * @bytes: length in bytes of each vector 31 * @ptrs: @disks size array of memory pointers 32 * 33 * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and 34 * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to 35 * @ptrs[@disks - 3]. 36 * 37 * @disks must be at least 4, and the memory pointed to by each member of @ptrs 38 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of 39 * 512. 40 * 41 * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying 42 * algorithm. 43 */ 44 void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs) 45 { 46 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 47 WARN_ON_ONCE(bytes & 511); 48 WARN_ON_ONCE(disks < RAID6_MIN_DISKS); 49 50 static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs); 51 } 52 EXPORT_SYMBOL_GPL(raid6_gen_syndrome); 53 54 /** 55 * raid6_xor_syndrome - update RAID6 P/Q parity 56 * @disks: number of "disks" to operate on including parity 57 * @start: first index into @disk to update 58 * @stop: last index into @disk to update 59 * @bytes: length in bytes of each vector 60 * @ptrs: @disks size array of memory pointers 61 * 62 * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and 63 * @ptrs[@disks - 1] respectively for the memory pointed to by 64 * @ptrs[@start..@stop]. 65 * 66 * This is used to update parity in place using the following sequence: 67 * 68 * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data. 69 * 2) update the the data in @ptrs[@start..@stop]. 70 * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data. 71 * 72 * Data between @start and @stop that is not changed should be filled 73 * with a pointer to the kernel zero page. 74 * 75 * @disks must be at least 4, and the memory pointed to by each member of @ptrs 76 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of 77 * 512. @stop must be larger or equal to @start. 78 */ 79 void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, 80 void **ptrs) 81 { 82 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 83 WARN_ON_ONCE(bytes & 511); 84 WARN_ON_ONCE(disks < RAID6_MIN_DISKS); 85 WARN_ON_ONCE(stop < start); 86 87 static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs); 88 } 89 EXPORT_SYMBOL_GPL(raid6_xor_syndrome); 90 91 /* 92 * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used 93 * 94 * Returns %true if raid6_can_xor_syndrome() can be used, else %false. 95 */ 96 bool raid6_can_xor_syndrome(void) 97 { 98 return !!static_call_query(raid6_xor_syndrome_impl); 99 } 100 EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome); 101 102 /** 103 * raid6_recov_2data - recover two missing data disks 104 * @disks: number of "disks" to operate on including parity 105 * @bytes: length in bytes of each vector 106 * @faila: first failed data disk index 107 * @failb: second failed data disk index 108 * @ptrs: @disks size array of memory pointers 109 * 110 * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the 111 * data in the remaining disks and the two parities pointed to by the other 112 * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks 113 * and the two parities. @faila must be smaller than @failb. 114 * 115 * Memory pointed to by each pointer in @ptrs must be page aligned and is 116 * limited to %PAGE_SIZE. 117 */ 118 void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, 119 void **ptrs) 120 { 121 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 122 WARN_ON_ONCE(bytes & 511); 123 WARN_ON_ONCE(bytes > PAGE_SIZE); 124 WARN_ON_ONCE(failb <= faila); 125 126 static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs); 127 } 128 EXPORT_SYMBOL_GPL(raid6_recov_2data); 129 130 /** 131 * raid6_recov_datap - recover a missing data disk and missing P-parity 132 * @disks: number of "disks" to operate on including parity 133 * @bytes: length in bytes of each vector 134 * @faila: failed data disk index 135 * @ptrs: @disks size array of memory pointers 136 * 137 * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in 138 * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity 139 * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks 140 * includes the data disks and the two parities. 141 * 142 * Memory pointed to by each pointer in @ptrs must be page aligned and is 143 * limited to %PAGE_SIZE. 144 */ 145 void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs) 146 { 147 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 148 WARN_ON_ONCE(bytes & 511); 149 WARN_ON_ONCE(bytes > PAGE_SIZE); 150 151 static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs); 152 } 153 EXPORT_SYMBOL_GPL(raid6_recov_datap); 154 155 #define BENCH_SIZE SZ_4K 156 #define NR_SRCS 8 157 #define NR_DISKS (NR_SRCS + 2) 158 #define REPS 800U 159 160 static int raid6_choose_gen(void *dptrs[NR_DISKS], const int disks) 161 { 162 const struct raid6_calls *best = NULL; 163 unsigned long bestgenperf = 0; 164 unsigned int i; 165 166 for (i = 0; i < raid6_nr_algos; i++) { 167 const struct raid6_calls *algo = raid6_algos[i]; 168 unsigned long perf = 0; 169 u64 t; 170 int i; 171 172 preempt_disable(); 173 t = ktime_get_ns(); 174 for (i = 0; i < REPS; i++) 175 algo->gen_syndrome(disks, BENCH_SIZE, dptrs); 176 t = max(ktime_get_ns() - t, 1); 177 preempt_enable(); 178 179 /* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */ 180 perf = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t); 181 if (perf > bestgenperf) { 182 bestgenperf = perf; 183 best = algo; 184 } 185 pr_info("raid6: %-8s gen() %5lu MB/s\n", algo->name, perf); 186 } 187 188 if (!best) { 189 pr_err("raid6: Yikes! No algorithm found!\n"); 190 return -EINVAL; 191 } 192 193 static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome); 194 static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); 195 196 pr_info("raid6: using algorithm %s gen() %ld MB/s\n", 197 best->name, bestgenperf); 198 199 if (best->xor_syndrome) { 200 /* work on the second half of the disks */ 201 int start = (disks / 2) - 1, stop = disks - 3; 202 u64 t; 203 204 preempt_disable(); 205 t = ktime_get_ns(); 206 for (i = 0; i < REPS; i++) 207 best->xor_syndrome(disks, start, stop, BENCH_SIZE, 208 dptrs); 209 t = max(ktime_get_ns() - t, 1); 210 preempt_enable(); 211 212 pr_info("raid6: .... xor() %llu MB/s, rmw enabled\n", 213 div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS / 2 * 1000, 214 t)); 215 } 216 217 return 0; 218 } 219 220 221 /* Try to pick the best algorithm */ 222 /* This code uses the gfmul table as convenient data set to abuse */ 223 224 static int __init raid6_select_algo(void) 225 { 226 const int disks = NR_DISKS; 227 void *dptrs[NR_DISKS]; 228 char *disk_ptr, *p; 229 int i, cycle; 230 int error; 231 232 if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) { 233 pr_info("raid6: skipped pq benchmark and selected %s\n", 234 raid6_algos[raid6_nr_algos - 1]->name); 235 return 0; 236 } 237 238 /* prepare the buffer and fill it circularly with gfmul table */ 239 disk_ptr = kmalloc_array(NR_DISKS, BENCH_SIZE, GFP_KERNEL); 240 if (!disk_ptr) { 241 pr_err("raid6: Yikes! No memory available.\n"); 242 return -ENOMEM; 243 } 244 245 p = disk_ptr; 246 for (i = 0; i < disks; i++) 247 dptrs[i] = p + BENCH_SIZE * i; 248 249 cycle = ((disks - 2) * BENCH_SIZE) / 65536; 250 for (i = 0; i < cycle; i++) { 251 memcpy(p, raid6_gfmul, 65536); 252 p += 65536; 253 } 254 255 if ((disks - 2) * BENCH_SIZE % 65536) 256 memcpy(p, raid6_gfmul, (disks - 2) * BENCH_SIZE % 65536); 257 258 /* select raid gen_syndrome function */ 259 error = raid6_choose_gen(dptrs, disks); 260 261 kfree(disk_ptr); 262 263 return error; 264 } 265 266 /* 267 * Register a RAID6 P/Q generation algorithm. The most optimized/unrolled 268 * implementation should be registered last so it will be selected when the 269 * boot-time benchmark is disabled. 270 */ 271 void __init raid6_algo_add(const struct raid6_calls *algo) 272 { 273 if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS)) 274 return; 275 raid6_algos[raid6_nr_algos++] = algo; 276 } 277 278 void __init raid6_algo_add_default(void) 279 { 280 raid6_algo_add(&raid6_intx1); 281 raid6_algo_add(&raid6_intx2); 282 raid6_algo_add(&raid6_intx4); 283 raid6_algo_add(&raid6_intx8); 284 } 285 286 void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo) 287 { 288 if (WARN_ON_ONCE(raid6_recov_algo)) 289 return; 290 raid6_recov_algo = algo; 291 } 292 293 #ifdef CONFIG_RAID6_PQ_ARCH 294 #include "pq_arch.h" 295 #else 296 static inline void arch_raid6_init(void) 297 { 298 raid6_algo_add_default(); 299 } 300 #endif /* CONFIG_RAID6_PQ_ARCH */ 301 302 static int __init raid6_init(void) 303 { 304 /* 305 * Architectures providing arch_raid6_init must add all PQ generation 306 * algorithms they want to consider in arch_raid6_init(), including 307 * the generic ones using raid6_algo_add_default() if wanted. 308 */ 309 arch_raid6_init(); 310 311 /* 312 * Architectures don't have to set a recovery algorithm, we'll just pick 313 * the generic integer one if none was set. 314 */ 315 if (!raid6_recov_algo) 316 raid6_recov_algo = &raid6_recov_intx1; 317 static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2); 318 static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap); 319 pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); 320 321 /* 322 * Pick the last registered implementation as the temporary default until 323 * calibration happens. 324 */ 325 static_call_update(raid6_gen_syndrome_impl, 326 raid6_algos[raid6_nr_algos - 1]->gen_syndrome); 327 static_call_update(raid6_xor_syndrome_impl, 328 raid6_algos[raid6_nr_algos - 1]->xor_syndrome); 329 330 #ifdef MODULE 331 return raid6_select_algo(); 332 #else 333 return 0; 334 #endif 335 } 336 337 static void __exit raid6_exit(void) 338 { 339 } 340 341 /* 342 * When built-in we must register the default implementation before md 343 * initializes, but we don't want calibration to run that early as that 344 * would delay the boot process. 345 */ 346 #ifndef MODULE 347 device_initcall(raid6_select_algo); 348 #endif 349 subsys_initcall(raid6_init); 350 module_exit(raid6_exit); 351 MODULE_LICENSE("GPL"); 352 MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); 353 354 #if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST) 355 const struct raid6_calls *raid6_algo_find(unsigned int idx) 356 { 357 if (idx >= raid6_nr_algos) { 358 /* 359 * Always include the simplest generic integer implementation in 360 * the unit tests as a baseline. 361 */ 362 if (idx == raid6_nr_algos && 363 raid6_algos[0] != &raid6_intx1) 364 return &raid6_intx1; 365 return NULL; 366 } 367 return raid6_algos[idx]; 368 } 369 EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find); 370 371 const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx) 372 { 373 switch (idx) { 374 case 0: 375 /* always test the generic integer implementation */ 376 return &raid6_recov_intx1; 377 case 1: 378 /* test the optimized implementation if there is one */ 379 if (raid6_recov_algo != &raid6_recov_intx1) 380 return raid6_recov_algo; 381 return NULL; 382 default: 383 return NULL; 384 } 385 } 386 EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find); 387 #endif /* CONFIG_RAID6_PQ_KUNIT_TEST */ 388