1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* -*- linux-c -*- ------------------------------------------------------- * 3 * 4 * Copyright 2002 H. Peter Anvin - All Rights Reserved 5 * 6 * ----------------------------------------------------------------------- */ 7 8 /* 9 * raid6/algos.c 10 * 11 * Algorithm list and algorithm selection for RAID-6 12 */ 13 14 #include <linux/module.h> 15 #include <linux/gfp.h> 16 #include <linux/raid/pq.h> 17 #include <linux/static_call.h> 18 #include <kunit/visibility.h> 19 #include "algos.h" 20 21 #define RAID6_MAX_ALGOS 16 22 static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS]; 23 static unsigned int raid6_nr_algos; 24 static const struct raid6_recov_calls *raid6_recov_algo; 25 26 /* Selected algorithm */ 27 DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome); 28 DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome); 29 30 /** 31 * raid6_gen_syndrome - generate RAID6 P/Q parity 32 * @disks: number of "disks" to operate on including parity 33 * @bytes: length in bytes of each vector 34 * @ptrs: @disks size array of memory pointers 35 * 36 * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and 37 * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to 38 * @ptrs[@disks - 3]. 39 * 40 * @disks must be at least 4, and the memory pointed to by each member of @ptrs 41 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of 42 * 512. 43 * 44 * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying 45 * algorithm. 46 */ 47 void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs) 48 { 49 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 50 WARN_ON_ONCE(bytes & 511); 51 WARN_ON_ONCE(disks < RAID6_MIN_DISKS); 52 53 static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs); 54 } 55 EXPORT_SYMBOL_GPL(raid6_gen_syndrome); 56 57 /** 58 * raid6_xor_syndrome - update RAID6 P/Q parity 59 * @disks: number of "disks" to operate on including parity 60 * @start: first index into @disk to update 61 * @stop: last index into @disk to update 62 * @bytes: length in bytes of each vector 63 * @ptrs: @disks size array of memory pointers 64 * 65 * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and 66 * @ptrs[@disks - 1] respectively for the memory pointed to by 67 * @ptrs[@start..@stop]. 68 * 69 * This is used to update parity in place using the following sequence: 70 * 71 * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data. 72 * 2) update the the data in @ptrs[@start..@stop]. 73 * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data. 74 * 75 * Data between @start and @stop that is not changed should be filled 76 * with a pointer to the kernel zero page. 77 * 78 * @disks must be at least 4, and the memory pointed to by each member of @ptrs 79 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of 80 * 512. @stop must be larger or equal to @start. 81 */ 82 void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, 83 void **ptrs) 84 { 85 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 86 WARN_ON_ONCE(bytes & 511); 87 WARN_ON_ONCE(disks < RAID6_MIN_DISKS); 88 WARN_ON_ONCE(stop < start); 89 90 static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs); 91 } 92 EXPORT_SYMBOL_GPL(raid6_xor_syndrome); 93 94 /* 95 * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used 96 * 97 * Returns %true if raid6_can_xor_syndrome() can be used, else %false. 98 */ 99 bool raid6_can_xor_syndrome(void) 100 { 101 return !!static_call_query(raid6_xor_syndrome_impl); 102 } 103 EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome); 104 105 /** 106 * raid6_recov_2data - recover two missing data disks 107 * @disks: number of "disks" to operate on including parity 108 * @bytes: length in bytes of each vector 109 * @faila: first failed data disk index 110 * @failb: second failed data disk index 111 * @ptrs: @disks size array of memory pointers 112 * 113 * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the 114 * data in the remaining disks and the two parities pointed to by the other 115 * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks 116 * and the two parities. @faila must be smaller than @failb. 117 * 118 * Memory pointed to by each pointer in @ptrs must be page aligned and is 119 * limited to %PAGE_SIZE. 120 */ 121 void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, 122 void **ptrs) 123 { 124 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 125 WARN_ON_ONCE(bytes & 511); 126 WARN_ON_ONCE(bytes > PAGE_SIZE); 127 WARN_ON_ONCE(failb <= faila); 128 129 raid6_recov_algo->data2(disks, bytes, faila, failb, ptrs); 130 } 131 EXPORT_SYMBOL_GPL(raid6_recov_2data); 132 133 /** 134 * raid6_recov_datap - recover a missing data disk and missing P-parity 135 * @disks: number of "disks" to operate on including parity 136 * @bytes: length in bytes of each vector 137 * @faila: failed data disk index 138 * @ptrs: @disks size array of memory pointers 139 * 140 * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in 141 * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity 142 * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks 143 * includes the data disks and the two parities. 144 * 145 * Memory pointed to by each pointer in @ptrs must be page aligned and is 146 * limited to %PAGE_SIZE. 147 */ 148 void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs) 149 { 150 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 151 WARN_ON_ONCE(bytes & 511); 152 WARN_ON_ONCE(bytes > PAGE_SIZE); 153 154 raid6_recov_algo->datap(disks, bytes, faila, ptrs); 155 } 156 EXPORT_SYMBOL_GPL(raid6_recov_datap); 157 158 #define RAID6_TIME_JIFFIES_LG2 4 159 #define RAID6_TEST_DISKS 8 160 #define RAID6_TEST_DISKS_ORDER 3 161 162 static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS], 163 const int disks) 164 { 165 /* work on the second half of the disks */ 166 int start = (disks >> 1) - 1, stop = disks - 3; 167 const struct raid6_calls *best = NULL; 168 unsigned long bestgenperf = 0; 169 unsigned int i; 170 171 for (i = 0; i < raid6_nr_algos; i++) { 172 const struct raid6_calls *algo = raid6_algos[i]; 173 unsigned long perf = 0, j0, j1; 174 175 preempt_disable(); 176 j0 = jiffies; 177 while ((j1 = jiffies) == j0) 178 cpu_relax(); 179 while (time_before(jiffies, 180 j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { 181 algo->gen_syndrome(disks, PAGE_SIZE, *dptrs); 182 perf++; 183 } 184 preempt_enable(); 185 186 if (perf > bestgenperf) { 187 bestgenperf = perf; 188 best = algo; 189 } 190 pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name, 191 (perf * HZ * (disks-2)) >> 192 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); 193 } 194 195 if (!best) { 196 pr_err("raid6: Yikes! No algorithm found!\n"); 197 return -EINVAL; 198 } 199 200 static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome); 201 static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); 202 203 pr_info("raid6: using algorithm %s gen() %ld MB/s\n", 204 best->name, 205 (bestgenperf * HZ * (disks - 2)) >> 206 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); 207 208 if (best->xor_syndrome) { 209 unsigned long perf = 0, j0, j1; 210 211 preempt_disable(); 212 j0 = jiffies; 213 while ((j1 = jiffies) == j0) 214 cpu_relax(); 215 while (time_before(jiffies, 216 j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { 217 best->xor_syndrome(disks, start, stop, 218 PAGE_SIZE, *dptrs); 219 perf++; 220 } 221 preempt_enable(); 222 223 pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", 224 (perf * HZ * (disks - 2)) >> 225 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); 226 } 227 228 return 0; 229 } 230 231 232 /* Try to pick the best algorithm */ 233 /* This code uses the gfmul table as convenient data set to abuse */ 234 235 static int __init raid6_select_algo(void) 236 { 237 const int disks = RAID6_TEST_DISKS; 238 char *disk_ptr, *p; 239 void *dptrs[RAID6_TEST_DISKS]; 240 int i, cycle; 241 int error; 242 243 if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) { 244 pr_info("raid6: skipped pq benchmark and selected %s\n", 245 raid6_algos[raid6_nr_algos - 1]->name); 246 static_call_update(raid6_gen_syndrome_impl, 247 raid6_algos[raid6_nr_algos - 1]->gen_syndrome); 248 static_call_update(raid6_xor_syndrome_impl, 249 raid6_algos[raid6_nr_algos - 1]->xor_syndrome); 250 return 0; 251 } 252 253 /* prepare the buffer and fill it circularly with gfmul table */ 254 disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); 255 if (!disk_ptr) { 256 pr_err("raid6: Yikes! No memory available.\n"); 257 return -ENOMEM; 258 } 259 260 p = disk_ptr; 261 for (i = 0; i < disks; i++) 262 dptrs[i] = p + PAGE_SIZE * i; 263 264 cycle = ((disks - 2) * PAGE_SIZE) / 65536; 265 for (i = 0; i < cycle; i++) { 266 memcpy(p, raid6_gfmul, 65536); 267 p += 65536; 268 } 269 270 if ((disks - 2) * PAGE_SIZE % 65536) 271 memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); 272 273 /* select raid gen_syndrome function */ 274 error = raid6_choose_gen(&dptrs, disks); 275 276 free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); 277 278 return error; 279 } 280 281 /* 282 * Register a RAID6 P/Q generation algorithm. The most optimized/unrolled 283 * implementation should be registered last so it will be selected when the 284 * boot-time benchmark is disabled. 285 */ 286 void __init raid6_algo_add(const struct raid6_calls *algo) 287 { 288 if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS)) 289 return; 290 raid6_algos[raid6_nr_algos++] = algo; 291 } 292 293 void __init raid6_algo_add_default(void) 294 { 295 raid6_algo_add(&raid6_intx1); 296 raid6_algo_add(&raid6_intx2); 297 raid6_algo_add(&raid6_intx4); 298 raid6_algo_add(&raid6_intx8); 299 } 300 301 void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo) 302 { 303 if (WARN_ON_ONCE(raid6_recov_algo)) 304 return; 305 raid6_recov_algo = algo; 306 } 307 308 #ifdef CONFIG_RAID6_PQ_ARCH 309 #include "pq_arch.h" 310 #else 311 static inline void arch_raid6_init(void) 312 { 313 raid6_algo_add_default(); 314 } 315 #endif /* CONFIG_RAID6_PQ_ARCH */ 316 317 static int __init raid6_init(void) 318 { 319 /* 320 * Architectures providing arch_raid6_init must add all PQ generation 321 * algorithms they want to consider in arch_raid6_init(), including 322 * the generic ones using raid6_algo_add_default() if wanted. 323 */ 324 arch_raid6_init(); 325 326 /* 327 * Architectures don't have to set a recovery algorithm, we'll just pick 328 * the generic integer one if none was set. 329 */ 330 if (!raid6_recov_algo) 331 raid6_recov_algo = &raid6_recov_intx1; 332 pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); 333 334 return raid6_select_algo(); 335 } 336 337 static void __exit raid6_exit(void) 338 { 339 } 340 341 subsys_initcall(raid6_init); 342 module_exit(raid6_exit); 343 MODULE_LICENSE("GPL"); 344 MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); 345 346 #if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST) 347 const struct raid6_calls *raid6_algo_find(unsigned int idx) 348 { 349 if (idx >= raid6_nr_algos) { 350 /* 351 * Always include the simplest generic integer implementation in 352 * the unit tests as a baseline. 353 */ 354 if (idx == raid6_nr_algos && 355 raid6_algos[0] != &raid6_intx1) 356 return &raid6_intx1; 357 return NULL; 358 } 359 return raid6_algos[idx]; 360 } 361 EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find); 362 363 const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx) 364 { 365 switch (idx) { 366 case 0: 367 /* always test the generic integer implementation */ 368 return &raid6_recov_intx1; 369 case 1: 370 /* test the optimized implementation if there is one */ 371 if (raid6_recov_algo != &raid6_recov_intx1) 372 return raid6_recov_algo; 373 return NULL; 374 default: 375 return NULL; 376 } 377 } 378 EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find); 379 #endif /* CONFIG_RAID6_PQ_KUNIT_TEST */ 380