1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* -*- linux-c -*- ------------------------------------------------------- * 3 * 4 * Copyright 2002 H. Peter Anvin - All Rights Reserved 5 * 6 * ----------------------------------------------------------------------- */ 7 8 /* 9 * raid6/algos.c 10 * 11 * Algorithm list and algorithm selection for RAID-6 12 */ 13 14 #include <linux/module.h> 15 #include <linux/gfp.h> 16 #include <linux/raid/pq.h> 17 #include <linux/static_call.h> 18 #include <kunit/visibility.h> 19 #include "algos.h" 20 21 #define RAID6_MAX_ALGOS 16 22 static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS]; 23 static unsigned int raid6_nr_algos; 24 static const struct raid6_recov_calls *raid6_recov_algo; 25 26 /* Selected algorithm */ 27 DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome); 28 DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome); 29 DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2); 30 DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap); 31 32 /** 33 * raid6_gen_syndrome - generate RAID6 P/Q parity 34 * @disks: number of "disks" to operate on including parity 35 * @bytes: length in bytes of each vector 36 * @ptrs: @disks size array of memory pointers 37 * 38 * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and 39 * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to 40 * @ptrs[@disks - 3]. 41 * 42 * @disks must be at least 4, and the memory pointed to by each member of @ptrs 43 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of 44 * 512. 45 * 46 * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying 47 * algorithm. 48 */ 49 void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs) 50 { 51 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 52 WARN_ON_ONCE(bytes & 511); 53 WARN_ON_ONCE(disks < RAID6_MIN_DISKS); 54 55 static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs); 56 } 57 EXPORT_SYMBOL_GPL(raid6_gen_syndrome); 58 59 /** 60 * raid6_xor_syndrome - update RAID6 P/Q parity 61 * @disks: number of "disks" to operate on including parity 62 * @start: first index into @disk to update 63 * @stop: last index into @disk to update 64 * @bytes: length in bytes of each vector 65 * @ptrs: @disks size array of memory pointers 66 * 67 * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and 68 * @ptrs[@disks - 1] respectively for the memory pointed to by 69 * @ptrs[@start..@stop]. 70 * 71 * This is used to update parity in place using the following sequence: 72 * 73 * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data. 74 * 2) update the the data in @ptrs[@start..@stop]. 75 * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data. 76 * 77 * Data between @start and @stop that is not changed should be filled 78 * with a pointer to the kernel zero page. 79 * 80 * @disks must be at least 4, and the memory pointed to by each member of @ptrs 81 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of 82 * 512. @stop must be larger or equal to @start. 83 */ 84 void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, 85 void **ptrs) 86 { 87 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 88 WARN_ON_ONCE(bytes & 511); 89 WARN_ON_ONCE(disks < RAID6_MIN_DISKS); 90 WARN_ON_ONCE(stop < start); 91 92 static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs); 93 } 94 EXPORT_SYMBOL_GPL(raid6_xor_syndrome); 95 96 /* 97 * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used 98 * 99 * Returns %true if raid6_can_xor_syndrome() can be used, else %false. 100 */ 101 bool raid6_can_xor_syndrome(void) 102 { 103 return !!static_call_query(raid6_xor_syndrome_impl); 104 } 105 EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome); 106 107 /** 108 * raid6_recov_2data - recover two missing data disks 109 * @disks: number of "disks" to operate on including parity 110 * @bytes: length in bytes of each vector 111 * @faila: first failed data disk index 112 * @failb: second failed data disk index 113 * @ptrs: @disks size array of memory pointers 114 * 115 * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the 116 * data in the remaining disks and the two parities pointed to by the other 117 * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks 118 * and the two parities. @faila must be smaller than @failb. 119 * 120 * Memory pointed to by each pointer in @ptrs must be page aligned and is 121 * limited to %PAGE_SIZE. 122 */ 123 void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, 124 void **ptrs) 125 { 126 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 127 WARN_ON_ONCE(bytes & 511); 128 WARN_ON_ONCE(bytes > PAGE_SIZE); 129 WARN_ON_ONCE(failb <= faila); 130 131 static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs); 132 } 133 EXPORT_SYMBOL_GPL(raid6_recov_2data); 134 135 /** 136 * raid6_recov_datap - recover a missing data disk and missing P-parity 137 * @disks: number of "disks" to operate on including parity 138 * @bytes: length in bytes of each vector 139 * @faila: failed data disk index 140 * @ptrs: @disks size array of memory pointers 141 * 142 * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in 143 * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity 144 * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks 145 * includes the data disks and the two parities. 146 * 147 * Memory pointed to by each pointer in @ptrs must be page aligned and is 148 * limited to %PAGE_SIZE. 149 */ 150 void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs) 151 { 152 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 153 WARN_ON_ONCE(bytes & 511); 154 WARN_ON_ONCE(bytes > PAGE_SIZE); 155 156 static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs); 157 } 158 EXPORT_SYMBOL_GPL(raid6_recov_datap); 159 160 #define RAID6_TIME_JIFFIES_LG2 4 161 #define RAID6_TEST_DISKS 8 162 #define RAID6_TEST_DISKS_ORDER 3 163 164 static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS], 165 const int disks) 166 { 167 /* work on the second half of the disks */ 168 int start = (disks >> 1) - 1, stop = disks - 3; 169 const struct raid6_calls *best = NULL; 170 unsigned long bestgenperf = 0; 171 unsigned int i; 172 173 for (i = 0; i < raid6_nr_algos; i++) { 174 const struct raid6_calls *algo = raid6_algos[i]; 175 unsigned long perf = 0, j0, j1; 176 177 preempt_disable(); 178 j0 = jiffies; 179 while ((j1 = jiffies) == j0) 180 cpu_relax(); 181 while (time_before(jiffies, 182 j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { 183 algo->gen_syndrome(disks, PAGE_SIZE, *dptrs); 184 perf++; 185 } 186 preempt_enable(); 187 188 if (perf > bestgenperf) { 189 bestgenperf = perf; 190 best = algo; 191 } 192 pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name, 193 (perf * HZ * (disks-2)) >> 194 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); 195 } 196 197 if (!best) { 198 pr_err("raid6: Yikes! No algorithm found!\n"); 199 return -EINVAL; 200 } 201 202 static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome); 203 static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); 204 205 pr_info("raid6: using algorithm %s gen() %ld MB/s\n", 206 best->name, 207 (bestgenperf * HZ * (disks - 2)) >> 208 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); 209 210 if (best->xor_syndrome) { 211 unsigned long perf = 0, j0, j1; 212 213 preempt_disable(); 214 j0 = jiffies; 215 while ((j1 = jiffies) == j0) 216 cpu_relax(); 217 while (time_before(jiffies, 218 j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { 219 best->xor_syndrome(disks, start, stop, 220 PAGE_SIZE, *dptrs); 221 perf++; 222 } 223 preempt_enable(); 224 225 pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", 226 (perf * HZ * (disks - 2)) >> 227 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); 228 } 229 230 return 0; 231 } 232 233 234 /* Try to pick the best algorithm */ 235 /* This code uses the gfmul table as convenient data set to abuse */ 236 237 static int __init raid6_select_algo(void) 238 { 239 const int disks = RAID6_TEST_DISKS; 240 char *disk_ptr, *p; 241 void *dptrs[RAID6_TEST_DISKS]; 242 int i, cycle; 243 int error; 244 245 if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) { 246 pr_info("raid6: skipped pq benchmark and selected %s\n", 247 raid6_algos[raid6_nr_algos - 1]->name); 248 static_call_update(raid6_gen_syndrome_impl, 249 raid6_algos[raid6_nr_algos - 1]->gen_syndrome); 250 static_call_update(raid6_xor_syndrome_impl, 251 raid6_algos[raid6_nr_algos - 1]->xor_syndrome); 252 return 0; 253 } 254 255 /* prepare the buffer and fill it circularly with gfmul table */ 256 disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); 257 if (!disk_ptr) { 258 pr_err("raid6: Yikes! No memory available.\n"); 259 return -ENOMEM; 260 } 261 262 p = disk_ptr; 263 for (i = 0; i < disks; i++) 264 dptrs[i] = p + PAGE_SIZE * i; 265 266 cycle = ((disks - 2) * PAGE_SIZE) / 65536; 267 for (i = 0; i < cycle; i++) { 268 memcpy(p, raid6_gfmul, 65536); 269 p += 65536; 270 } 271 272 if ((disks - 2) * PAGE_SIZE % 65536) 273 memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); 274 275 /* select raid gen_syndrome function */ 276 error = raid6_choose_gen(&dptrs, disks); 277 278 free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); 279 280 return error; 281 } 282 283 /* 284 * Register a RAID6 P/Q generation algorithm. The most optimized/unrolled 285 * implementation should be registered last so it will be selected when the 286 * boot-time benchmark is disabled. 287 */ 288 void __init raid6_algo_add(const struct raid6_calls *algo) 289 { 290 if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS)) 291 return; 292 raid6_algos[raid6_nr_algos++] = algo; 293 } 294 295 void __init raid6_algo_add_default(void) 296 { 297 raid6_algo_add(&raid6_intx1); 298 raid6_algo_add(&raid6_intx2); 299 raid6_algo_add(&raid6_intx4); 300 raid6_algo_add(&raid6_intx8); 301 } 302 303 void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo) 304 { 305 if (WARN_ON_ONCE(raid6_recov_algo)) 306 return; 307 raid6_recov_algo = algo; 308 } 309 310 #ifdef CONFIG_RAID6_PQ_ARCH 311 #include "pq_arch.h" 312 #else 313 static inline void arch_raid6_init(void) 314 { 315 raid6_algo_add_default(); 316 } 317 #endif /* CONFIG_RAID6_PQ_ARCH */ 318 319 static int __init raid6_init(void) 320 { 321 /* 322 * Architectures providing arch_raid6_init must add all PQ generation 323 * algorithms they want to consider in arch_raid6_init(), including 324 * the generic ones using raid6_algo_add_default() if wanted. 325 */ 326 arch_raid6_init(); 327 328 /* 329 * Architectures don't have to set a recovery algorithm, we'll just pick 330 * the generic integer one if none was set. 331 */ 332 if (!raid6_recov_algo) 333 raid6_recov_algo = &raid6_recov_intx1; 334 static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2); 335 static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap); 336 pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); 337 338 return raid6_select_algo(); 339 } 340 341 static void __exit raid6_exit(void) 342 { 343 } 344 345 subsys_initcall(raid6_init); 346 module_exit(raid6_exit); 347 MODULE_LICENSE("GPL"); 348 MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); 349 350 #if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST) 351 const struct raid6_calls *raid6_algo_find(unsigned int idx) 352 { 353 if (idx >= raid6_nr_algos) { 354 /* 355 * Always include the simplest generic integer implementation in 356 * the unit tests as a baseline. 357 */ 358 if (idx == raid6_nr_algos && 359 raid6_algos[0] != &raid6_intx1) 360 return &raid6_intx1; 361 return NULL; 362 } 363 return raid6_algos[idx]; 364 } 365 EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find); 366 367 const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx) 368 { 369 switch (idx) { 370 case 0: 371 /* always test the generic integer implementation */ 372 return &raid6_recov_intx1; 373 case 1: 374 /* test the optimized implementation if there is one */ 375 if (raid6_recov_algo != &raid6_recov_intx1) 376 return raid6_recov_algo; 377 return NULL; 378 default: 379 return NULL; 380 } 381 } 382 EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find); 383 #endif /* CONFIG_RAID6_PQ_KUNIT_TEST */ 384