1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright 2002 H. Peter Anvin - All Rights Reserved 4 * 5 * Algorithm list and algorithm selection for RAID-6 6 */ 7 8 #include <linux/module.h> 9 #include <linux/gfp.h> 10 #include <linux/raid/pq.h> 11 #include <linux/static_call.h> 12 #include <kunit/visibility.h> 13 #include "algos.h" 14 15 #define RAID6_MAX_ALGOS 16 16 static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS]; 17 static unsigned int raid6_nr_algos; 18 static const struct raid6_recov_calls *raid6_recov_algo; 19 20 /* Selected algorithm */ 21 DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome); 22 DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome); 23 DEFINE_STATIC_CALL_NULL(raid6_recov_2data_impl, *raid6_recov_intx1.data2); 24 DEFINE_STATIC_CALL_NULL(raid6_recov_datap_impl, *raid6_recov_intx1.datap); 25 26 /** 27 * raid6_gen_syndrome - generate RAID6 P/Q parity 28 * @disks: number of "disks" to operate on including parity 29 * @bytes: length in bytes of each vector 30 * @ptrs: @disks size array of memory pointers 31 * 32 * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and 33 * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to 34 * @ptrs[@disks - 3]. 35 * 36 * @disks must be at least 4, and the memory pointed to by each member of @ptrs 37 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of 38 * 512. 39 * 40 * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying 41 * algorithm. 42 */ 43 void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs) 44 { 45 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 46 WARN_ON_ONCE(bytes & 511); 47 WARN_ON_ONCE(disks < RAID6_MIN_DISKS); 48 49 static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs); 50 } 51 EXPORT_SYMBOL_GPL(raid6_gen_syndrome); 52 53 /** 54 * raid6_xor_syndrome - update RAID6 P/Q parity 55 * @disks: number of "disks" to operate on including parity 56 * @start: first index into @disk to update 57 * @stop: last index into @disk to update 58 * @bytes: length in bytes of each vector 59 * @ptrs: @disks size array of memory pointers 60 * 61 * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and 62 * @ptrs[@disks - 1] respectively for the memory pointed to by 63 * @ptrs[@start..@stop]. 64 * 65 * This is used to update parity in place using the following sequence: 66 * 67 * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data. 68 * 2) update the the data in @ptrs[@start..@stop]. 69 * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data. 70 * 71 * Data between @start and @stop that is not changed should be filled 72 * with a pointer to the kernel zero page. 73 * 74 * @disks must be at least 4, and the memory pointed to by each member of @ptrs 75 * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of 76 * 512. @stop must be larger or equal to @start. 77 */ 78 void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, 79 void **ptrs) 80 { 81 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 82 WARN_ON_ONCE(bytes & 511); 83 WARN_ON_ONCE(disks < RAID6_MIN_DISKS); 84 WARN_ON_ONCE(stop < start); 85 86 static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs); 87 } 88 EXPORT_SYMBOL_GPL(raid6_xor_syndrome); 89 90 /* 91 * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used 92 * 93 * Returns %true if raid6_can_xor_syndrome() can be used, else %false. 94 */ 95 bool raid6_can_xor_syndrome(void) 96 { 97 return !!static_call_query(raid6_xor_syndrome_impl); 98 } 99 EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome); 100 101 /** 102 * raid6_recov_2data - recover two missing data disks 103 * @disks: number of "disks" to operate on including parity 104 * @bytes: length in bytes of each vector 105 * @faila: first failed data disk index 106 * @failb: second failed data disk index 107 * @ptrs: @disks size array of memory pointers 108 * 109 * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the 110 * data in the remaining disks and the two parities pointed to by the other 111 * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks 112 * and the two parities. @faila must be smaller than @failb. 113 * 114 * Memory pointed to by each pointer in @ptrs must be page aligned and is 115 * limited to %PAGE_SIZE. 116 */ 117 void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, 118 void **ptrs) 119 { 120 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 121 WARN_ON_ONCE(bytes & 511); 122 WARN_ON_ONCE(bytes > PAGE_SIZE); 123 WARN_ON_ONCE(failb <= faila); 124 125 static_call(raid6_recov_2data_impl)(disks, bytes, faila, failb, ptrs); 126 } 127 EXPORT_SYMBOL_GPL(raid6_recov_2data); 128 129 /** 130 * raid6_recov_datap - recover a missing data disk and missing P-parity 131 * @disks: number of "disks" to operate on including parity 132 * @bytes: length in bytes of each vector 133 * @faila: failed data disk index 134 * @ptrs: @disks size array of memory pointers 135 * 136 * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in 137 * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity 138 * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks 139 * includes the data disks and the two parities. 140 * 141 * Memory pointed to by each pointer in @ptrs must be page aligned and is 142 * limited to %PAGE_SIZE. 143 */ 144 void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs) 145 { 146 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 147 WARN_ON_ONCE(bytes & 511); 148 WARN_ON_ONCE(bytes > PAGE_SIZE); 149 150 static_call(raid6_recov_datap_impl)(disks, bytes, faila, ptrs); 151 } 152 EXPORT_SYMBOL_GPL(raid6_recov_datap); 153 154 #define RAID6_TIME_JIFFIES_LG2 4 155 #define RAID6_TEST_DISKS 8 156 #define RAID6_TEST_DISKS_ORDER 3 157 158 static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS], 159 const int disks) 160 { 161 /* work on the second half of the disks */ 162 int start = (disks >> 1) - 1, stop = disks - 3; 163 const struct raid6_calls *best = NULL; 164 unsigned long bestgenperf = 0; 165 unsigned int i; 166 167 for (i = 0; i < raid6_nr_algos; i++) { 168 const struct raid6_calls *algo = raid6_algos[i]; 169 unsigned long perf = 0, j0, j1; 170 171 preempt_disable(); 172 j0 = jiffies; 173 while ((j1 = jiffies) == j0) 174 cpu_relax(); 175 while (time_before(jiffies, 176 j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { 177 algo->gen_syndrome(disks, PAGE_SIZE, *dptrs); 178 perf++; 179 } 180 preempt_enable(); 181 182 if (perf > bestgenperf) { 183 bestgenperf = perf; 184 best = algo; 185 } 186 pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name, 187 (perf * HZ * (disks-2)) >> 188 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); 189 } 190 191 if (!best) { 192 pr_err("raid6: Yikes! No algorithm found!\n"); 193 return -EINVAL; 194 } 195 196 static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome); 197 static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome); 198 199 pr_info("raid6: using algorithm %s gen() %ld MB/s\n", 200 best->name, 201 (bestgenperf * HZ * (disks - 2)) >> 202 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2)); 203 204 if (best->xor_syndrome) { 205 unsigned long perf = 0, j0, j1; 206 207 preempt_disable(); 208 j0 = jiffies; 209 while ((j1 = jiffies) == j0) 210 cpu_relax(); 211 while (time_before(jiffies, 212 j1 + (1 << RAID6_TIME_JIFFIES_LG2))) { 213 best->xor_syndrome(disks, start, stop, 214 PAGE_SIZE, *dptrs); 215 perf++; 216 } 217 preempt_enable(); 218 219 pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", 220 (perf * HZ * (disks - 2)) >> 221 (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1)); 222 } 223 224 return 0; 225 } 226 227 228 /* Try to pick the best algorithm */ 229 /* This code uses the gfmul table as convenient data set to abuse */ 230 231 static int __init raid6_select_algo(void) 232 { 233 const int disks = RAID6_TEST_DISKS; 234 char *disk_ptr, *p; 235 void *dptrs[RAID6_TEST_DISKS]; 236 int i, cycle; 237 int error; 238 239 if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) { 240 pr_info("raid6: skipped pq benchmark and selected %s\n", 241 raid6_algos[raid6_nr_algos - 1]->name); 242 static_call_update(raid6_gen_syndrome_impl, 243 raid6_algos[raid6_nr_algos - 1]->gen_syndrome); 244 static_call_update(raid6_xor_syndrome_impl, 245 raid6_algos[raid6_nr_algos - 1]->xor_syndrome); 246 return 0; 247 } 248 249 /* prepare the buffer and fill it circularly with gfmul table */ 250 disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER); 251 if (!disk_ptr) { 252 pr_err("raid6: Yikes! No memory available.\n"); 253 return -ENOMEM; 254 } 255 256 p = disk_ptr; 257 for (i = 0; i < disks; i++) 258 dptrs[i] = p + PAGE_SIZE * i; 259 260 cycle = ((disks - 2) * PAGE_SIZE) / 65536; 261 for (i = 0; i < cycle; i++) { 262 memcpy(p, raid6_gfmul, 65536); 263 p += 65536; 264 } 265 266 if ((disks - 2) * PAGE_SIZE % 65536) 267 memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536); 268 269 /* select raid gen_syndrome function */ 270 error = raid6_choose_gen(&dptrs, disks); 271 272 free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER); 273 274 return error; 275 } 276 277 /* 278 * Register a RAID6 P/Q generation algorithm. The most optimized/unrolled 279 * implementation should be registered last so it will be selected when the 280 * boot-time benchmark is disabled. 281 */ 282 void __init raid6_algo_add(const struct raid6_calls *algo) 283 { 284 if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS)) 285 return; 286 raid6_algos[raid6_nr_algos++] = algo; 287 } 288 289 void __init raid6_algo_add_default(void) 290 { 291 raid6_algo_add(&raid6_intx1); 292 raid6_algo_add(&raid6_intx2); 293 raid6_algo_add(&raid6_intx4); 294 raid6_algo_add(&raid6_intx8); 295 } 296 297 void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo) 298 { 299 if (WARN_ON_ONCE(raid6_recov_algo)) 300 return; 301 raid6_recov_algo = algo; 302 } 303 304 #ifdef CONFIG_RAID6_PQ_ARCH 305 #include "pq_arch.h" 306 #else 307 static inline void arch_raid6_init(void) 308 { 309 raid6_algo_add_default(); 310 } 311 #endif /* CONFIG_RAID6_PQ_ARCH */ 312 313 static int __init raid6_init(void) 314 { 315 /* 316 * Architectures providing arch_raid6_init must add all PQ generation 317 * algorithms they want to consider in arch_raid6_init(), including 318 * the generic ones using raid6_algo_add_default() if wanted. 319 */ 320 arch_raid6_init(); 321 322 /* 323 * Architectures don't have to set a recovery algorithm, we'll just pick 324 * the generic integer one if none was set. 325 */ 326 if (!raid6_recov_algo) 327 raid6_recov_algo = &raid6_recov_intx1; 328 static_call_update(raid6_recov_2data_impl, raid6_recov_algo->data2); 329 static_call_update(raid6_recov_datap_impl, raid6_recov_algo->datap); 330 pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name); 331 332 return raid6_select_algo(); 333 } 334 335 static void __exit raid6_exit(void) 336 { 337 } 338 339 subsys_initcall(raid6_init); 340 module_exit(raid6_exit); 341 MODULE_LICENSE("GPL"); 342 MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); 343 344 #if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST) 345 const struct raid6_calls *raid6_algo_find(unsigned int idx) 346 { 347 if (idx >= raid6_nr_algos) { 348 /* 349 * Always include the simplest generic integer implementation in 350 * the unit tests as a baseline. 351 */ 352 if (idx == raid6_nr_algos && 353 raid6_algos[0] != &raid6_intx1) 354 return &raid6_intx1; 355 return NULL; 356 } 357 return raid6_algos[idx]; 358 } 359 EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find); 360 361 const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx) 362 { 363 switch (idx) { 364 case 0: 365 /* always test the generic integer implementation */ 366 return &raid6_recov_intx1; 367 case 1: 368 /* test the optimized implementation if there is one */ 369 if (raid6_recov_algo != &raid6_recov_intx1) 370 return raid6_recov_algo; 371 return NULL; 372 default: 373 return NULL; 374 } 375 } 376 EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find); 377 #endif /* CONFIG_RAID6_PQ_KUNIT_TEST */ 378