1 /* -*- linux-c -*- ------------------------------------------------------- * 2 * 3 * Copyright 2002 H. Peter Anvin - All Rights Reserved 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation, Inc., 53 Temple Place Ste 330, 8 * Boston MA 02111-1307, USA; either version 2 of the License, or 9 * (at your option) any later version; incorporated herein by reference. 10 * 11 * ----------------------------------------------------------------------- */ 12 13 /* 14 * raid6/algos.c 15 * 16 * Algorithm list and algorithm selection for RAID-6 17 */ 18 19 #include <linux/raid/pq.h> 20 #ifndef __KERNEL__ 21 #include <sys/mman.h> 22 #include <stdio.h> 23 #else 24 #include <linux/module.h> 25 #include <linux/gfp.h> 26 #if !RAID6_USE_EMPTY_ZERO_PAGE 27 /* In .bss so it's zeroed */ 28 const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(256))); 29 EXPORT_SYMBOL(raid6_empty_zero_page); 30 #endif 31 #endif 32 33 struct raid6_calls raid6_call; 34 EXPORT_SYMBOL_GPL(raid6_call); 35 36 const struct raid6_calls * const raid6_algos[] = { 37 #if defined(__ia64__) 38 &raid6_intx16, 39 &raid6_intx32, 40 #endif 41 #if defined(__i386__) && !defined(__arch_um__) 42 &raid6_mmxx1, 43 &raid6_mmxx2, 44 &raid6_sse1x1, 45 &raid6_sse1x2, 46 &raid6_sse2x1, 47 &raid6_sse2x2, 48 #ifdef CONFIG_AS_AVX2 49 &raid6_avx2x1, 50 &raid6_avx2x2, 51 #endif 52 #ifdef CONFIG_AS_AVX512 53 &raid6_avx512x1, 54 &raid6_avx512x2, 55 #endif 56 #endif 57 #if defined(__x86_64__) && !defined(__arch_um__) 58 &raid6_sse2x1, 59 &raid6_sse2x2, 60 &raid6_sse2x4, 61 #ifdef CONFIG_AS_AVX2 62 &raid6_avx2x1, 63 &raid6_avx2x2, 64 &raid6_avx2x4, 65 #endif 66 #ifdef CONFIG_AS_AVX512 67 &raid6_avx512x1, 68 &raid6_avx512x2, 69 &raid6_avx512x4, 70 #endif 71 #endif 72 #ifdef CONFIG_ALTIVEC 73 &raid6_altivec1, 74 &raid6_altivec2, 75 &raid6_altivec4, 76 &raid6_altivec8, 77 &raid6_vpermxor1, 78 &raid6_vpermxor2, 79 &raid6_vpermxor4, 80 &raid6_vpermxor8, 81 #endif 82 #if defined(CONFIG_TILEGX) 83 &raid6_tilegx8, 84 #endif 85 #if defined(CONFIG_S390) 86 &raid6_s390vx8, 87 #endif 88 &raid6_intx1, 89 &raid6_intx2, 90 &raid6_intx4, 91 &raid6_intx8, 92 #ifdef CONFIG_KERNEL_MODE_NEON 93 &raid6_neonx1, 94 &raid6_neonx2, 95 &raid6_neonx4, 96 &raid6_neonx8, 97 #endif 98 NULL 99 }; 100 101 void (*raid6_2data_recov)(int, size_t, int, int, void **); 102 EXPORT_SYMBOL_GPL(raid6_2data_recov); 103 104 void (*raid6_datap_recov)(int, size_t, int, void **); 105 EXPORT_SYMBOL_GPL(raid6_datap_recov); 106 107 const struct raid6_recov_calls *const raid6_recov_algos[] = { 108 #ifdef CONFIG_AS_AVX512 109 &raid6_recov_avx512, 110 #endif 111 #ifdef CONFIG_AS_AVX2 112 &raid6_recov_avx2, 113 #endif 114 #ifdef CONFIG_AS_SSSE3 115 &raid6_recov_ssse3, 116 #endif 117 #ifdef CONFIG_S390 118 &raid6_recov_s390xc, 119 #endif 120 #if defined(CONFIG_KERNEL_MODE_NEON) 121 &raid6_recov_neon, 122 #endif 123 &raid6_recov_intx1, 124 NULL 125 }; 126 127 #ifdef __KERNEL__ 128 #define RAID6_TIME_JIFFIES_LG2 4 129 #else 130 /* Need more time to be stable in userspace */ 131 #define RAID6_TIME_JIFFIES_LG2 9 132 #define time_before(x, y) ((x) < (y)) 133 #endif 134 135 static inline const struct raid6_recov_calls *raid6_choose_recov(void) 136 { 137 const struct raid6_recov_calls *const *algo; 138 const struct raid6_recov_calls *best; 139 140 for (best = NULL, algo = raid6_recov_algos; *algo; algo++) 141 if (!best || (*algo)->priority > best->priority) 142 if (!(*algo)->valid || (*algo)->valid()) 143 best = *algo; 144 145 if (best) { 146 raid6_2data_recov = best->data2; 147 raid6_datap_recov = best->datap; 148 149 pr_info("raid6: using %s recovery algorithm\n", best->name); 150 } else 151 pr_err("raid6: Yikes! No recovery algorithm found!\n"); 152 153 return best; 154 } 155 156 static inline const struct raid6_calls *raid6_choose_gen( 157 void *(*const dptrs)[(65536/PAGE_SIZE)+2], const int disks) 158 { 159 unsigned long perf, bestgenperf, bestxorperf, j0, j1; 160 int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */ 161 const struct raid6_calls *const *algo; 162 const struct raid6_calls *best; 163 164 for (bestgenperf = 0, bestxorperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) { 165 if (!best || (*algo)->prefer >= best->prefer) { 166 if ((*algo)->valid && !(*algo)->valid()) 167 continue; 168 169 perf = 0; 170 171 preempt_disable(); 172 j0 = jiffies; 173 while ((j1 = jiffies) == j0) 174 cpu_relax(); 175 while (time_before(jiffies, 176 j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { 177 (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs); 178 perf++; 179 } 180 preempt_enable(); 181 182 if (perf > bestgenperf) { 183 bestgenperf = perf; 184 best = *algo; 185 } 186 pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name, 187 (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2)); 188 189 if (!(*algo)->xor_syndrome) 190 continue; 191 192 perf = 0; 193 194 preempt_disable(); 195 j0 = jiffies; 196 while ((j1 = jiffies) == j0) 197 cpu_relax(); 198 while (time_before(jiffies, 199 j1 + (1<<RAID6_TIME_JIFFIES_LG2))) { 200 (*algo)->xor_syndrome(disks, start, stop, 201 PAGE_SIZE, *dptrs); 202 perf++; 203 } 204 preempt_enable(); 205 206 if (best == *algo) 207 bestxorperf = perf; 208 209 pr_info("raid6: %-8s xor() %5ld MB/s\n", (*algo)->name, 210 (perf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1)); 211 } 212 } 213 214 if (best) { 215 pr_info("raid6: using algorithm %s gen() %ld MB/s\n", 216 best->name, 217 (bestgenperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2)); 218 if (best->xor_syndrome) 219 pr_info("raid6: .... xor() %ld MB/s, rmw enabled\n", 220 (bestxorperf*HZ) >> (20-16+RAID6_TIME_JIFFIES_LG2+1)); 221 raid6_call = *best; 222 } else 223 pr_err("raid6: Yikes! No algorithm found!\n"); 224 225 return best; 226 } 227 228 229 /* Try to pick the best algorithm */ 230 /* This code uses the gfmul table as convenient data set to abuse */ 231 232 int __init raid6_select_algo(void) 233 { 234 const int disks = (65536/PAGE_SIZE)+2; 235 236 const struct raid6_calls *gen_best; 237 const struct raid6_recov_calls *rec_best; 238 char *syndromes; 239 void *dptrs[(65536/PAGE_SIZE)+2]; 240 int i; 241 242 for (i = 0; i < disks-2; i++) 243 dptrs[i] = ((char *)raid6_gfmul) + PAGE_SIZE*i; 244 245 /* Normal code - use a 2-page allocation to avoid D$ conflict */ 246 syndromes = (void *) __get_free_pages(GFP_KERNEL, 1); 247 248 if (!syndromes) { 249 pr_err("raid6: Yikes! No memory available.\n"); 250 return -ENOMEM; 251 } 252 253 dptrs[disks-2] = syndromes; 254 dptrs[disks-1] = syndromes + PAGE_SIZE; 255 256 /* select raid gen_syndrome function */ 257 gen_best = raid6_choose_gen(&dptrs, disks); 258 259 /* select raid recover functions */ 260 rec_best = raid6_choose_recov(); 261 262 free_pages((unsigned long)syndromes, 1); 263 264 return gen_best && rec_best ? 0 : -EINVAL; 265 } 266 267 static void raid6_exit(void) 268 { 269 do { } while (0); 270 } 271 272 subsys_initcall(raid6_select_algo); 273 module_exit(raid6_exit); 274 MODULE_LICENSE("GPL"); 275 MODULE_DESCRIPTION("RAID6 Q-syndrome calculations"); 276