1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 4 * Ingo Molnar, Matti Aarnio, Jakub Jelinek, Richard Henderson. 5 * 6 * Dispatch optimized XOR parity functions. 7 */ 8 9 #include <linux/module.h> 10 #include <linux/gfp.h> 11 #include <linux/raid/xor.h> 12 #include <linux/jiffies.h> 13 #include <linux/preempt.h> 14 #include <asm/xor.h> 15 16 #ifndef XOR_SELECT_TEMPLATE 17 #define XOR_SELECT_TEMPLATE(x) (x) 18 #endif 19 20 /* The xor routines to use. */ 21 static struct xor_block_template *active_template; 22 23 void 24 xor_blocks(unsigned int src_count, unsigned int bytes, void *dest, void **srcs) 25 { 26 unsigned long *p1, *p2, *p3, *p4; 27 28 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 29 30 p1 = (unsigned long *) srcs[0]; 31 if (src_count == 1) { 32 active_template->do_2(bytes, dest, p1); 33 return; 34 } 35 36 p2 = (unsigned long *) srcs[1]; 37 if (src_count == 2) { 38 active_template->do_3(bytes, dest, p1, p2); 39 return; 40 } 41 42 p3 = (unsigned long *) srcs[2]; 43 if (src_count == 3) { 44 active_template->do_4(bytes, dest, p1, p2, p3); 45 return; 46 } 47 48 p4 = (unsigned long *) srcs[3]; 49 active_template->do_5(bytes, dest, p1, p2, p3, p4); 50 } 51 EXPORT_SYMBOL(xor_blocks); 52 53 /* Set of all registered templates. */ 54 static struct xor_block_template *__initdata template_list; 55 static bool __initdata xor_forced = false; 56 57 static void __init do_xor_register(struct xor_block_template *tmpl) 58 { 59 tmpl->next = template_list; 60 template_list = tmpl; 61 } 62 63 #define BENCH_SIZE 4096 64 #define REPS 800U 65 66 static void __init 67 do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2) 68 { 69 int speed; 70 unsigned long reps; 71 ktime_t min, start, t0; 72 73 preempt_disable(); 74 75 reps = 0; 76 t0 = ktime_get(); 77 /* delay start until time has advanced */ 78 while ((start = ktime_get()) == t0) 79 cpu_relax(); 80 do { 81 mb(); /* prevent loop optimization */ 82 tmpl->do_2(BENCH_SIZE, b1, b2); 83 mb(); 84 } while (reps++ < REPS || (t0 = ktime_get()) == start); 85 min = ktime_sub(t0, start); 86 87 preempt_enable(); 88 89 // bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] 90 speed = (1000 * reps * BENCH_SIZE) / (unsigned int)ktime_to_ns(min); 91 tmpl->speed = speed; 92 93 pr_info(" %-16s: %5d MB/sec\n", tmpl->name, speed); 94 } 95 96 static int __init calibrate_xor_blocks(void) 97 { 98 void *b1, *b2; 99 struct xor_block_template *f, *fastest; 100 101 if (xor_forced) 102 return 0; 103 104 b1 = (void *) __get_free_pages(GFP_KERNEL, 2); 105 if (!b1) { 106 pr_warn("xor: Yikes! No memory available.\n"); 107 return -ENOMEM; 108 } 109 b2 = b1 + 2*PAGE_SIZE + BENCH_SIZE; 110 111 pr_info("xor: measuring software checksum speed\n"); 112 fastest = template_list; 113 for (f = template_list; f; f = f->next) { 114 do_xor_speed(f, b1, b2); 115 if (f->speed > fastest->speed) 116 fastest = f; 117 } 118 active_template = fastest; 119 pr_info("xor: using function: %s (%d MB/sec)\n", 120 fastest->name, fastest->speed); 121 122 free_pages((unsigned long)b1, 2); 123 return 0; 124 } 125 126 static int __init xor_init(void) 127 { 128 /* 129 * If this arch/cpu has a short-circuited selection, don't loop through 130 * all the possible functions, just use the best one. 131 */ 132 active_template = XOR_SELECT_TEMPLATE(NULL); 133 if (active_template) { 134 pr_info("xor: automatically using best checksumming function %-10s\n", 135 active_template->name); 136 xor_forced = true; 137 return 0; 138 } 139 140 #define xor_speed do_xor_register 141 XOR_TRY_TEMPLATES; 142 #undef xor_speed 143 144 #ifdef MODULE 145 return calibrate_xor_blocks(); 146 #else 147 /* 148 * Pick the first template as the temporary default until calibration 149 * happens. 150 */ 151 active_template = template_list; 152 return 0; 153 #endif 154 } 155 156 static __exit void xor_exit(void) 157 { 158 } 159 160 MODULE_DESCRIPTION("RAID-5 checksumming functions"); 161 MODULE_LICENSE("GPL"); 162 163 /* 164 * When built-in we must register the default template before md, but we don't 165 * want calibration to run that early as that would delay the boot process. 166 */ 167 #ifndef MODULE 168 __initcall(calibrate_xor_blocks); 169 #endif 170 core_initcall(xor_init); 171 module_exit(xor_exit); 172