1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 4 * Ingo Molnar, Matti Aarnio, Jakub Jelinek, Richard Henderson. 5 * 6 * Dispatch optimized XOR parity functions. 7 */ 8 9 #include <linux/module.h> 10 #include <linux/gfp.h> 11 #include <linux/slab.h> 12 #include <linux/raid/xor.h> 13 #include <linux/preempt.h> 14 #include <linux/static_call.h> 15 #include "xor_impl.h" 16 17 DEFINE_STATIC_CALL_NULL(xor_gen_impl, *xor_block_8regs.xor_gen); 18 19 /** 20 * xor_gen - generate RAID-style XOR information 21 * @dest: destination vector 22 * @srcs: source vectors 23 * @src_cnt: number of source vectors 24 * @bytes: length in bytes of each vector 25 * 26 * Performs bit-wise XOR operation into @dest for each of the @src_cnt vectors 27 * in @srcs for a length of @bytes bytes. @src_cnt must be non-zero, and the 28 * memory pointed to by @dest and each member of @srcs must be at least 64-byte 29 * aligned. @bytes must be non-zero and a multiple of 512. 30 * 31 * Note: for typical RAID uses, @dest either needs to be zeroed, or filled with 32 * the first disk, which then needs to be removed from @srcs. 33 */ 34 void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes) 35 { 36 WARN_ON_ONCE(!in_task() || irqs_disabled() || softirq_count()); 37 WARN_ON_ONCE(bytes == 0); 38 WARN_ON_ONCE(bytes & 511); 39 40 static_call(xor_gen_impl)(dest, srcs, src_cnt, bytes); 41 } 42 EXPORT_SYMBOL(xor_gen); 43 44 /* Set of all registered templates. */ 45 static struct xor_block_template *__initdata template_list; 46 static struct xor_block_template *forced_template; 47 48 /** 49 * xor_register - register a XOR template 50 * @tmpl: template to register 51 * 52 * Register a XOR implementation with the core. Registered implementations 53 * will be measured by a trivial benchmark, and the fastest one is chosen 54 * unless an implementation is forced using xor_force(). 55 */ 56 void __init xor_register(struct xor_block_template *tmpl) 57 { 58 tmpl->next = template_list; 59 template_list = tmpl; 60 } 61 62 /** 63 * xor_force - force use of a XOR template 64 * @tmpl: template to register 65 * 66 * Register a XOR implementation with the core and force using it. Forcing 67 * an implementation will make the core ignore any template registered using 68 * xor_register(), or any previous implementation forced using xor_force(). 69 */ 70 void __init xor_force(struct xor_block_template *tmpl) 71 { 72 forced_template = tmpl; 73 } 74 75 #define BENCH_SIZE SZ_4K 76 #define NR_SRCS 4 77 #define REPS 800U 78 79 static void __init do_xor_speed(struct xor_block_template *tmpl, void *dest, 80 void *srcs[NR_SRCS]) 81 { 82 u64 t; 83 int i; 84 85 preempt_disable(); 86 t = ktime_get_ns(); 87 for (i = 0; i < REPS; i++) { 88 mb(); /* prevent loop optimization */ 89 tmpl->xor_gen(dest, srcs, NR_SRCS, BENCH_SIZE); 90 mb(); 91 } 92 t = max(ktime_get_ns() - t, 1); 93 preempt_enable(); 94 95 /* bytes/ns == GB/s, multiply by 1000 to get MB/s [not MiB/s] */ 96 tmpl->speed = div64_u64((u64)BENCH_SIZE * REPS * NR_SRCS * 1000, t); 97 98 pr_info(" %-16s: %5d MB/sec\n", tmpl->name, tmpl->speed); 99 } 100 101 static int __init calibrate_xor_blocks(void) 102 { 103 struct xor_block_template *f, *fastest; 104 void *srcs[NR_SRCS]; 105 void *buf, *dest; 106 int i; 107 108 if (forced_template) 109 return 0; 110 111 buf = kmalloc(BENCH_SIZE * (NR_SRCS + 1), GFP_KERNEL); 112 if (!buf) { 113 pr_warn("xor: Yikes! No memory available.\n"); 114 return -ENOMEM; 115 } 116 get_random_bytes(buf, BENCH_SIZE * (NR_SRCS + 1)); 117 dest = buf; 118 for (i = 0; i < NR_SRCS; i++) 119 srcs[i] = buf + (i + 1) * BENCH_SIZE; 120 121 pr_info("xor: measuring software checksum speed\n"); 122 fastest = template_list; 123 for (f = template_list; f; f = f->next) { 124 do_xor_speed(f, dest, srcs); 125 if (f->speed > fastest->speed) 126 fastest = f; 127 } 128 static_call_update(xor_gen_impl, fastest->xor_gen); 129 pr_info("xor: using function: %s (%d MB/sec)\n", 130 fastest->name, fastest->speed); 131 132 kfree(buf); 133 return 0; 134 } 135 #undef NR_SRCS 136 137 #ifdef CONFIG_XOR_BLOCKS_ARCH 138 #include "xor_arch.h" /* $SRCARCH/xor_arch.h */ 139 #else 140 static void __init arch_xor_init(void) 141 { 142 xor_register(&xor_block_8regs); 143 xor_register(&xor_block_8regs_p); 144 xor_register(&xor_block_32regs); 145 xor_register(&xor_block_32regs_p); 146 } 147 #endif /* CONFIG_XOR_BLOCKS_ARCH */ 148 149 static int __init xor_init(void) 150 { 151 arch_xor_init(); 152 153 /* 154 * If this arch/cpu has a short-circuited selection, don't loop through 155 * all the possible functions, just use the best one. 156 */ 157 if (forced_template) { 158 pr_info("xor: automatically using best checksumming function %-10s\n", 159 forced_template->name); 160 static_call_update(xor_gen_impl, forced_template->xor_gen); 161 return 0; 162 } 163 164 #ifdef MODULE 165 return calibrate_xor_blocks(); 166 #else 167 /* 168 * Pick the first template as the temporary default until calibration 169 * happens. 170 */ 171 static_call_update(xor_gen_impl, template_list->xor_gen); 172 return 0; 173 #endif 174 } 175 176 static __exit void xor_exit(void) 177 { 178 } 179 180 MODULE_DESCRIPTION("RAID-5 checksumming functions"); 181 MODULE_LICENSE("GPL"); 182 183 /* 184 * When built-in we must register the default template before md, but we don't 185 * want calibration to run that early as that would delay the boot process. 186 */ 187 #ifndef MODULE 188 __initcall(calibrate_xor_blocks); 189 #endif 190 core_initcall(xor_init); 191 module_exit(xor_exit); 192