1 // SPDX-License-Identifier: GPL-2.0-or-later 2 #include <linux/prefetch.h> 3 #include <linux/raid/xor_impl.h> 4 #include <asm-generic/xor.h> 5 6 static void 7 xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1, 8 const unsigned long * __restrict p2) 9 { 10 long lines = bytes / (sizeof (long)) / 8 - 1; 11 prefetchw(p1); 12 prefetch(p2); 13 14 do { 15 prefetchw(p1+8); 16 prefetch(p2+8); 17 once_more: 18 p1[0] ^= p2[0]; 19 p1[1] ^= p2[1]; 20 p1[2] ^= p2[2]; 21 p1[3] ^= p2[3]; 22 p1[4] ^= p2[4]; 23 p1[5] ^= p2[5]; 24 p1[6] ^= p2[6]; 25 p1[7] ^= p2[7]; 26 p1 += 8; 27 p2 += 8; 28 } while (--lines > 0); 29 if (lines == 0) 30 goto once_more; 31 } 32 33 static void 34 xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1, 35 const unsigned long * __restrict p2, 36 const unsigned long * __restrict p3) 37 { 38 long lines = bytes / (sizeof (long)) / 8 - 1; 39 prefetchw(p1); 40 prefetch(p2); 41 prefetch(p3); 42 43 do { 44 prefetchw(p1+8); 45 prefetch(p2+8); 46 prefetch(p3+8); 47 once_more: 48 p1[0] ^= p2[0] ^ p3[0]; 49 p1[1] ^= p2[1] ^ p3[1]; 50 p1[2] ^= p2[2] ^ p3[2]; 51 p1[3] ^= p2[3] ^ p3[3]; 52 p1[4] ^= p2[4] ^ p3[4]; 53 p1[5] ^= p2[5] ^ p3[5]; 54 p1[6] ^= p2[6] ^ p3[6]; 55 p1[7] ^= p2[7] ^ p3[7]; 56 p1 += 8; 57 p2 += 8; 58 p3 += 8; 59 } while (--lines > 0); 60 if (lines == 0) 61 goto once_more; 62 } 63 64 static void 65 xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1, 66 const unsigned long * __restrict p2, 67 const unsigned long * __restrict p3, 68 const unsigned long * __restrict p4) 69 { 70 long lines = bytes / (sizeof (long)) / 8 - 1; 71 72 prefetchw(p1); 73 prefetch(p2); 74 prefetch(p3); 75 prefetch(p4); 76 77 do { 78 prefetchw(p1+8); 79 prefetch(p2+8); 80 prefetch(p3+8); 81 prefetch(p4+8); 82 once_more: 83 p1[0] ^= p2[0] ^ p3[0] ^ p4[0]; 84 p1[1] ^= p2[1] ^ p3[1] ^ p4[1]; 85 p1[2] ^= p2[2] ^ p3[2] ^ p4[2]; 86 p1[3] ^= p2[3] ^ p3[3] ^ p4[3]; 87 p1[4] ^= p2[4] ^ p3[4] ^ p4[4]; 88 p1[5] ^= p2[5] ^ p3[5] ^ p4[5]; 89 p1[6] ^= p2[6] ^ p3[6] ^ p4[6]; 90 p1[7] ^= p2[7] ^ p3[7] ^ p4[7]; 91 p1 += 8; 92 p2 += 8; 93 p3 += 8; 94 p4 += 8; 95 } while (--lines > 0); 96 if (lines == 0) 97 goto once_more; 98 } 99 100 static void 101 xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1, 102 const unsigned long * __restrict p2, 103 const unsigned long * __restrict p3, 104 const unsigned long * __restrict p4, 105 const unsigned long * __restrict p5) 106 { 107 long lines = bytes / (sizeof (long)) / 8 - 1; 108 109 prefetchw(p1); 110 prefetch(p2); 111 prefetch(p3); 112 prefetch(p4); 113 prefetch(p5); 114 115 do { 116 prefetchw(p1+8); 117 prefetch(p2+8); 118 prefetch(p3+8); 119 prefetch(p4+8); 120 prefetch(p5+8); 121 once_more: 122 p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0]; 123 p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1]; 124 p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2]; 125 p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3]; 126 p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4]; 127 p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5]; 128 p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6]; 129 p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7]; 130 p1 += 8; 131 p2 += 8; 132 p3 += 8; 133 p4 += 8; 134 p5 += 8; 135 } while (--lines > 0); 136 if (lines == 0) 137 goto once_more; 138 } 139 140 struct xor_block_template xor_block_8regs_p = { 141 .name = "8regs_prefetch", 142 .do_2 = xor_8regs_p_2, 143 .do_3 = xor_8regs_p_3, 144 .do_4 = xor_8regs_p_4, 145 .do_5 = xor_8regs_p_5, 146 }; 147