1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2024 Institute of Software, CAS. 4 * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn> 5 */ 6 7 #include <asm/vector.h> 8 #include <linux/raid/pq.h> 9 10 static int rvv_has_vector(void) 11 { 12 return has_vector(); 13 } 14 15 static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp, 16 u8 *dq, const u8 *pbmul, 17 const u8 *qmul) 18 { 19 asm volatile (".option push\n" 20 ".option arch,+v\n" 21 "vsetvli x0, %[avl], e8, m1, ta, ma\n" 22 ".option pop\n" 23 : : 24 [avl]"r"(16) 25 ); 26 27 /* 28 * while ( bytes-- ) { 29 * uint8_t px, qx, db; 30 * 31 * px = *p ^ *dp; 32 * qx = qmul[*q ^ *dq]; 33 * *dq++ = db = pbmul[px] ^ qx; 34 * *dp++ = db ^ px; 35 * p++; q++; 36 * } 37 */ 38 while (bytes) { 39 /* 40 * v0:px, v1:dp, 41 * v2:qx, v3:dq, 42 * v4:vx, v5:vy, 43 * v6:qm0, v7:qm1, 44 * v8:pm0, v9:pm1, 45 * v14:p/qm[vx], v15:p/qm[vy] 46 */ 47 asm volatile (".option push\n" 48 ".option arch,+v\n" 49 "vle8.v v0, (%[px])\n" 50 "vle8.v v1, (%[dp])\n" 51 "vxor.vv v0, v0, v1\n" 52 "vle8.v v2, (%[qx])\n" 53 "vle8.v v3, (%[dq])\n" 54 "vxor.vv v4, v2, v3\n" 55 "vsrl.vi v5, v4, 4\n" 56 "vand.vi v4, v4, 0xf\n" 57 "vle8.v v6, (%[qm0])\n" 58 "vle8.v v7, (%[qm1])\n" 59 "vrgather.vv v14, v6, v4\n" /* v14 = qm[vx] */ 60 "vrgather.vv v15, v7, v5\n" /* v15 = qm[vy] */ 61 "vxor.vv v2, v14, v15\n" /* v2 = qmul[*q ^ *dq] */ 62 63 "vsrl.vi v5, v0, 4\n" 64 "vand.vi v4, v0, 0xf\n" 65 "vle8.v v8, (%[pm0])\n" 66 "vle8.v v9, (%[pm1])\n" 67 "vrgather.vv v14, v8, v4\n" /* v14 = pm[vx] */ 68 "vrgather.vv v15, v9, v5\n" /* v15 = pm[vy] */ 69 "vxor.vv v4, v14, v15\n" /* v4 = pbmul[px] */ 70 "vxor.vv v3, v4, v2\n" /* v3 = db = pbmul[px] ^ qx */ 71 "vxor.vv v1, v3, v0\n" /* v1 = db ^ px; */ 72 "vse8.v v3, (%[dq])\n" 73 "vse8.v v1, (%[dp])\n" 74 ".option pop\n" 75 : : 76 [px]"r"(p), 77 [dp]"r"(dp), 78 [qx]"r"(q), 79 [dq]"r"(dq), 80 [qm0]"r"(qmul), 81 [qm1]"r"(qmul + 16), 82 [pm0]"r"(pbmul), 83 [pm1]"r"(pbmul + 16) 84 :); 85 86 bytes -= 16; 87 p += 16; 88 q += 16; 89 dp += 16; 90 dq += 16; 91 } 92 } 93 94 static void __raid6_datap_recov_rvv(int bytes, u8 *p, u8 *q, 95 u8 *dq, const u8 *qmul) 96 { 97 asm volatile (".option push\n" 98 ".option arch,+v\n" 99 "vsetvli x0, %[avl], e8, m1, ta, ma\n" 100 ".option pop\n" 101 : : 102 [avl]"r"(16) 103 ); 104 105 /* 106 * while (bytes--) { 107 * *p++ ^= *dq = qmul[*q ^ *dq]; 108 * q++; dq++; 109 * } 110 */ 111 while (bytes) { 112 /* 113 * v0:vx, v1:vy, 114 * v2:dq, v3:p, 115 * v4:qm0, v5:qm1, 116 * v10:m[vx], v11:m[vy] 117 */ 118 asm volatile (".option push\n" 119 ".option arch,+v\n" 120 "vle8.v v0, (%[vx])\n" 121 "vle8.v v2, (%[dq])\n" 122 "vxor.vv v0, v0, v2\n" 123 "vsrl.vi v1, v0, 4\n" 124 "vand.vi v0, v0, 0xf\n" 125 "vle8.v v4, (%[qm0])\n" 126 "vle8.v v5, (%[qm1])\n" 127 "vrgather.vv v10, v4, v0\n" 128 "vrgather.vv v11, v5, v1\n" 129 "vxor.vv v0, v10, v11\n" 130 "vle8.v v1, (%[vy])\n" 131 "vxor.vv v1, v0, v1\n" 132 "vse8.v v0, (%[dq])\n" 133 "vse8.v v1, (%[vy])\n" 134 ".option pop\n" 135 : : 136 [vx]"r"(q), 137 [vy]"r"(p), 138 [dq]"r"(dq), 139 [qm0]"r"(qmul), 140 [qm1]"r"(qmul + 16) 141 :); 142 143 bytes -= 16; 144 p += 16; 145 q += 16; 146 dq += 16; 147 } 148 } 149 150 static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila, 151 int failb, void **ptrs) 152 { 153 u8 *p, *q, *dp, *dq; 154 const u8 *pbmul; /* P multiplier table for B data */ 155 const u8 *qmul; /* Q multiplier table (for both) */ 156 157 p = (u8 *)ptrs[disks - 2]; 158 q = (u8 *)ptrs[disks - 1]; 159 160 /* 161 * Compute syndrome with zero for the missing data pages 162 * Use the dead data pages as temporary storage for 163 * delta p and delta q 164 */ 165 dp = (u8 *)ptrs[faila]; 166 ptrs[faila] = raid6_get_zero_page(); 167 ptrs[disks - 2] = dp; 168 dq = (u8 *)ptrs[failb]; 169 ptrs[failb] = raid6_get_zero_page(); 170 ptrs[disks - 1] = dq; 171 172 raid6_call.gen_syndrome(disks, bytes, ptrs); 173 174 /* Restore pointer table */ 175 ptrs[faila] = dp; 176 ptrs[failb] = dq; 177 ptrs[disks - 2] = p; 178 ptrs[disks - 1] = q; 179 180 /* Now, pick the proper data tables */ 181 pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]]; 182 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ 183 raid6_gfexp[failb]]]; 184 185 kernel_vector_begin(); 186 __raid6_2data_recov_rvv(bytes, p, q, dp, dq, pbmul, qmul); 187 kernel_vector_end(); 188 } 189 190 static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, 191 void **ptrs) 192 { 193 u8 *p, *q, *dq; 194 const u8 *qmul; /* Q multiplier table */ 195 196 p = (u8 *)ptrs[disks - 2]; 197 q = (u8 *)ptrs[disks - 1]; 198 199 /* 200 * Compute syndrome with zero for the missing data page 201 * Use the dead data page as temporary storage for delta q 202 */ 203 dq = (u8 *)ptrs[faila]; 204 ptrs[faila] = raid6_get_zero_page(); 205 ptrs[disks - 1] = dq; 206 207 raid6_call.gen_syndrome(disks, bytes, ptrs); 208 209 /* Restore pointer table */ 210 ptrs[faila] = dq; 211 ptrs[disks - 1] = q; 212 213 /* Now, pick the proper data tables */ 214 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; 215 216 kernel_vector_begin(); 217 __raid6_datap_recov_rvv(bytes, p, q, dq, qmul); 218 kernel_vector_end(); 219 } 220 221 const struct raid6_recov_calls raid6_recov_rvv = { 222 .data2 = raid6_2data_recov_rvv, 223 .datap = raid6_datap_recov_rvv, 224 .valid = rvv_has_vector, 225 .name = "rvv", 226 .priority = 1, 227 }; 228