1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2024 Institute of Software, CAS. 4 * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn> 5 */ 6 7 #include <linux/raid/pq.h> 8 #include "rvv.h" 9 10 static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp, 11 u8 *dq, const u8 *pbmul, 12 const u8 *qmul) 13 { 14 asm volatile (".option push\n" 15 ".option arch,+v\n" 16 "vsetvli x0, %[avl], e8, m1, ta, ma\n" 17 ".option pop\n" 18 : : 19 [avl]"r"(16) 20 ); 21 22 /* 23 * while ( bytes-- ) { 24 * uint8_t px, qx, db; 25 * 26 * px = *p ^ *dp; 27 * qx = qmul[*q ^ *dq]; 28 * *dq++ = db = pbmul[px] ^ qx; 29 * *dp++ = db ^ px; 30 * p++; q++; 31 * } 32 */ 33 while (bytes) { 34 /* 35 * v0:px, v1:dp, 36 * v2:qx, v3:dq, 37 * v4:vx, v5:vy, 38 * v6:qm0, v7:qm1, 39 * v8:pm0, v9:pm1, 40 * v14:p/qm[vx], v15:p/qm[vy] 41 */ 42 asm volatile (".option push\n" 43 ".option arch,+v\n" 44 "vle8.v v0, (%[px])\n" 45 "vle8.v v1, (%[dp])\n" 46 "vxor.vv v0, v0, v1\n" 47 "vle8.v v2, (%[qx])\n" 48 "vle8.v v3, (%[dq])\n" 49 "vxor.vv v4, v2, v3\n" 50 "vsrl.vi v5, v4, 4\n" 51 "vand.vi v4, v4, 0xf\n" 52 "vle8.v v6, (%[qm0])\n" 53 "vle8.v v7, (%[qm1])\n" 54 "vrgather.vv v14, v6, v4\n" /* v14 = qm[vx] */ 55 "vrgather.vv v15, v7, v5\n" /* v15 = qm[vy] */ 56 "vxor.vv v2, v14, v15\n" /* v2 = qmul[*q ^ *dq] */ 57 58 "vsrl.vi v5, v0, 4\n" 59 "vand.vi v4, v0, 0xf\n" 60 "vle8.v v8, (%[pm0])\n" 61 "vle8.v v9, (%[pm1])\n" 62 "vrgather.vv v14, v8, v4\n" /* v14 = pm[vx] */ 63 "vrgather.vv v15, v9, v5\n" /* v15 = pm[vy] */ 64 "vxor.vv v4, v14, v15\n" /* v4 = pbmul[px] */ 65 "vxor.vv v3, v4, v2\n" /* v3 = db = pbmul[px] ^ qx */ 66 "vxor.vv v1, v3, v0\n" /* v1 = db ^ px; */ 67 "vse8.v v3, (%[dq])\n" 68 "vse8.v v1, (%[dp])\n" 69 ".option pop\n" 70 : : 71 [px]"r"(p), 72 [dp]"r"(dp), 73 [qx]"r"(q), 74 [dq]"r"(dq), 75 [qm0]"r"(qmul), 76 [qm1]"r"(qmul + 16), 77 [pm0]"r"(pbmul), 78 [pm1]"r"(pbmul + 16) 79 :); 80 81 bytes -= 16; 82 p += 16; 83 q += 16; 84 dp += 16; 85 dq += 16; 86 } 87 } 88 89 static void __raid6_datap_recov_rvv(int bytes, u8 *p, u8 *q, 90 u8 *dq, const u8 *qmul) 91 { 92 asm volatile (".option push\n" 93 ".option arch,+v\n" 94 "vsetvli x0, %[avl], e8, m1, ta, ma\n" 95 ".option pop\n" 96 : : 97 [avl]"r"(16) 98 ); 99 100 /* 101 * while (bytes--) { 102 * *p++ ^= *dq = qmul[*q ^ *dq]; 103 * q++; dq++; 104 * } 105 */ 106 while (bytes) { 107 /* 108 * v0:vx, v1:vy, 109 * v2:dq, v3:p, 110 * v4:qm0, v5:qm1, 111 * v10:m[vx], v11:m[vy] 112 */ 113 asm volatile (".option push\n" 114 ".option arch,+v\n" 115 "vle8.v v0, (%[vx])\n" 116 "vle8.v v2, (%[dq])\n" 117 "vxor.vv v0, v0, v2\n" 118 "vsrl.vi v1, v0, 4\n" 119 "vand.vi v0, v0, 0xf\n" 120 "vle8.v v4, (%[qm0])\n" 121 "vle8.v v5, (%[qm1])\n" 122 "vrgather.vv v10, v4, v0\n" 123 "vrgather.vv v11, v5, v1\n" 124 "vxor.vv v0, v10, v11\n" 125 "vle8.v v1, (%[vy])\n" 126 "vxor.vv v1, v0, v1\n" 127 "vse8.v v0, (%[dq])\n" 128 "vse8.v v1, (%[vy])\n" 129 ".option pop\n" 130 : : 131 [vx]"r"(q), 132 [vy]"r"(p), 133 [dq]"r"(dq), 134 [qm0]"r"(qmul), 135 [qm1]"r"(qmul + 16) 136 :); 137 138 bytes -= 16; 139 p += 16; 140 q += 16; 141 dq += 16; 142 } 143 } 144 145 static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila, 146 int failb, void **ptrs) 147 { 148 u8 *p, *q, *dp, *dq; 149 const u8 *pbmul; /* P multiplier table for B data */ 150 const u8 *qmul; /* Q multiplier table (for both) */ 151 152 p = (u8 *)ptrs[disks - 2]; 153 q = (u8 *)ptrs[disks - 1]; 154 155 /* 156 * Compute syndrome with zero for the missing data pages 157 * Use the dead data pages as temporary storage for 158 * delta p and delta q 159 */ 160 dp = (u8 *)ptrs[faila]; 161 ptrs[faila] = raid6_get_zero_page(); 162 ptrs[disks - 2] = dp; 163 dq = (u8 *)ptrs[failb]; 164 ptrs[failb] = raid6_get_zero_page(); 165 ptrs[disks - 1] = dq; 166 167 raid6_call.gen_syndrome(disks, bytes, ptrs); 168 169 /* Restore pointer table */ 170 ptrs[faila] = dp; 171 ptrs[failb] = dq; 172 ptrs[disks - 2] = p; 173 ptrs[disks - 1] = q; 174 175 /* Now, pick the proper data tables */ 176 pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]]; 177 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ 178 raid6_gfexp[failb]]]; 179 180 kernel_vector_begin(); 181 __raid6_2data_recov_rvv(bytes, p, q, dp, dq, pbmul, qmul); 182 kernel_vector_end(); 183 } 184 185 static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, 186 void **ptrs) 187 { 188 u8 *p, *q, *dq; 189 const u8 *qmul; /* Q multiplier table */ 190 191 p = (u8 *)ptrs[disks - 2]; 192 q = (u8 *)ptrs[disks - 1]; 193 194 /* 195 * Compute syndrome with zero for the missing data page 196 * Use the dead data page as temporary storage for delta q 197 */ 198 dq = (u8 *)ptrs[faila]; 199 ptrs[faila] = raid6_get_zero_page(); 200 ptrs[disks - 1] = dq; 201 202 raid6_call.gen_syndrome(disks, bytes, ptrs); 203 204 /* Restore pointer table */ 205 ptrs[faila] = dq; 206 ptrs[disks - 1] = q; 207 208 /* Now, pick the proper data tables */ 209 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; 210 211 kernel_vector_begin(); 212 __raid6_datap_recov_rvv(bytes, p, q, dq, qmul); 213 kernel_vector_end(); 214 } 215 216 const struct raid6_recov_calls raid6_recov_rvv = { 217 .data2 = raid6_2data_recov_rvv, 218 .datap = raid6_datap_recov_rvv, 219 .valid = rvv_has_vector, 220 .name = "rvv", 221 .priority = 1, 222 }; 223