1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2024 Institute of Software, CAS. 4 * Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn> 5 */ 6 7 #include <linux/mm.h> 8 #include <linux/raid/pq.h> 9 #include "algos.h" 10 #include "rvv.h" 11 12 static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp, 13 u8 *dq, const u8 *pbmul, 14 const u8 *qmul) 15 { 16 asm volatile (".option push\n" 17 ".option arch,+v\n" 18 "vsetvli x0, %[avl], e8, m1, ta, ma\n" 19 ".option pop\n" 20 : : 21 [avl]"r"(16) 22 ); 23 24 /* 25 * while ( bytes-- ) { 26 * uint8_t px, qx, db; 27 * 28 * px = *p ^ *dp; 29 * qx = qmul[*q ^ *dq]; 30 * *dq++ = db = pbmul[px] ^ qx; 31 * *dp++ = db ^ px; 32 * p++; q++; 33 * } 34 */ 35 while (bytes) { 36 /* 37 * v0:px, v1:dp, 38 * v2:qx, v3:dq, 39 * v4:vx, v5:vy, 40 * v6:qm0, v7:qm1, 41 * v8:pm0, v9:pm1, 42 * v14:p/qm[vx], v15:p/qm[vy] 43 */ 44 asm volatile (".option push\n" 45 ".option arch,+v\n" 46 "vle8.v v0, (%[px])\n" 47 "vle8.v v1, (%[dp])\n" 48 "vxor.vv v0, v0, v1\n" 49 "vle8.v v2, (%[qx])\n" 50 "vle8.v v3, (%[dq])\n" 51 "vxor.vv v4, v2, v3\n" 52 "vsrl.vi v5, v4, 4\n" 53 "vand.vi v4, v4, 0xf\n" 54 "vle8.v v6, (%[qm0])\n" 55 "vle8.v v7, (%[qm1])\n" 56 "vrgather.vv v14, v6, v4\n" /* v14 = qm[vx] */ 57 "vrgather.vv v15, v7, v5\n" /* v15 = qm[vy] */ 58 "vxor.vv v2, v14, v15\n" /* v2 = qmul[*q ^ *dq] */ 59 60 "vsrl.vi v5, v0, 4\n" 61 "vand.vi v4, v0, 0xf\n" 62 "vle8.v v8, (%[pm0])\n" 63 "vle8.v v9, (%[pm1])\n" 64 "vrgather.vv v14, v8, v4\n" /* v14 = pm[vx] */ 65 "vrgather.vv v15, v9, v5\n" /* v15 = pm[vy] */ 66 "vxor.vv v4, v14, v15\n" /* v4 = pbmul[px] */ 67 "vxor.vv v3, v4, v2\n" /* v3 = db = pbmul[px] ^ qx */ 68 "vxor.vv v1, v3, v0\n" /* v1 = db ^ px; */ 69 "vse8.v v3, (%[dq])\n" 70 "vse8.v v1, (%[dp])\n" 71 ".option pop\n" 72 : : 73 [px]"r"(p), 74 [dp]"r"(dp), 75 [qx]"r"(q), 76 [dq]"r"(dq), 77 [qm0]"r"(qmul), 78 [qm1]"r"(qmul + 16), 79 [pm0]"r"(pbmul), 80 [pm1]"r"(pbmul + 16) 81 :); 82 83 bytes -= 16; 84 p += 16; 85 q += 16; 86 dp += 16; 87 dq += 16; 88 } 89 } 90 91 static void __raid6_datap_recov_rvv(int bytes, u8 *p, u8 *q, 92 u8 *dq, const u8 *qmul) 93 { 94 asm volatile (".option push\n" 95 ".option arch,+v\n" 96 "vsetvli x0, %[avl], e8, m1, ta, ma\n" 97 ".option pop\n" 98 : : 99 [avl]"r"(16) 100 ); 101 102 /* 103 * while (bytes--) { 104 * *p++ ^= *dq = qmul[*q ^ *dq]; 105 * q++; dq++; 106 * } 107 */ 108 while (bytes) { 109 /* 110 * v0:vx, v1:vy, 111 * v2:dq, v3:p, 112 * v4:qm0, v5:qm1, 113 * v10:m[vx], v11:m[vy] 114 */ 115 asm volatile (".option push\n" 116 ".option arch,+v\n" 117 "vle8.v v0, (%[vx])\n" 118 "vle8.v v2, (%[dq])\n" 119 "vxor.vv v0, v0, v2\n" 120 "vsrl.vi v1, v0, 4\n" 121 "vand.vi v0, v0, 0xf\n" 122 "vle8.v v4, (%[qm0])\n" 123 "vle8.v v5, (%[qm1])\n" 124 "vrgather.vv v10, v4, v0\n" 125 "vrgather.vv v11, v5, v1\n" 126 "vxor.vv v0, v10, v11\n" 127 "vle8.v v1, (%[vy])\n" 128 "vxor.vv v1, v0, v1\n" 129 "vse8.v v0, (%[dq])\n" 130 "vse8.v v1, (%[vy])\n" 131 ".option pop\n" 132 : : 133 [vx]"r"(q), 134 [vy]"r"(p), 135 [dq]"r"(dq), 136 [qm0]"r"(qmul), 137 [qm1]"r"(qmul + 16) 138 :); 139 140 bytes -= 16; 141 p += 16; 142 q += 16; 143 dq += 16; 144 } 145 } 146 147 static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila, 148 int failb, void **ptrs) 149 { 150 u8 *p, *q, *dp, *dq; 151 const u8 *pbmul; /* P multiplier table for B data */ 152 const u8 *qmul; /* Q multiplier table (for both) */ 153 154 p = (u8 *)ptrs[disks - 2]; 155 q = (u8 *)ptrs[disks - 1]; 156 157 /* 158 * Compute syndrome with zero for the missing data pages 159 * Use the dead data pages as temporary storage for 160 * delta p and delta q 161 */ 162 dp = (u8 *)ptrs[faila]; 163 ptrs[faila] = page_address(ZERO_PAGE(0)); 164 ptrs[disks - 2] = dp; 165 dq = (u8 *)ptrs[failb]; 166 ptrs[failb] = page_address(ZERO_PAGE(0)); 167 ptrs[disks - 1] = dq; 168 169 raid6_gen_syndrome(disks, bytes, ptrs); 170 171 /* Restore pointer table */ 172 ptrs[faila] = dp; 173 ptrs[failb] = dq; 174 ptrs[disks - 2] = p; 175 ptrs[disks - 1] = q; 176 177 /* Now, pick the proper data tables */ 178 pbmul = raid6_vgfmul[raid6_gfexi[failb - faila]]; 179 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ 180 raid6_gfexp[failb]]]; 181 182 kernel_vector_begin(); 183 __raid6_2data_recov_rvv(bytes, p, q, dp, dq, pbmul, qmul); 184 kernel_vector_end(); 185 } 186 187 static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila, 188 void **ptrs) 189 { 190 u8 *p, *q, *dq; 191 const u8 *qmul; /* Q multiplier table */ 192 193 p = (u8 *)ptrs[disks - 2]; 194 q = (u8 *)ptrs[disks - 1]; 195 196 /* 197 * Compute syndrome with zero for the missing data page 198 * Use the dead data page as temporary storage for delta q 199 */ 200 dq = (u8 *)ptrs[faila]; 201 ptrs[faila] = page_address(ZERO_PAGE(0)); 202 ptrs[disks - 1] = dq; 203 204 raid6_gen_syndrome(disks, bytes, ptrs); 205 206 /* Restore pointer table */ 207 ptrs[faila] = dq; 208 ptrs[disks - 1] = q; 209 210 /* Now, pick the proper data tables */ 211 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; 212 213 kernel_vector_begin(); 214 __raid6_datap_recov_rvv(bytes, p, q, dq, qmul); 215 kernel_vector_end(); 216 } 217 218 const struct raid6_recov_calls raid6_recov_rvv = { 219 .data2 = raid6_2data_recov_rvv, 220 .datap = raid6_datap_recov_rvv, 221 .name = "rvv", 222 }; 223