1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2012 Intel Corporation 4 * Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org> 5 */ 6 7 #include <linux/raid/pq.h> 8 9 #ifdef __KERNEL__ 10 #include <asm/simd.h> 11 #include "neon.h" 12 #else 13 #define scoped_ksimd() 14 #define cpu_has_neon() (1) 15 #endif 16 17 static int raid6_has_neon(void) 18 { 19 return cpu_has_neon(); 20 } 21 22 static void raid6_2data_recov_neon(int disks, size_t bytes, int faila, 23 int failb, void **ptrs) 24 { 25 u8 *p, *q, *dp, *dq; 26 const u8 *pbmul; /* P multiplier table for B data */ 27 const u8 *qmul; /* Q multiplier table (for both) */ 28 29 p = (u8 *)ptrs[disks - 2]; 30 q = (u8 *)ptrs[disks - 1]; 31 32 /* 33 * Compute syndrome with zero for the missing data pages 34 * Use the dead data pages as temporary storage for 35 * delta p and delta q 36 */ 37 dp = (u8 *)ptrs[faila]; 38 ptrs[faila] = raid6_get_zero_page(); 39 ptrs[disks - 2] = dp; 40 dq = (u8 *)ptrs[failb]; 41 ptrs[failb] = raid6_get_zero_page(); 42 ptrs[disks - 1] = dq; 43 44 raid6_call.gen_syndrome(disks, bytes, ptrs); 45 46 /* Restore pointer table */ 47 ptrs[faila] = dp; 48 ptrs[failb] = dq; 49 ptrs[disks - 2] = p; 50 ptrs[disks - 1] = q; 51 52 /* Now, pick the proper data tables */ 53 pbmul = raid6_vgfmul[raid6_gfexi[failb-faila]]; 54 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila] ^ 55 raid6_gfexp[failb]]]; 56 57 scoped_ksimd() 58 __raid6_2data_recov_neon(bytes, p, q, dp, dq, pbmul, qmul); 59 } 60 61 static void raid6_datap_recov_neon(int disks, size_t bytes, int faila, 62 void **ptrs) 63 { 64 u8 *p, *q, *dq; 65 const u8 *qmul; /* Q multiplier table */ 66 67 p = (u8 *)ptrs[disks - 2]; 68 q = (u8 *)ptrs[disks - 1]; 69 70 /* 71 * Compute syndrome with zero for the missing data page 72 * Use the dead data page as temporary storage for delta q 73 */ 74 dq = (u8 *)ptrs[faila]; 75 ptrs[faila] = raid6_get_zero_page(); 76 ptrs[disks - 1] = dq; 77 78 raid6_call.gen_syndrome(disks, bytes, ptrs); 79 80 /* Restore pointer table */ 81 ptrs[faila] = dq; 82 ptrs[disks - 1] = q; 83 84 /* Now, pick the proper data tables */ 85 qmul = raid6_vgfmul[raid6_gfinv[raid6_gfexp[faila]]]; 86 87 scoped_ksimd() 88 __raid6_datap_recov_neon(bytes, p, q, dq, qmul); 89 } 90 91 const struct raid6_recov_calls raid6_recov_neon = { 92 .data2 = raid6_2data_recov_neon, 93 .datap = raid6_datap_recov_neon, 94 .valid = raid6_has_neon, 95 .name = "neon", 96 .priority = 10, 97 }; 98