1// SPDX-License-Identifier: GPL-2.0 2/* 3 * raid6_vx$#.c 4 * 5 * $#-way unrolled RAID6 gen/xor functions for s390 6 * based on the vector facility 7 * 8 * Copyright IBM Corp. 2016 9 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 10 * 11 * This file is postprocessed using unroll.awk. 12 */ 13 14#include <linux/raid/pq.h> 15#include <asm/fpu.h> 16 17#define NSIZE 16 18 19static __always_inline void LOAD_CONST(void) 20{ 21 fpu_vrepib(24, 0x07); 22 fpu_vrepib(25, 0x1d); 23} 24 25/* 26 * The SHLBYTE() operation shifts each of the 16 bytes in 27 * vector register y left by 1 bit and stores the result in 28 * vector register x. 29 */ 30#define SHLBYTE(x, y) fpu_vab(x, y, y) 31 32/* 33 * For each of the 16 bytes in the vector register y the MASK() 34 * operation returns 0xFF if the high bit of the byte is 1, 35 * or 0x00 if the high bit is 0. The result is stored in vector 36 * register x. 37 */ 38#define MASK(x, y) fpu_vesravb(x, y, 24) 39 40#define AND(x, y, z) fpu_vn(x, y, z) 41#define XOR(x, y, z) fpu_vx(x, y, z) 42#define LOAD_DATA(x, ptr) fpu_vlm(x, x + $# - 1, ptr) 43#define STORE_DATA(x, ptr) fpu_vstm(x, x + $# - 1, ptr) 44#define COPY_VEC(x, y) fpu_vlr(x, y) 45 46static void raid6_s390vx$#_gen_syndrome(int disks, size_t bytes, void **ptrs) 47{ 48 DECLARE_KERNEL_FPU_ONSTACK32(vxstate); 49 u8 **dptr, *p, *q; 50 int d, z, z0; 51 52 kernel_fpu_begin(&vxstate, KERNEL_VXR); 53 LOAD_CONST(); 54 55 dptr = (u8 **) ptrs; 56 z0 = disks - 3; /* Highest data disk */ 57 p = dptr[z0 + 1]; /* XOR parity */ 58 q = dptr[z0 + 2]; /* RS syndrome */ 59 60 for (d = 0; d < bytes; d += $#*NSIZE) { 61 LOAD_DATA(0,&dptr[z0][d]); 62 COPY_VEC(8+$$,0+$$); 63 for (z = z0 - 1; z >= 0; z--) { 64 MASK(16+$$,8+$$); 65 AND(16+$$,16+$$,25); 66 SHLBYTE(8+$$,8+$$); 67 XOR(8+$$,8+$$,16+$$); 68 LOAD_DATA(16,&dptr[z][d]); 69 XOR(0+$$,0+$$,16+$$); 70 XOR(8+$$,8+$$,16+$$); 71 } 72 STORE_DATA(0,&p[d]); 73 STORE_DATA(8,&q[d]); 74 } 75 kernel_fpu_end(&vxstate, KERNEL_VXR); 76} 77 78static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop, 79 size_t bytes, void **ptrs) 80{ 81 DECLARE_KERNEL_FPU_ONSTACK32(vxstate); 82 u8 **dptr, *p, *q; 83 int d, z, z0; 84 85 dptr = (u8 **) ptrs; 86 z0 = stop; /* P/Q right side optimization */ 87 p = dptr[disks - 2]; /* XOR parity */ 88 q = dptr[disks - 1]; /* RS syndrome */ 89 90 kernel_fpu_begin(&vxstate, KERNEL_VXR); 91 LOAD_CONST(); 92 93 for (d = 0; d < bytes; d += $#*NSIZE) { 94 /* P/Q data pages */ 95 LOAD_DATA(0,&dptr[z0][d]); 96 COPY_VEC(8+$$,0+$$); 97 for (z = z0 - 1; z >= start; z--) { 98 MASK(16+$$,8+$$); 99 AND(16+$$,16+$$,25); 100 SHLBYTE(8+$$,8+$$); 101 XOR(8+$$,8+$$,16+$$); 102 LOAD_DATA(16,&dptr[z][d]); 103 XOR(0+$$,0+$$,16+$$); 104 XOR(8+$$,8+$$,16+$$); 105 } 106 /* P/Q left side optimization */ 107 for (z = start - 1; z >= 0; z--) { 108 MASK(16+$$,8+$$); 109 AND(16+$$,16+$$,25); 110 SHLBYTE(8+$$,8+$$); 111 XOR(8+$$,8+$$,16+$$); 112 } 113 LOAD_DATA(16,&p[d]); 114 XOR(16+$$,16+$$,0+$$); 115 STORE_DATA(16,&p[d]); 116 LOAD_DATA(16,&q[d]); 117 XOR(16+$$,16+$$,8+$$); 118 STORE_DATA(16,&q[d]); 119 } 120 kernel_fpu_end(&vxstate, KERNEL_VXR); 121} 122 123static int raid6_s390vx$#_valid(void) 124{ 125 return cpu_has_vx(); 126} 127 128const struct raid6_calls raid6_s390vx$# = { 129 raid6_s390vx$#_gen_syndrome, 130 raid6_s390vx$#_xor_syndrome, 131 raid6_s390vx$#_valid, 132 "vx128x$#", 133 1 134}; 135