1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_WORD_AT_A_TIME_H 3 #define _ASM_WORD_AT_A_TIME_H 4 5 #include <linux/kernel.h> 6 #include <asm/byteorder.h> 7 8 #ifdef __BIG_ENDIAN 9 10 struct word_at_a_time { 11 const unsigned long high_bits, low_bits; 12 }; 13 14 #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0xfe) + 1, REPEAT_BYTE(0x7f) } 15 16 /* Bit set in the bytes that have a zero */ 17 static inline long prep_zero_mask(unsigned long val, unsigned long rhs, const struct word_at_a_time *c) 18 { 19 unsigned long mask = (val & c->low_bits) + c->low_bits; 20 return ~(mask | rhs); 21 } 22 23 #define create_zero_mask(mask) (mask) 24 25 static inline long find_zero(unsigned long mask) 26 { 27 long byte = 0; 28 #ifdef CONFIG_64BIT 29 if (mask >> 32) 30 mask >>= 32; 31 else 32 byte = 4; 33 #endif 34 if (mask >> 16) 35 mask >>= 16; 36 else 37 byte += 2; 38 return (mask >> 8) ? byte : byte + 1; 39 } 40 41 static inline unsigned long has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c) 42 { 43 unsigned long rhs = val | c->low_bits; 44 *data = rhs; 45 return (val + c->high_bits) & ~rhs; 46 } 47 48 #ifndef zero_bytemask 49 #define zero_bytemask(mask) (~1ul << __fls(mask)) 50 #endif 51 52 #else 53 54 /* 55 * The optimal byte mask counting is probably going to be something 56 * that is architecture-specific. If you have a reliably fast 57 * bit count instruction, that might be better than the multiply 58 * and shift, for example. 59 */ 60 struct word_at_a_time { 61 const unsigned long one_bits, high_bits; 62 }; 63 64 #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) } 65 66 #ifdef CONFIG_64BIT 67 68 /* 69 * Jan Achrenius on G+: microoptimized version of 70 * the simpler "(mask & ONEBYTES) * ONEBYTES >> 56" 71 * that works for the bytemasks without having to 72 * mask them first. 73 */ 74 static inline long count_masked_bytes(unsigned long mask) 75 { 76 return mask*0x0001020304050608ul >> 56; 77 } 78 79 #else /* 32-bit case */ 80 81 /* Carl Chatfield / Jan Achrenius G+ version for 32-bit */ 82 static inline long count_masked_bytes(long mask) 83 { 84 /* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */ 85 long a = (0x0ff0001+mask) >> 23; 86 /* Fix the 1 for 00 case */ 87 return a & mask; 88 } 89 90 #endif 91 92 /* Return nonzero if it has a zero */ 93 static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c) 94 { 95 unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits; 96 *bits = mask; 97 return mask; 98 } 99 100 static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c) 101 { 102 return bits; 103 } 104 105 static inline unsigned long create_zero_mask(unsigned long bits) 106 { 107 bits = (bits - 1) & ~bits; 108 return bits >> 7; 109 } 110 111 /* The mask we created is directly usable as a bytemask */ 112 #define zero_bytemask(mask) (mask) 113 114 static inline unsigned long find_zero(unsigned long mask) 115 { 116 return count_masked_bytes(mask); 117 } 118 119 #endif /* __BIG_ENDIAN */ 120 121 #endif /* _ASM_WORD_AT_A_TIME_H */ 122