1 /* 2 * Copyright (C) 2013 ARM Ltd. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * along with this program. If not, see <http://www.gnu.org/licenses/>. 15 */ 16 #ifndef __ASM_WORD_AT_A_TIME_H 17 #define __ASM_WORD_AT_A_TIME_H 18 19 #ifndef __AARCH64EB__ 20 21 #include <linux/kernel.h> 22 23 struct word_at_a_time { 24 const unsigned long one_bits, high_bits; 25 }; 26 27 #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) } 28 29 static inline unsigned long has_zero(unsigned long a, unsigned long *bits, 30 const struct word_at_a_time *c) 31 { 32 unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits; 33 *bits = mask; 34 return mask; 35 } 36 37 #define prep_zero_mask(a, bits, c) (bits) 38 39 static inline unsigned long create_zero_mask(unsigned long bits) 40 { 41 bits = (bits - 1) & ~bits; 42 return bits >> 7; 43 } 44 45 static inline unsigned long find_zero(unsigned long mask) 46 { 47 return fls64(mask) >> 3; 48 } 49 50 #define zero_bytemask(mask) (mask) 51 52 #else /* __AARCH64EB__ */ 53 #include <asm-generic/word-at-a-time.h> 54 #endif 55 56 /* 57 * Load an unaligned word from kernel space. 58 * 59 * In the (very unlikely) case of the word being a page-crosser 60 * and the next page not being mapped, take the exception and 61 * return zeroes in the non-existing part. 62 */ 63 static inline unsigned long load_unaligned_zeropad(const void *addr) 64 { 65 unsigned long ret, offset; 66 67 /* Load word from unaligned pointer addr */ 68 asm( 69 "1: ldr %0, %3\n" 70 "2:\n" 71 " .pushsection .fixup,\"ax\"\n" 72 " .align 2\n" 73 "3: and %1, %2, #0x7\n" 74 " bic %2, %2, #0x7\n" 75 " ldr %0, [%2]\n" 76 " lsl %1, %1, #0x3\n" 77 #ifndef __AARCH64EB__ 78 " lsr %0, %0, %1\n" 79 #else 80 " lsl %0, %0, %1\n" 81 #endif 82 " b 2b\n" 83 " .popsection\n" 84 " .pushsection __ex_table,\"a\"\n" 85 " .align 3\n" 86 " .quad 1b, 3b\n" 87 " .popsection" 88 : "=&r" (ret), "=&r" (offset) 89 : "r" (addr), "Q" (*(unsigned long *)addr)); 90 91 return ret; 92 } 93 94 #endif /* __ASM_WORD_AT_A_TIME_H */ 95