1 /* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #ifndef _ASM_BITOPS_H 10 #define _ASM_BITOPS_H 11 12 #ifndef _LINUX_BITOPS_H 13 #error only <linux/bitops.h> can be included directly 14 #endif 15 16 #ifndef __ASSEMBLY__ 17 18 #include <linux/types.h> 19 #include <linux/compiler.h> 20 #include <asm/barrier.h> 21 #ifndef CONFIG_ARC_HAS_LLSC 22 #include <asm/smp.h> 23 #endif 24 25 #if defined(CONFIG_ARC_HAS_LLSC) 26 27 /* 28 * Hardware assisted Atomic-R-M-W 29 */ 30 31 #define BIT_OP(op, c_op, asm_op) \ 32 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\ 33 { \ 34 unsigned int temp; \ 35 \ 36 m += nr >> 5; \ 37 \ 38 /* \ 39 * ARC ISA micro-optimization: \ 40 * \ 41 * Instructions dealing with bitpos only consider lower 5 bits \ 42 * e.g (x << 33) is handled like (x << 1) by ASL instruction \ 43 * (mem pointer still needs adjustment to point to next word) \ 44 * \ 45 * Hence the masking to clamp @nr arg can be elided in general. \ 46 * \ 47 * However if @nr is a constant (above assumed in a register), \ 48 * and greater than 31, gcc can optimize away (x << 33) to 0, \ 49 * as overflow, given the 32-bit ISA. Thus masking needs to be \ 50 * done for const @nr, but no code is generated due to gcc \ 51 * const prop. \ 52 */ \ 53 if (__builtin_constant_p(nr)) \ 54 nr &= 0x1f; \ 55 \ 56 __asm__ __volatile__( \ 57 "1: llock %0, [%1] \n" \ 58 " " #asm_op " %0, %0, %2 \n" \ 59 " scond %0, [%1] \n" \ 60 " bnz 1b \n" \ 61 : "=&r"(temp) /* Early clobber, to prevent reg reuse */ \ 62 : "r"(m), /* Not "m": llock only supports reg direct addr mode */ \ 63 "ir"(nr) \ 64 : "cc"); \ 65 } 66 67 /* 68 * Semantically: 69 * Test the bit 70 * if clear 71 * set it and return 0 (old value) 72 * else 73 * return 1 (old value). 74 * 75 * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally 76 * and the old value of bit is returned 77 */ 78 #define TEST_N_BIT_OP(op, c_op, asm_op) \ 79 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\ 80 { \ 81 unsigned long old, temp; \ 82 \ 83 m += nr >> 5; \ 84 \ 85 if (__builtin_constant_p(nr)) \ 86 nr &= 0x1f; \ 87 \ 88 /* \ 89 * Explicit full memory barrier needed before/after as \ 90 * LLOCK/SCOND themselves don't provide any such smenatic \ 91 */ \ 92 smp_mb(); \ 93 \ 94 __asm__ __volatile__( \ 95 "1: llock %0, [%2] \n" \ 96 " " #asm_op " %1, %0, %3 \n" \ 97 " scond %1, [%2] \n" \ 98 " bnz 1b \n" \ 99 : "=&r"(old), "=&r"(temp) \ 100 : "r"(m), "ir"(nr) \ 101 : "cc"); \ 102 \ 103 smp_mb(); \ 104 \ 105 return (old & (1 << nr)) != 0; \ 106 } 107 108 #else /* !CONFIG_ARC_HAS_LLSC */ 109 110 /* 111 * Non hardware assisted Atomic-R-M-W 112 * Locking would change to irq-disabling only (UP) and spinlocks (SMP) 113 * 114 * There's "significant" micro-optimization in writing our own variants of 115 * bitops (over generic variants) 116 * 117 * (1) The generic APIs have "signed" @nr while we have it "unsigned" 118 * This avoids extra code to be generated for pointer arithmatic, since 119 * is "not sure" that index is NOT -ve 120 * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc 121 * only consider bottom 5 bits of @nr, so NO need to mask them off. 122 * (GCC Quirk: however for constant @nr we still need to do the masking 123 * at compile time) 124 */ 125 126 #define BIT_OP(op, c_op, asm_op) \ 127 static inline void op##_bit(unsigned long nr, volatile unsigned long *m)\ 128 { \ 129 unsigned long temp, flags; \ 130 m += nr >> 5; \ 131 \ 132 if (__builtin_constant_p(nr)) \ 133 nr &= 0x1f; \ 134 \ 135 /* \ 136 * spin lock/unlock provide the needed smp_mb() before/after \ 137 */ \ 138 bitops_lock(flags); \ 139 \ 140 temp = *m; \ 141 *m = temp c_op (1UL << nr); \ 142 \ 143 bitops_unlock(flags); \ 144 } 145 146 #define TEST_N_BIT_OP(op, c_op, asm_op) \ 147 static inline int test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\ 148 { \ 149 unsigned long old, flags; \ 150 m += nr >> 5; \ 151 \ 152 if (__builtin_constant_p(nr)) \ 153 nr &= 0x1f; \ 154 \ 155 bitops_lock(flags); \ 156 \ 157 old = *m; \ 158 *m = old c_op (1 << nr); \ 159 \ 160 bitops_unlock(flags); \ 161 \ 162 return (old & (1 << nr)) != 0; \ 163 } 164 165 #endif /* CONFIG_ARC_HAS_LLSC */ 166 167 /*************************************** 168 * Non atomic variants 169 **************************************/ 170 171 #define __BIT_OP(op, c_op, asm_op) \ 172 static inline void __##op##_bit(unsigned long nr, volatile unsigned long *m) \ 173 { \ 174 unsigned long temp; \ 175 m += nr >> 5; \ 176 \ 177 if (__builtin_constant_p(nr)) \ 178 nr &= 0x1f; \ 179 \ 180 temp = *m; \ 181 *m = temp c_op (1UL << nr); \ 182 } 183 184 #define __TEST_N_BIT_OP(op, c_op, asm_op) \ 185 static inline int __test_and_##op##_bit(unsigned long nr, volatile unsigned long *m)\ 186 { \ 187 unsigned long old; \ 188 m += nr >> 5; \ 189 \ 190 if (__builtin_constant_p(nr)) \ 191 nr &= 0x1f; \ 192 \ 193 old = *m; \ 194 *m = old c_op (1 << nr); \ 195 \ 196 return (old & (1 << nr)) != 0; \ 197 } 198 199 #define BIT_OPS(op, c_op, asm_op) \ 200 \ 201 /* set_bit(), clear_bit(), change_bit() */ \ 202 BIT_OP(op, c_op, asm_op) \ 203 \ 204 /* test_and_set_bit(), test_and_clear_bit(), test_and_change_bit() */\ 205 TEST_N_BIT_OP(op, c_op, asm_op) \ 206 \ 207 /* __set_bit(), __clear_bit(), __change_bit() */ \ 208 __BIT_OP(op, c_op, asm_op) \ 209 \ 210 /* __test_and_set_bit(), __test_and_clear_bit(), __test_and_change_bit() */\ 211 __TEST_N_BIT_OP(op, c_op, asm_op) 212 213 BIT_OPS(set, |, bset) 214 BIT_OPS(clear, & ~, bclr) 215 BIT_OPS(change, ^, bxor) 216 217 /* 218 * This routine doesn't need to be atomic. 219 */ 220 static inline int 221 test_bit(unsigned int nr, const volatile unsigned long *addr) 222 { 223 unsigned long mask; 224 225 addr += nr >> 5; 226 227 if (__builtin_constant_p(nr)) 228 nr &= 0x1f; 229 230 mask = 1 << nr; 231 232 return ((mask & *addr) != 0); 233 } 234 235 #ifdef CONFIG_ISA_ARCOMPACT 236 237 /* 238 * Count the number of zeros, starting from MSB 239 * Helper for fls( ) friends 240 * This is a pure count, so (1-32) or (0-31) doesn't apply 241 * It could be 0 to 32, based on num of 0's in there 242 * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31 243 */ 244 static inline __attribute__ ((const)) int clz(unsigned int x) 245 { 246 unsigned int res; 247 248 __asm__ __volatile__( 249 " norm.f %0, %1 \n" 250 " mov.n %0, 0 \n" 251 " add.p %0, %0, 1 \n" 252 : "=r"(res) 253 : "r"(x) 254 : "cc"); 255 256 return res; 257 } 258 259 static inline int constant_fls(int x) 260 { 261 int r = 32; 262 263 if (!x) 264 return 0; 265 if (!(x & 0xffff0000u)) { 266 x <<= 16; 267 r -= 16; 268 } 269 if (!(x & 0xff000000u)) { 270 x <<= 8; 271 r -= 8; 272 } 273 if (!(x & 0xf0000000u)) { 274 x <<= 4; 275 r -= 4; 276 } 277 if (!(x & 0xc0000000u)) { 278 x <<= 2; 279 r -= 2; 280 } 281 if (!(x & 0x80000000u)) { 282 x <<= 1; 283 r -= 1; 284 } 285 return r; 286 } 287 288 /* 289 * fls = Find Last Set in word 290 * @result: [1-32] 291 * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0 292 */ 293 static inline __attribute__ ((const)) int fls(unsigned long x) 294 { 295 if (__builtin_constant_p(x)) 296 return constant_fls(x); 297 298 return 32 - clz(x); 299 } 300 301 /* 302 * __fls: Similar to fls, but zero based (0-31) 303 */ 304 static inline __attribute__ ((const)) int __fls(unsigned long x) 305 { 306 if (!x) 307 return 0; 308 else 309 return fls(x) - 1; 310 } 311 312 /* 313 * ffs = Find First Set in word (LSB to MSB) 314 * @result: [1-32], 0 if all 0's 315 */ 316 #define ffs(x) ({ unsigned long __t = (x); fls(__t & -__t); }) 317 318 /* 319 * __ffs: Similar to ffs, but zero based (0-31) 320 */ 321 static inline __attribute__ ((const)) int __ffs(unsigned long word) 322 { 323 if (!word) 324 return word; 325 326 return ffs(word) - 1; 327 } 328 329 #else /* CONFIG_ISA_ARCV2 */ 330 331 /* 332 * fls = Find Last Set in word 333 * @result: [1-32] 334 * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0 335 */ 336 static inline __attribute__ ((const)) int fls(unsigned long x) 337 { 338 int n; 339 340 asm volatile( 341 " fls.f %0, %1 \n" /* 0:31; 0(Z) if src 0 */ 342 " add.nz %0, %0, 1 \n" /* 0:31 -> 1:32 */ 343 : "=r"(n) /* Early clobber not needed */ 344 : "r"(x) 345 : "cc"); 346 347 return n; 348 } 349 350 /* 351 * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set 352 */ 353 static inline __attribute__ ((const)) int __fls(unsigned long x) 354 { 355 /* FLS insn has exactly same semantics as the API */ 356 return __builtin_arc_fls(x); 357 } 358 359 /* 360 * ffs = Find First Set in word (LSB to MSB) 361 * @result: [1-32], 0 if all 0's 362 */ 363 static inline __attribute__ ((const)) int ffs(unsigned long x) 364 { 365 int n; 366 367 asm volatile( 368 " ffs.f %0, %1 \n" /* 0:31; 31(Z) if src 0 */ 369 " add.nz %0, %0, 1 \n" /* 0:31 -> 1:32 */ 370 " mov.z %0, 0 \n" /* 31(Z)-> 0 */ 371 : "=r"(n) /* Early clobber not needed */ 372 : "r"(x) 373 : "cc"); 374 375 return n; 376 } 377 378 /* 379 * __ffs: Similar to ffs, but zero based (0-31) 380 */ 381 static inline __attribute__ ((const)) int __ffs(unsigned long x) 382 { 383 int n; 384 385 asm volatile( 386 " ffs.f %0, %1 \n" /* 0:31; 31(Z) if src 0 */ 387 " mov.z %0, 0 \n" /* 31(Z)-> 0 */ 388 : "=r"(n) 389 : "r"(x) 390 : "cc"); 391 392 return n; 393 394 } 395 396 #endif /* CONFIG_ISA_ARCOMPACT */ 397 398 /* 399 * ffz = Find First Zero in word. 400 * @return:[0-31], 32 if all 1's 401 */ 402 #define ffz(x) __ffs(~(x)) 403 404 #include <asm-generic/bitops/hweight.h> 405 #include <asm-generic/bitops/fls64.h> 406 #include <asm-generic/bitops/sched.h> 407 #include <asm-generic/bitops/lock.h> 408 409 #include <asm-generic/bitops/find.h> 410 #include <asm-generic/bitops/le.h> 411 #include <asm-generic/bitops/ext2-atomic-setbit.h> 412 413 #endif /* !__ASSEMBLY__ */ 414 415 #endif 416