1 /* 2 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 */ 8 9 #ifndef _ASM_BITOPS_H 10 #define _ASM_BITOPS_H 11 12 #ifndef _LINUX_BITOPS_H 13 #error only <linux/bitops.h> can be included directly 14 #endif 15 16 #ifndef __ASSEMBLY__ 17 18 #include <linux/types.h> 19 #include <linux/compiler.h> 20 #include <asm/barrier.h> 21 22 /* 23 * Hardware assisted read-modify-write using ARC700 LLOCK/SCOND insns. 24 * The Kconfig glue ensures that in SMP, this is only set if the container 25 * SoC/platform has cross-core coherent LLOCK/SCOND 26 */ 27 #if defined(CONFIG_ARC_HAS_LLSC) 28 29 static inline void set_bit(unsigned long nr, volatile unsigned long *m) 30 { 31 unsigned int temp; 32 33 m += nr >> 5; 34 35 /* 36 * ARC ISA micro-optimization: 37 * 38 * Instructions dealing with bitpos only consider lower 5 bits (0-31) 39 * e.g (x << 33) is handled like (x << 1) by ASL instruction 40 * (mem pointer still needs adjustment to point to next word) 41 * 42 * Hence the masking to clamp @nr arg can be elided in general. 43 * 44 * However if @nr is a constant (above assumed it in a register), 45 * and greater than 31, gcc can optimize away (x << 33) to 0, 46 * as overflow, given the 32-bit ISA. Thus masking needs to be done 47 * for constant @nr, but no code is generated due to const prop. 48 */ 49 if (__builtin_constant_p(nr)) 50 nr &= 0x1f; 51 52 __asm__ __volatile__( 53 "1: llock %0, [%1] \n" 54 " bset %0, %0, %2 \n" 55 " scond %0, [%1] \n" 56 " bnz 1b \n" 57 : "=&r"(temp) 58 : "r"(m), "ir"(nr) 59 : "cc"); 60 } 61 62 static inline void clear_bit(unsigned long nr, volatile unsigned long *m) 63 { 64 unsigned int temp; 65 66 m += nr >> 5; 67 68 if (__builtin_constant_p(nr)) 69 nr &= 0x1f; 70 71 __asm__ __volatile__( 72 "1: llock %0, [%1] \n" 73 " bclr %0, %0, %2 \n" 74 " scond %0, [%1] \n" 75 " bnz 1b \n" 76 : "=&r"(temp) 77 : "r"(m), "ir"(nr) 78 : "cc"); 79 } 80 81 static inline void change_bit(unsigned long nr, volatile unsigned long *m) 82 { 83 unsigned int temp; 84 85 m += nr >> 5; 86 87 if (__builtin_constant_p(nr)) 88 nr &= 0x1f; 89 90 __asm__ __volatile__( 91 "1: llock %0, [%1] \n" 92 " bxor %0, %0, %2 \n" 93 " scond %0, [%1] \n" 94 " bnz 1b \n" 95 : "=&r"(temp) 96 : "r"(m), "ir"(nr) 97 : "cc"); 98 } 99 100 /* 101 * Semantically: 102 * Test the bit 103 * if clear 104 * set it and return 0 (old value) 105 * else 106 * return 1 (old value). 107 * 108 * Since ARC lacks a equivalent h/w primitive, the bit is set unconditionally 109 * and the old value of bit is returned 110 */ 111 static inline int test_and_set_bit(unsigned long nr, volatile unsigned long *m) 112 { 113 unsigned long old, temp; 114 115 m += nr >> 5; 116 117 if (__builtin_constant_p(nr)) 118 nr &= 0x1f; 119 120 __asm__ __volatile__( 121 "1: llock %0, [%2] \n" 122 " bset %1, %0, %3 \n" 123 " scond %1, [%2] \n" 124 " bnz 1b \n" 125 : "=&r"(old), "=&r"(temp) 126 : "r"(m), "ir"(nr) 127 : "cc"); 128 129 return (old & (1 << nr)) != 0; 130 } 131 132 static inline int 133 test_and_clear_bit(unsigned long nr, volatile unsigned long *m) 134 { 135 unsigned int old, temp; 136 137 m += nr >> 5; 138 139 if (__builtin_constant_p(nr)) 140 nr &= 0x1f; 141 142 __asm__ __volatile__( 143 "1: llock %0, [%2] \n" 144 " bclr %1, %0, %3 \n" 145 " scond %1, [%2] \n" 146 " bnz 1b \n" 147 : "=&r"(old), "=&r"(temp) 148 : "r"(m), "ir"(nr) 149 : "cc"); 150 151 return (old & (1 << nr)) != 0; 152 } 153 154 static inline int 155 test_and_change_bit(unsigned long nr, volatile unsigned long *m) 156 { 157 unsigned int old, temp; 158 159 m += nr >> 5; 160 161 if (__builtin_constant_p(nr)) 162 nr &= 0x1f; 163 164 __asm__ __volatile__( 165 "1: llock %0, [%2] \n" 166 " bxor %1, %0, %3 \n" 167 " scond %1, [%2] \n" 168 " bnz 1b \n" 169 : "=&r"(old), "=&r"(temp) 170 : "r"(m), "ir"(nr) 171 : "cc"); 172 173 return (old & (1 << nr)) != 0; 174 } 175 176 #else /* !CONFIG_ARC_HAS_LLSC */ 177 178 #include <asm/smp.h> 179 180 /* 181 * Non hardware assisted Atomic-R-M-W 182 * Locking would change to irq-disabling only (UP) and spinlocks (SMP) 183 * 184 * There's "significant" micro-optimization in writing our own variants of 185 * bitops (over generic variants) 186 * 187 * (1) The generic APIs have "signed" @nr while we have it "unsigned" 188 * This avoids extra code to be generated for pointer arithmatic, since 189 * is "not sure" that index is NOT -ve 190 * (2) Utilize the fact that ARCompact bit fidding insn (BSET/BCLR/ASL) etc 191 * only consider bottom 5 bits of @nr, so NO need to mask them off. 192 * (GCC Quirk: however for constant @nr we still need to do the masking 193 * at compile time) 194 */ 195 196 static inline void set_bit(unsigned long nr, volatile unsigned long *m) 197 { 198 unsigned long temp, flags; 199 m += nr >> 5; 200 201 if (__builtin_constant_p(nr)) 202 nr &= 0x1f; 203 204 bitops_lock(flags); 205 206 temp = *m; 207 *m = temp | (1UL << nr); 208 209 bitops_unlock(flags); 210 } 211 212 static inline void clear_bit(unsigned long nr, volatile unsigned long *m) 213 { 214 unsigned long temp, flags; 215 m += nr >> 5; 216 217 if (__builtin_constant_p(nr)) 218 nr &= 0x1f; 219 220 bitops_lock(flags); 221 222 temp = *m; 223 *m = temp & ~(1UL << nr); 224 225 bitops_unlock(flags); 226 } 227 228 static inline void change_bit(unsigned long nr, volatile unsigned long *m) 229 { 230 unsigned long temp, flags; 231 m += nr >> 5; 232 233 if (__builtin_constant_p(nr)) 234 nr &= 0x1f; 235 236 bitops_lock(flags); 237 238 temp = *m; 239 *m = temp ^ (1UL << nr); 240 241 bitops_unlock(flags); 242 } 243 244 static inline int test_and_set_bit(unsigned long nr, volatile unsigned long *m) 245 { 246 unsigned long old, flags; 247 m += nr >> 5; 248 249 if (__builtin_constant_p(nr)) 250 nr &= 0x1f; 251 252 bitops_lock(flags); 253 254 old = *m; 255 *m = old | (1 << nr); 256 257 bitops_unlock(flags); 258 259 return (old & (1 << nr)) != 0; 260 } 261 262 static inline int 263 test_and_clear_bit(unsigned long nr, volatile unsigned long *m) 264 { 265 unsigned long old, flags; 266 m += nr >> 5; 267 268 if (__builtin_constant_p(nr)) 269 nr &= 0x1f; 270 271 bitops_lock(flags); 272 273 old = *m; 274 *m = old & ~(1 << nr); 275 276 bitops_unlock(flags); 277 278 return (old & (1 << nr)) != 0; 279 } 280 281 static inline int 282 test_and_change_bit(unsigned long nr, volatile unsigned long *m) 283 { 284 unsigned long old, flags; 285 m += nr >> 5; 286 287 if (__builtin_constant_p(nr)) 288 nr &= 0x1f; 289 290 bitops_lock(flags); 291 292 old = *m; 293 *m = old ^ (1 << nr); 294 295 bitops_unlock(flags); 296 297 return (old & (1 << nr)) != 0; 298 } 299 300 #endif /* CONFIG_ARC_HAS_LLSC */ 301 302 /*************************************** 303 * Non atomic variants 304 **************************************/ 305 306 static inline void __set_bit(unsigned long nr, volatile unsigned long *m) 307 { 308 unsigned long temp; 309 m += nr >> 5; 310 311 if (__builtin_constant_p(nr)) 312 nr &= 0x1f; 313 314 temp = *m; 315 *m = temp | (1UL << nr); 316 } 317 318 static inline void __clear_bit(unsigned long nr, volatile unsigned long *m) 319 { 320 unsigned long temp; 321 m += nr >> 5; 322 323 if (__builtin_constant_p(nr)) 324 nr &= 0x1f; 325 326 temp = *m; 327 *m = temp & ~(1UL << nr); 328 } 329 330 static inline void __change_bit(unsigned long nr, volatile unsigned long *m) 331 { 332 unsigned long temp; 333 m += nr >> 5; 334 335 if (__builtin_constant_p(nr)) 336 nr &= 0x1f; 337 338 temp = *m; 339 *m = temp ^ (1UL << nr); 340 } 341 342 static inline int 343 __test_and_set_bit(unsigned long nr, volatile unsigned long *m) 344 { 345 unsigned long old; 346 m += nr >> 5; 347 348 if (__builtin_constant_p(nr)) 349 nr &= 0x1f; 350 351 old = *m; 352 *m = old | (1 << nr); 353 354 return (old & (1 << nr)) != 0; 355 } 356 357 static inline int 358 __test_and_clear_bit(unsigned long nr, volatile unsigned long *m) 359 { 360 unsigned long old; 361 m += nr >> 5; 362 363 if (__builtin_constant_p(nr)) 364 nr &= 0x1f; 365 366 old = *m; 367 *m = old & ~(1 << nr); 368 369 return (old & (1 << nr)) != 0; 370 } 371 372 static inline int 373 __test_and_change_bit(unsigned long nr, volatile unsigned long *m) 374 { 375 unsigned long old; 376 m += nr >> 5; 377 378 if (__builtin_constant_p(nr)) 379 nr &= 0x1f; 380 381 old = *m; 382 *m = old ^ (1 << nr); 383 384 return (old & (1 << nr)) != 0; 385 } 386 387 /* 388 * This routine doesn't need to be atomic. 389 */ 390 static inline int 391 test_bit(unsigned int nr, const volatile unsigned long *addr) 392 { 393 unsigned long mask; 394 395 addr += nr >> 5; 396 397 if (__builtin_constant_p(nr)) 398 nr &= 0x1f; 399 400 mask = 1 << nr; 401 402 return ((mask & *addr) != 0); 403 } 404 405 #ifdef CONFIG_ISA_ARCOMPACT 406 407 /* 408 * Count the number of zeros, starting from MSB 409 * Helper for fls( ) friends 410 * This is a pure count, so (1-32) or (0-31) doesn't apply 411 * It could be 0 to 32, based on num of 0's in there 412 * clz(0x8000_0000) = 0, clz(0xFFFF_FFFF)=0, clz(0) = 32, clz(1) = 31 413 */ 414 static inline __attribute__ ((const)) int clz(unsigned int x) 415 { 416 unsigned int res; 417 418 __asm__ __volatile__( 419 " norm.f %0, %1 \n" 420 " mov.n %0, 0 \n" 421 " add.p %0, %0, 1 \n" 422 : "=r"(res) 423 : "r"(x) 424 : "cc"); 425 426 return res; 427 } 428 429 static inline int constant_fls(int x) 430 { 431 int r = 32; 432 433 if (!x) 434 return 0; 435 if (!(x & 0xffff0000u)) { 436 x <<= 16; 437 r -= 16; 438 } 439 if (!(x & 0xff000000u)) { 440 x <<= 8; 441 r -= 8; 442 } 443 if (!(x & 0xf0000000u)) { 444 x <<= 4; 445 r -= 4; 446 } 447 if (!(x & 0xc0000000u)) { 448 x <<= 2; 449 r -= 2; 450 } 451 if (!(x & 0x80000000u)) { 452 x <<= 1; 453 r -= 1; 454 } 455 return r; 456 } 457 458 /* 459 * fls = Find Last Set in word 460 * @result: [1-32] 461 * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0 462 */ 463 static inline __attribute__ ((const)) int fls(unsigned long x) 464 { 465 if (__builtin_constant_p(x)) 466 return constant_fls(x); 467 468 return 32 - clz(x); 469 } 470 471 /* 472 * __fls: Similar to fls, but zero based (0-31) 473 */ 474 static inline __attribute__ ((const)) int __fls(unsigned long x) 475 { 476 if (!x) 477 return 0; 478 else 479 return fls(x) - 1; 480 } 481 482 /* 483 * ffs = Find First Set in word (LSB to MSB) 484 * @result: [1-32], 0 if all 0's 485 */ 486 #define ffs(x) ({ unsigned long __t = (x); fls(__t & -__t); }) 487 488 /* 489 * __ffs: Similar to ffs, but zero based (0-31) 490 */ 491 static inline __attribute__ ((const)) int __ffs(unsigned long word) 492 { 493 if (!word) 494 return word; 495 496 return ffs(word) - 1; 497 } 498 499 #else /* CONFIG_ISA_ARCV2 */ 500 501 /* 502 * fls = Find Last Set in word 503 * @result: [1-32] 504 * fls(1) = 1, fls(0x80000000) = 32, fls(0) = 0 505 */ 506 static inline __attribute__ ((const)) int fls(unsigned long x) 507 { 508 int n; 509 510 asm volatile( 511 " fls.f %0, %1 \n" /* 0:31; 0(Z) if src 0 */ 512 " add.nz %0, %0, 1 \n" /* 0:31 -> 1:32 */ 513 : "=r"(n) /* Early clobber not needed */ 514 : "r"(x) 515 : "cc"); 516 517 return n; 518 } 519 520 /* 521 * __fls: Similar to fls, but zero based (0-31). Also 0 if no bit set 522 */ 523 static inline __attribute__ ((const)) int __fls(unsigned long x) 524 { 525 /* FLS insn has exactly same semantics as the API */ 526 return __builtin_arc_fls(x); 527 } 528 529 /* 530 * ffs = Find First Set in word (LSB to MSB) 531 * @result: [1-32], 0 if all 0's 532 */ 533 static inline __attribute__ ((const)) int ffs(unsigned long x) 534 { 535 int n; 536 537 asm volatile( 538 " ffs.f %0, %1 \n" /* 0:31; 31(Z) if src 0 */ 539 " add.nz %0, %0, 1 \n" /* 0:31 -> 1:32 */ 540 " mov.z %0, 0 \n" /* 31(Z)-> 0 */ 541 : "=r"(n) /* Early clobber not needed */ 542 : "r"(x) 543 : "cc"); 544 545 return n; 546 } 547 548 /* 549 * __ffs: Similar to ffs, but zero based (0-31) 550 */ 551 static inline __attribute__ ((const)) int __ffs(unsigned long x) 552 { 553 int n; 554 555 asm volatile( 556 " ffs.f %0, %1 \n" /* 0:31; 31(Z) if src 0 */ 557 " mov.z %0, 0 \n" /* 31(Z)-> 0 */ 558 : "=r"(n) 559 : "r"(x) 560 : "cc"); 561 562 return n; 563 564 } 565 566 #endif /* CONFIG_ISA_ARCOMPACT */ 567 568 /* 569 * ffz = Find First Zero in word. 570 * @return:[0-31], 32 if all 1's 571 */ 572 #define ffz(x) __ffs(~(x)) 573 574 #include <asm-generic/bitops/hweight.h> 575 #include <asm-generic/bitops/fls64.h> 576 #include <asm-generic/bitops/sched.h> 577 #include <asm-generic/bitops/lock.h> 578 579 #include <asm-generic/bitops/find.h> 580 #include <asm-generic/bitops/le.h> 581 #include <asm-generic/bitops/ext2-atomic-setbit.h> 582 583 #endif /* !__ASSEMBLY__ */ 584 585 #endif 586