arch/mips/include/asm/bitops.h

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (c) 1994 - 1997, 99, 2000, 06, 07  Ralf Baechle (ralf@linux-mips.org)
   7  * Copyright (c) 1999, 2000  Silicon Graphics, Inc.
   8  */
   9 #ifndef _ASM_BITOPS_H
  10 #define _ASM_BITOPS_H
  11
  12 #ifndef _LINUX_BITOPS_H
  13 #error only <linux/bitops.h> can be included directly
  14 #endif
  15
  16 #include <linux/compiler.h>
  17 #include <linux/irqflags.h>
  18 #include <linux/types.h>
  19 #include <asm/barrier.h>
  20 #include <asm/bug.h>
  21 #include <asm/byteorder.h>              /* sigh ... */
  22 #include <asm/cpu-features.h>
  23 #include <asm/sgidefs.h>
  24 #include <asm/war.h>
  25
  26 #if _MIPS_SZLONG == 32
  27 #define SZLONG_LOG 5
  28 #define SZLONG_MASK 31UL
  29 #define __LL            "ll     "
  30 #define __SC            "sc     "
  31 #define __INS           "ins    "
  32 #define __EXT           "ext    "
  33 #elif _MIPS_SZLONG == 64
  34 #define SZLONG_LOG 6
  35 #define SZLONG_MASK 63UL
  36 #define __LL            "lld    "
  37 #define __SC            "scd    "
  38 #define __INS           "dins    "
  39 #define __EXT           "dext    "
  40 #endif
  41
  42 /*
  43  * clear_bit() doesn't provide any barrier for the compiler.
  44  */
  45 #define smp_mb__before_clear_bit()      smp_llsc_mb()
  46 #define smp_mb__after_clear_bit()       smp_llsc_mb()
  47
  48 /*
  49  * set_bit - Atomically set a bit in memory
  50  * @nr: the bit to set
  51  * @addr: the address to start counting from
  52  *
  53  * This function is atomic and may not be reordered.  See __set_bit()
  54  * if you do not require the atomic guarantees.
  55  * Note that @nr may be almost arbitrarily large; this function is not
  56  * restricted to acting on a single-word quantity.
  57  */
  58 static inline void set_bit(unsigned long nr, volatile unsigned long *addr)
  59 {
  60         unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
  61         unsigned short bit = nr & SZLONG_MASK;
  62         unsigned long temp;
  63
  64         if (cpu_has_llsc && R10000_LLSC_WAR) {
  65                 __asm__ __volatile__(
  66                 "       .set    mips3                                   \n"
  67                 "1:     " __LL "%0, %1                  # set_bit       \n"
  68                 "       or      %0, %2                                  \n"
  69                 "       " __SC  "%0, %1                                 \n"
  70                 "       beqzl   %0, 1b                                  \n"
  71                 "       .set    mips0                                   \n"
  72                 : "=&r" (temp), "=m" (*m)
  73                 : "ir" (1UL << bit), "m" (*m));
  74 #ifdef CONFIG_CPU_MIPSR2
  75         } else if (__builtin_constant_p(bit)) {
  76                 __asm__ __volatile__(
  77                 "1:     " __LL "%0, %1                  # set_bit       \n"
  78                 "       " __INS "%0, %4, %2, 1                          \n"
  79                 "       " __SC "%0, %1                                  \n"
  80                 "       beqz    %0, 2f                                  \n"
  81                 "       .subsection 2                                   \n"
  82                 "2:     b       1b                                      \n"
  83                 "       .previous                                       \n"
  84                 : "=&r" (temp), "=m" (*m)
  85                 : "ir" (bit), "m" (*m), "r" (~0));
  86 #endif /* CONFIG_CPU_MIPSR2 */
  87         } else if (cpu_has_llsc) {
  88                 __asm__ __volatile__(
  89                 "       .set    mips3                                   \n"
  90                 "1:     " __LL "%0, %1                  # set_bit       \n"
  91                 "       or      %0, %2                                  \n"
  92                 "       " __SC  "%0, %1                                 \n"
  93                 "       beqz    %0, 2f                                  \n"
  94                 "       .subsection 2                                   \n"
  95                 "2:     b       1b                                      \n"
  96                 "       .previous                                       \n"
  97                 "       .set    mips0                                   \n"
  98                 : "=&r" (temp), "=m" (*m)
  99                 : "ir" (1UL << bit), "m" (*m));
 100         } else {
 101                 volatile unsigned long *a = addr;
 102                 unsigned long mask;
 103                 unsigned long flags;
 104
 105                 a += nr >> SZLONG_LOG;
 106                 mask = 1UL << bit;
 107                 raw_local_irq_save(flags);
 108                 *a |= mask;
 109                 raw_local_irq_restore(flags);
 110         }
 111 }
 112
 113 /*
 114  * clear_bit - Clears a bit in memory
 115  * @nr: Bit to clear
 116  * @addr: Address to start counting from
 117  *
 118  * clear_bit() is atomic and may not be reordered.  However, it does
 119  * not contain a memory barrier, so if it is used for locking purposes,
 120  * you should call smp_mb__before_clear_bit() and/or smp_mb__after_clear_bit()
 121  * in order to ensure changes are visible on other processors.
 122  */
 123 static inline void clear_bit(unsigned long nr, volatile unsigned long *addr)
 124 {
 125         unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 126         unsigned short bit = nr & SZLONG_MASK;
 127         unsigned long temp;
 128
 129         if (cpu_has_llsc && R10000_LLSC_WAR) {
 130                 __asm__ __volatile__(
 131                 "       .set    mips3                                   \n"
 132                 "1:     " __LL "%0, %1                  # clear_bit     \n"
 133                 "       and     %0, %2                                  \n"
 134                 "       " __SC "%0, %1                                  \n"
 135                 "       beqzl   %0, 1b                                  \n"
 136                 "       .set    mips0                                   \n"
 137                 : "=&r" (temp), "=m" (*m)
 138                 : "ir" (~(1UL << bit)), "m" (*m));
 139 #ifdef CONFIG_CPU_MIPSR2
 140         } else if (__builtin_constant_p(bit)) {
 141                 __asm__ __volatile__(
 142                 "1:     " __LL "%0, %1                  # clear_bit     \n"
 143                 "       " __INS "%0, $0, %2, 1                          \n"
 144                 "       " __SC "%0, %1                                  \n"
 145                 "       beqz    %0, 2f                                  \n"
 146                 "       .subsection 2                                   \n"
 147                 "2:     b       1b                                      \n"
 148                 "       .previous                                       \n"
 149                 : "=&r" (temp), "=m" (*m)
 150                 : "ir" (bit), "m" (*m));
 151 #endif /* CONFIG_CPU_MIPSR2 */
 152         } else if (cpu_has_llsc) {
 153                 __asm__ __volatile__(
 154                 "       .set    mips3                                   \n"
 155                 "1:     " __LL "%0, %1                  # clear_bit     \n"
 156                 "       and     %0, %2                                  \n"
 157                 "       " __SC "%0, %1                                  \n"
 158                 "       beqz    %0, 2f                                  \n"
 159                 "       .subsection 2                                   \n"
 160                 "2:     b       1b                                      \n"
 161                 "       .previous                                       \n"
 162                 "       .set    mips0                                   \n"
 163                 : "=&r" (temp), "=m" (*m)
 164                 : "ir" (~(1UL << bit)), "m" (*m));
 165         } else {
 166                 volatile unsigned long *a = addr;
 167                 unsigned long mask;
 168                 unsigned long flags;
 169
 170                 a += nr >> SZLONG_LOG;
 171                 mask = 1UL << bit;
 172                 raw_local_irq_save(flags);
 173                 *a &= ~mask;
 174                 raw_local_irq_restore(flags);
 175         }
 176 }
 177
 178 /*
 179  * clear_bit_unlock - Clears a bit in memory
 180  * @nr: Bit to clear
 181  * @addr: Address to start counting from
 182  *
 183  * clear_bit() is atomic and implies release semantics before the memory
 184  * operation. It can be used for an unlock.
 185  */
 186 static inline void clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
 187 {
 188         smp_mb__before_clear_bit();
 189         clear_bit(nr, addr);
 190 }
 191
 192 /*
 193  * change_bit - Toggle a bit in memory
 194  * @nr: Bit to change
 195  * @addr: Address to start counting from
 196  *
 197  * change_bit() is atomic and may not be reordered.
 198  * Note that @nr may be almost arbitrarily large; this function is not
 199  * restricted to acting on a single-word quantity.
 200  */
 201 static inline void change_bit(unsigned long nr, volatile unsigned long *addr)
 202 {
 203         unsigned short bit = nr & SZLONG_MASK;
 204
 205         if (cpu_has_llsc && R10000_LLSC_WAR) {
 206                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 207                 unsigned long temp;
 208
 209                 __asm__ __volatile__(
 210                 "       .set    mips3                           \n"
 211                 "1:     " __LL "%0, %1          # change_bit    \n"
 212                 "       xor     %0, %2                          \n"
 213                 "       " __SC  "%0, %1                         \n"
 214                 "       beqzl   %0, 1b                          \n"
 215                 "       .set    mips0                           \n"
 216                 : "=&r" (temp), "=m" (*m)
 217                 : "ir" (1UL << bit), "m" (*m));
 218         } else if (cpu_has_llsc) {
 219                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 220                 unsigned long temp;
 221
 222                 __asm__ __volatile__(
 223                 "       .set    mips3                           \n"
 224                 "1:     " __LL "%0, %1          # change_bit    \n"
 225                 "       xor     %0, %2                          \n"
 226                 "       " __SC  "%0, %1                         \n"
 227                 "       beqz    %0, 2f                          \n"
 228                 "       .subsection 2                           \n"
 229                 "2:     b       1b                              \n"
 230                 "       .previous                               \n"
 231                 "       .set    mips0                           \n"
 232                 : "=&r" (temp), "=m" (*m)
 233                 : "ir" (1UL << bit), "m" (*m));
 234         } else {
 235                 volatile unsigned long *a = addr;
 236                 unsigned long mask;
 237                 unsigned long flags;
 238
 239                 a += nr >> SZLONG_LOG;
 240                 mask = 1UL << bit;
 241                 raw_local_irq_save(flags);
 242                 *a ^= mask;
 243                 raw_local_irq_restore(flags);
 244         }
 245 }
 246
 247 /*
 248  * test_and_set_bit - Set a bit and return its old value
 249  * @nr: Bit to set
 250  * @addr: Address to count from
 251  *
 252  * This operation is atomic and cannot be reordered.
 253  * It also implies a memory barrier.
 254  */
 255 static inline int test_and_set_bit(unsigned long nr,
 256         volatile unsigned long *addr)
 257 {
 258         unsigned short bit = nr & SZLONG_MASK;
 259         unsigned long res;
 260
 261         smp_llsc_mb();
 262
 263         if (cpu_has_llsc && R10000_LLSC_WAR) {
 264                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 265                 unsigned long temp;
 266
 267                 __asm__ __volatile__(
 268                 "       .set    mips3                                   \n"
 269                 "1:     " __LL "%0, %1          # test_and_set_bit      \n"
 270                 "       or      %2, %0, %3                              \n"
 271                 "       " __SC  "%2, %1                                 \n"
 272                 "       beqzl   %2, 1b                                  \n"
 273                 "       and     %2, %0, %3                              \n"
 274                 "       .set    mips0                                   \n"
 275                 : "=&r" (temp), "=m" (*m), "=&r" (res)
 276                 : "r" (1UL << bit), "m" (*m)
 277                 : "memory");
 278         } else if (cpu_has_llsc) {
 279                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 280                 unsigned long temp;
 281
 282                 __asm__ __volatile__(
 283                 "       .set    push                                    \n"
 284                 "       .set    noreorder                               \n"
 285                 "       .set    mips3                                   \n"
 286                 "1:     " __LL "%0, %1          # test_and_set_bit      \n"
 287                 "       or      %2, %0, %3                              \n"
 288                 "       " __SC  "%2, %1                                 \n"
 289                 "       beqz    %2, 2f                                  \n"
 290                 "        and    %2, %0, %3                              \n"
 291                 "       .subsection 2                                   \n"
 292                 "2:     b       1b                                      \n"
 293                 "        nop                                            \n"
 294                 "       .previous                                       \n"
 295                 "       .set    pop                                     \n"
 296                 : "=&r" (temp), "=m" (*m), "=&r" (res)
 297                 : "r" (1UL << bit), "m" (*m)
 298                 : "memory");
 299         } else {
 300                 volatile unsigned long *a = addr;
 301                 unsigned long mask;
 302                 unsigned long flags;
 303
 304                 a += nr >> SZLONG_LOG;
 305                 mask = 1UL << bit;
 306                 raw_local_irq_save(flags);
 307                 res = (mask & *a);
 308                 *a |= mask;
 309                 raw_local_irq_restore(flags);
 310         }
 311
 312         smp_llsc_mb();
 313
 314         return res != 0;
 315 }
 316
 317 /*
 318  * test_and_set_bit_lock - Set a bit and return its old value
 319  * @nr: Bit to set
 320  * @addr: Address to count from
 321  *
 322  * This operation is atomic and implies acquire ordering semantics
 323  * after the memory operation.
 324  */
 325 static inline int test_and_set_bit_lock(unsigned long nr,
 326         volatile unsigned long *addr)
 327 {
 328         unsigned short bit = nr & SZLONG_MASK;
 329         unsigned long res;
 330
 331         if (cpu_has_llsc && R10000_LLSC_WAR) {
 332                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 333                 unsigned long temp;
 334
 335                 __asm__ __volatile__(
 336                 "       .set    mips3                                   \n"
 337                 "1:     " __LL "%0, %1          # test_and_set_bit      \n"
 338                 "       or      %2, %0, %3                              \n"
 339                 "       " __SC  "%2, %1                                 \n"
 340                 "       beqzl   %2, 1b                                  \n"
 341                 "       and     %2, %0, %3                              \n"
 342                 "       .set    mips0                                   \n"
 343                 : "=&r" (temp), "=m" (*m), "=&r" (res)
 344                 : "r" (1UL << bit), "m" (*m)
 345                 : "memory");
 346         } else if (cpu_has_llsc) {
 347                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 348                 unsigned long temp;
 349
 350                 __asm__ __volatile__(
 351                 "       .set    push                                    \n"
 352                 "       .set    noreorder                               \n"
 353                 "       .set    mips3                                   \n"
 354                 "1:     " __LL "%0, %1          # test_and_set_bit      \n"
 355                 "       or      %2, %0, %3                              \n"
 356                 "       " __SC  "%2, %1                                 \n"
 357                 "       beqz    %2, 2f                                  \n"
 358                 "        and    %2, %0, %3                              \n"
 359                 "       .subsection 2                                   \n"
 360                 "2:     b       1b                                      \n"
 361                 "        nop                                            \n"
 362                 "       .previous                                       \n"
 363                 "       .set    pop                                     \n"
 364                 : "=&r" (temp), "=m" (*m), "=&r" (res)
 365                 : "r" (1UL << bit), "m" (*m)
 366                 : "memory");
 367         } else {
 368                 volatile unsigned long *a = addr;
 369                 unsigned long mask;
 370                 unsigned long flags;
 371
 372                 a += nr >> SZLONG_LOG;
 373                 mask = 1UL << bit;
 374                 raw_local_irq_save(flags);
 375                 res = (mask & *a);
 376                 *a |= mask;
 377                 raw_local_irq_restore(flags);
 378         }
 379
 380         smp_llsc_mb();
 381
 382         return res != 0;
 383 }
 384 /*
 385  * test_and_clear_bit - Clear a bit and return its old value
 386  * @nr: Bit to clear
 387  * @addr: Address to count from
 388  *
 389  * This operation is atomic and cannot be reordered.
 390  * It also implies a memory barrier.
 391  */
 392 static inline int test_and_clear_bit(unsigned long nr,
 393         volatile unsigned long *addr)
 394 {
 395         unsigned short bit = nr & SZLONG_MASK;
 396         unsigned long res;
 397
 398         smp_llsc_mb();
 399
 400         if (cpu_has_llsc && R10000_LLSC_WAR) {
 401                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 402                 unsigned long temp;
 403
 404                 __asm__ __volatile__(
 405                 "       .set    mips3                                   \n"
 406                 "1:     " __LL  "%0, %1         # test_and_clear_bit    \n"
 407                 "       or      %2, %0, %3                              \n"
 408                 "       xor     %2, %3                                  \n"
 409                 "       " __SC  "%2, %1                                 \n"
 410                 "       beqzl   %2, 1b                                  \n"
 411                 "       and     %2, %0, %3                              \n"
 412                 "       .set    mips0                                   \n"
 413                 : "=&r" (temp), "=m" (*m), "=&r" (res)
 414                 : "r" (1UL << bit), "m" (*m)
 415                 : "memory");
 416 #ifdef CONFIG_CPU_MIPSR2
 417         } else if (__builtin_constant_p(nr)) {
 418                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 419                 unsigned long temp;
 420
 421                 __asm__ __volatile__(
 422                 "1:     " __LL  "%0, %1         # test_and_clear_bit    \n"
 423                 "       " __EXT "%2, %0, %3, 1                          \n"
 424                 "       " __INS "%0, $0, %3, 1                          \n"
 425                 "       " __SC  "%0, %1                                 \n"
 426                 "       beqz    %0, 2f                                  \n"
 427                 "       .subsection 2                                   \n"
 428                 "2:     b       1b                                      \n"
 429                 "       .previous                                       \n"
 430                 : "=&r" (temp), "=m" (*m), "=&r" (res)
 431                 : "ir" (bit), "m" (*m)
 432                 : "memory");
 433 #endif
 434         } else if (cpu_has_llsc) {
 435                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 436                 unsigned long temp;
 437
 438                 __asm__ __volatile__(
 439                 "       .set    push                                    \n"
 440                 "       .set    noreorder                               \n"
 441                 "       .set    mips3                                   \n"
 442                 "1:     " __LL  "%0, %1         # test_and_clear_bit    \n"
 443                 "       or      %2, %0, %3                              \n"
 444                 "       xor     %2, %3                                  \n"
 445                 "       " __SC  "%2, %1                                 \n"
 446                 "       beqz    %2, 2f                                  \n"
 447                 "        and    %2, %0, %3                              \n"
 448                 "       .subsection 2                                   \n"
 449                 "2:     b       1b                                      \n"
 450                 "        nop                                            \n"
 451                 "       .previous                                       \n"
 452                 "       .set    pop                                     \n"
 453                 : "=&r" (temp), "=m" (*m), "=&r" (res)
 454                 : "r" (1UL << bit), "m" (*m)
 455                 : "memory");
 456         } else {
 457                 volatile unsigned long *a = addr;
 458                 unsigned long mask;
 459                 unsigned long flags;
 460
 461                 a += nr >> SZLONG_LOG;
 462                 mask = 1UL << bit;
 463                 raw_local_irq_save(flags);
 464                 res = (mask & *a);
 465                 *a &= ~mask;
 466                 raw_local_irq_restore(flags);
 467         }
 468
 469         smp_llsc_mb();
 470
 471         return res != 0;
 472 }
 473
 474 /*
 475  * test_and_change_bit - Change a bit and return its old value
 476  * @nr: Bit to change
 477  * @addr: Address to count from
 478  *
 479  * This operation is atomic and cannot be reordered.
 480  * It also implies a memory barrier.
 481  */
 482 static inline int test_and_change_bit(unsigned long nr,
 483         volatile unsigned long *addr)
 484 {
 485         unsigned short bit = nr & SZLONG_MASK;
 486         unsigned long res;
 487
 488         smp_llsc_mb();
 489
 490         if (cpu_has_llsc && R10000_LLSC_WAR) {
 491                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 492                 unsigned long temp;
 493
 494                 __asm__ __volatile__(
 495                 "       .set    mips3                                   \n"
 496                 "1:     " __LL  "%0, %1         # test_and_change_bit   \n"
 497                 "       xor     %2, %0, %3                              \n"
 498                 "       " __SC  "%2, %1                                 \n"
 499                 "       beqzl   %2, 1b                                  \n"
 500                 "       and     %2, %0, %3                              \n"
 501                 "       .set    mips0                                   \n"
 502                 : "=&r" (temp), "=m" (*m), "=&r" (res)
 503                 : "r" (1UL << bit), "m" (*m)
 504                 : "memory");
 505         } else if (cpu_has_llsc) {
 506                 unsigned long *m = ((unsigned long *) addr) + (nr >> SZLONG_LOG);
 507                 unsigned long temp;
 508
 509                 __asm__ __volatile__(
 510                 "       .set    push                                    \n"
 511                 "       .set    noreorder                               \n"
 512                 "       .set    mips3                                   \n"
 513                 "1:     " __LL  "%0, %1         # test_and_change_bit   \n"
 514                 "       xor     %2, %0, %3                              \n"
 515                 "       " __SC  "\t%2, %1                               \n"
 516                 "       beqz    %2, 2f                                  \n"
 517                 "        and    %2, %0, %3                              \n"
 518                 "       .subsection 2                                   \n"
 519                 "2:     b       1b                                      \n"
 520                 "        nop                                            \n"
 521                 "       .previous                                       \n"
 522                 "       .set    pop                                     \n"
 523                 : "=&r" (temp), "=m" (*m), "=&r" (res)
 524                 : "r" (1UL << bit), "m" (*m)
 525                 : "memory");
 526         } else {
 527                 volatile unsigned long *a = addr;
 528                 unsigned long mask;
 529                 unsigned long flags;
 530
 531                 a += nr >> SZLONG_LOG;
 532                 mask = 1UL << bit;
 533                 raw_local_irq_save(flags);
 534                 res = (mask & *a);
 535                 *a ^= mask;
 536                 raw_local_irq_restore(flags);
 537         }
 538
 539         smp_llsc_mb();
 540
 541         return res != 0;
 542 }
 543
 544 #include <asm-generic/bitops/non-atomic.h>
 545
 546 /*
 547  * __clear_bit_unlock - Clears a bit in memory
 548  * @nr: Bit to clear
 549  * @addr: Address to start counting from
 550  *
 551  * __clear_bit() is non-atomic and implies release semantics before the memory
 552  * operation. It can be used for an unlock if no other CPUs can concurrently
 553  * modify other bits in the word.
 554  */
 555 static inline void __clear_bit_unlock(unsigned long nr, volatile unsigned long *addr)
 556 {
 557         smp_mb();
 558         __clear_bit(nr, addr);
 559 }
 560
 561 /*
 562  * Return the bit position (0..63) of the most significant 1 bit in a word
 563  * Returns -1 if no 1 bit exists
 564  */
 565 static inline unsigned long __fls(unsigned long word)
 566 {
 567         int num;
 568
 569         if (BITS_PER_LONG == 32 &&
 570             __builtin_constant_p(cpu_has_mips_r) && cpu_has_mips_r) {
 571                 __asm__(
 572                 "       .set    push                                    \n"
 573                 "       .set    mips32                                  \n"
 574                 "       clz     %0, %1                                  \n"
 575                 "       .set    pop                                     \n"
 576                 : "=r" (num)
 577                 : "r" (word));
 578
 579                 return 31 - num;
 580         }
 581
 582         if (BITS_PER_LONG == 64 &&
 583             __builtin_constant_p(cpu_has_mips64) && cpu_has_mips64) {
 584                 __asm__(
 585                 "       .set    push                                    \n"
 586                 "       .set    mips64                                  \n"
 587                 "       dclz    %0, %1                                  \n"
 588                 "       .set    pop                                     \n"
 589                 : "=r" (num)
 590                 : "r" (word));
 591
 592                 return 63 - num;
 593         }
 594
 595         num = BITS_PER_LONG - 1;
 596
 597 #if BITS_PER_LONG == 64
 598         if (!(word & (~0ul << 32))) {
 599                 num -= 32;
 600                 word <<= 32;
 601         }
 602 #endif
 603         if (!(word & (~0ul << (BITS_PER_LONG-16)))) {
 604                 num -= 16;
 605                 word <<= 16;
 606         }
 607         if (!(word & (~0ul << (BITS_PER_LONG-8)))) {
 608                 num -= 8;
 609                 word <<= 8;
 610         }
 611         if (!(word & (~0ul << (BITS_PER_LONG-4)))) {
 612                 num -= 4;
 613                 word <<= 4;
 614         }
 615         if (!(word & (~0ul << (BITS_PER_LONG-2)))) {
 616                 num -= 2;
 617                 word <<= 2;
 618         }
 619         if (!(word & (~0ul << (BITS_PER_LONG-1))))
 620                 num -= 1;
 621         return num;
 622 }
 623
 624 /*
 625  * __ffs - find first bit in word.
 626  * @word: The word to search
 627  *
 628  * Returns 0..SZLONG-1
 629  * Undefined if no bit exists, so code should check against 0 first.
 630  */
 631 static inline unsigned long __ffs(unsigned long word)
 632 {
 633         return __fls(word & -word);
 634 }
 635
 636 /*
 637  * fls - find last bit set.
 638  * @word: The word to search
 639  *
 640  * This is defined the same way as ffs.
 641  * Note fls(0) = 0, fls(1) = 1, fls(0x80000000) = 32.
 642  */
 643 static inline int fls(int x)
 644 {
 645         int r;
 646
 647         if (__builtin_constant_p(cpu_has_mips_r) && cpu_has_mips_r) {
 648                 __asm__("clz %0, %1" : "=r" (x) : "r" (x));
 649
 650                 return 32 - x;
 651         }
 652
 653         r = 32;
 654         if (!x)
 655                 return 0;
 656         if (!(x & 0xffff0000u)) {
 657                 x <<= 16;
 658                 r -= 16;
 659         }
 660         if (!(x & 0xff000000u)) {
 661                 x <<= 8;
 662                 r -= 8;
 663         }
 664         if (!(x & 0xf0000000u)) {
 665                 x <<= 4;
 666                 r -= 4;
 667         }
 668         if (!(x & 0xc0000000u)) {
 669                 x <<= 2;
 670                 r -= 2;
 671         }
 672         if (!(x & 0x80000000u)) {
 673                 x <<= 1;
 674                 r -= 1;
 675         }
 676         return r;
 677 }
 678
 679 #include <asm-generic/bitops/fls64.h>
 680
 681 /*
 682  * ffs - find first bit set.
 683  * @word: The word to search
 684  *
 685  * This is defined the same way as
 686  * the libc and compiler builtin ffs routines, therefore
 687  * differs in spirit from the above ffz (man ffs).
 688  */
 689 static inline int ffs(int word)
 690 {
 691         if (!word)
 692                 return 0;
 693
 694         return fls(word & -word);
 695 }
 696
 697 #include <asm-generic/bitops/ffz.h>
 698 #include <asm-generic/bitops/find.h>
 699
 700 #ifdef __KERNEL__
 701
 702 #include <asm-generic/bitops/sched.h>
 703 #include <asm-generic/bitops/hweight.h>
 704 #include <asm-generic/bitops/ext2-non-atomic.h>
 705 #include <asm-generic/bitops/ext2-atomic.h>
 706 #include <asm-generic/bitops/minix.h>
 707
 708 #endif /* __KERNEL__ */
 709
 710 #endif /* _ASM_BITOPS_H */