arch/arc/include/asm/spinlock.h

   1 /*
   2  * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com)
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  */
   8
   9 #ifndef __ASM_SPINLOCK_H
  10 #define __ASM_SPINLOCK_H
  11
  12 #include <asm/spinlock_types.h>
  13 #include <asm/processor.h>
  14 #include <asm/barrier.h>
  15
  16 #define arch_spin_is_locked(x)  ((x)->slock != __ARCH_SPIN_LOCK_UNLOCKED__)
  17 #define arch_spin_lock_flags(lock, flags)       arch_spin_lock(lock)
  18 #define arch_spin_unlock_wait(x) \
  19         do { while (arch_spin_is_locked(x)) cpu_relax(); } while (0)
  20
  21 #ifdef CONFIG_ARC_HAS_LLSC
  22
  23 /*
  24  * A normal LLOCK/SCOND based system, w/o need for livelock workaround
  25  */
  26 #ifndef CONFIG_ARC_STAR_9000923308
  27
  28 static inline void arch_spin_lock(arch_spinlock_t *lock)
  29 {
  30         unsigned int val;
  31
  32         smp_mb();
  33
  34         __asm__ __volatile__(
  35         "1:     llock   %[val], [%[slock]]      \n"
  36         "       breq    %[val], %[LOCKED], 1b   \n"     /* spin while LOCKED */
  37         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
  38         "       bnz     1b                      \n"
  39         "                                       \n"
  40         : [val]         "=&r"   (val)
  41         : [slock]       "r"     (&(lock->slock)),
  42           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
  43         : "memory", "cc");
  44
  45         smp_mb();
  46 }
  47
  48 /* 1 - lock taken successfully */
  49 static inline int arch_spin_trylock(arch_spinlock_t *lock)
  50 {
  51         unsigned int val, got_it = 0;
  52
  53         smp_mb();
  54
  55         __asm__ __volatile__(
  56         "1:     llock   %[val], [%[slock]]      \n"
  57         "       breq    %[val], %[LOCKED], 4f   \n"     /* already LOCKED, just bail */
  58         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
  59         "       bnz     1b                      \n"
  60         "       mov     %[got_it], 1            \n"
  61         "4:                                     \n"
  62         "                                       \n"
  63         : [val]         "=&r"   (val),
  64           [got_it]      "+&r"   (got_it)
  65         : [slock]       "r"     (&(lock->slock)),
  66           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
  67         : "memory", "cc");
  68
  69         smp_mb();
  70
  71         return got_it;
  72 }
  73
  74 static inline void arch_spin_unlock(arch_spinlock_t *lock)
  75 {
  76         smp_mb();
  77
  78         lock->slock = __ARCH_SPIN_LOCK_UNLOCKED__;
  79
  80         smp_mb();
  81 }
  82
  83 /*
  84  * Read-write spinlocks, allowing multiple readers but only one writer.
  85  * Unfair locking as Writers could be starved indefinitely by Reader(s)
  86  */
  87
  88 static inline void arch_read_lock(arch_rwlock_t *rw)
  89 {
  90         unsigned int val;
  91
  92         smp_mb();
  93
  94         /*
  95          * zero means writer holds the lock exclusively, deny Reader.
  96          * Otherwise grant lock to first/subseq reader
  97          *
  98          *      if (rw->counter > 0) {
  99          *              rw->counter--;
 100          *              ret = 1;
 101          *      }
 102          */
 103
 104         __asm__ __volatile__(
 105         "1:     llock   %[val], [%[rwlock]]     \n"
 106         "       brls    %[val], %[WR_LOCKED], 1b\n"     /* <= 0: spin while write locked */
 107         "       sub     %[val], %[val], 1       \n"     /* reader lock */
 108         "       scond   %[val], [%[rwlock]]     \n"
 109         "       bnz     1b                      \n"
 110         "                                       \n"
 111         : [val]         "=&r"   (val)
 112         : [rwlock]      "r"     (&(rw->counter)),
 113           [WR_LOCKED]   "ir"    (0)
 114         : "memory", "cc");
 115
 116         smp_mb();
 117 }
 118
 119 /* 1 - lock taken successfully */
 120 static inline int arch_read_trylock(arch_rwlock_t *rw)
 121 {
 122         unsigned int val, got_it = 0;
 123
 124         smp_mb();
 125
 126         __asm__ __volatile__(
 127         "1:     llock   %[val], [%[rwlock]]     \n"
 128         "       brls    %[val], %[WR_LOCKED], 4f\n"     /* <= 0: already write locked, bail */
 129         "       sub     %[val], %[val], 1       \n"     /* counter-- */
 130         "       scond   %[val], [%[rwlock]]     \n"
 131         "       bnz     1b                      \n"     /* retry if collided with someone */
 132         "       mov     %[got_it], 1            \n"
 133         "                                       \n"
 134         "4: ; --- done ---                      \n"
 135
 136         : [val]         "=&r"   (val),
 137           [got_it]      "+&r"   (got_it)
 138         : [rwlock]      "r"     (&(rw->counter)),
 139           [WR_LOCKED]   "ir"    (0)
 140         : "memory", "cc");
 141
 142         smp_mb();
 143
 144         return got_it;
 145 }
 146
 147 static inline void arch_write_lock(arch_rwlock_t *rw)
 148 {
 149         unsigned int val;
 150
 151         smp_mb();
 152
 153         /*
 154          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
 155          * deny writer. Otherwise if unlocked grant to writer
 156          * Hence the claim that Linux rwlocks are unfair to writers.
 157          * (can be starved for an indefinite time by readers).
 158          *
 159          *      if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
 160          *              rw->counter = 0;
 161          *              ret = 1;
 162          *      }
 163          */
 164
 165         __asm__ __volatile__(
 166         "1:     llock   %[val], [%[rwlock]]     \n"
 167         "       brne    %[val], %[UNLOCKED], 1b \n"     /* while !UNLOCKED spin */
 168         "       mov     %[val], %[WR_LOCKED]    \n"
 169         "       scond   %[val], [%[rwlock]]     \n"
 170         "       bnz     1b                      \n"
 171         "                                       \n"
 172         : [val]         "=&r"   (val)
 173         : [rwlock]      "r"     (&(rw->counter)),
 174           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 175           [WR_LOCKED]   "ir"    (0)
 176         : "memory", "cc");
 177
 178         smp_mb();
 179 }
 180
 181 /* 1 - lock taken successfully */
 182 static inline int arch_write_trylock(arch_rwlock_t *rw)
 183 {
 184         unsigned int val, got_it = 0;
 185
 186         smp_mb();
 187
 188         __asm__ __volatile__(
 189         "1:     llock   %[val], [%[rwlock]]     \n"
 190         "       brne    %[val], %[UNLOCKED], 4f \n"     /* !UNLOCKED, bail */
 191         "       mov     %[val], %[WR_LOCKED]    \n"
 192         "       scond   %[val], [%[rwlock]]     \n"
 193         "       bnz     1b                      \n"     /* retry if collided with someone */
 194         "       mov     %[got_it], 1            \n"
 195         "                                       \n"
 196         "4: ; --- done ---                      \n"
 197
 198         : [val]         "=&r"   (val),
 199           [got_it]      "+&r"   (got_it)
 200         : [rwlock]      "r"     (&(rw->counter)),
 201           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 202           [WR_LOCKED]   "ir"    (0)
 203         : "memory", "cc");
 204
 205         smp_mb();
 206
 207         return got_it;
 208 }
 209
 210 static inline void arch_read_unlock(arch_rwlock_t *rw)
 211 {
 212         unsigned int val;
 213
 214         smp_mb();
 215
 216         /*
 217          * rw->counter++;
 218          */
 219         __asm__ __volatile__(
 220         "1:     llock   %[val], [%[rwlock]]     \n"
 221         "       add     %[val], %[val], 1       \n"
 222         "       scond   %[val], [%[rwlock]]     \n"
 223         "       bnz     1b                      \n"
 224         "                                       \n"
 225         : [val]         "=&r"   (val)
 226         : [rwlock]      "r"     (&(rw->counter))
 227         : "memory", "cc");
 228
 229         smp_mb();
 230 }
 231
 232 static inline void arch_write_unlock(arch_rwlock_t *rw)
 233 {
 234         smp_mb();
 235
 236         rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
 237
 238         smp_mb();
 239 }
 240
 241 #else   /* CONFIG_ARC_STAR_9000923308 */
 242
 243 /*
 244  * HS38x4 could get into a LLOCK/SCOND livelock in case of multiple overlapping
 245  * coherency transactions in the SCU. The exclusive line state keeps rotating
 246  * among contenting cores leading to a never ending cycle. So break the cycle
 247  * by deferring the retry of failed exclusive access (SCOND). The actual delay
 248  * needed is function of number of contending cores as well as the unrelated
 249  * coherency traffic from other cores. To keep the code simple, start off with
 250  * small delay of 1 which would suffice most cases and in case of contention
 251  * double the delay. Eventually the delay is sufficient such that the coherency
 252  * pipeline is drained, thus a subsequent exclusive access would succeed.
 253  */
 254
 255 #define SCOND_FAIL_RETRY_VAR_DEF                                                \
 256         unsigned int delay, tmp;                                                \
 257
 258 #define SCOND_FAIL_RETRY_ASM                                                    \
 259         "   ; --- scond fail delay ---          \n"                             \
 260         "       mov     %[tmp], %[delay]        \n"     /* tmp = delay */       \
 261         "2:     brne.d  %[tmp], 0, 2b           \n"     /* while (tmp != 0) */  \
 262         "       sub     %[tmp], %[tmp], 1       \n"     /* tmp-- */             \
 263         "       rol     %[delay], %[delay]      \n"     /* delay *= 2 */        \
 264         "       b       1b                      \n"     /* start over */        \
 265         "                                       \n"                             \
 266         "4: ; --- done ---                      \n"                             \
 267
 268 #define SCOND_FAIL_RETRY_VARS                                                   \
 269           ,[delay] "=&r" (delay), [tmp] "=&r"   (tmp)                           \
 270
 271 static inline void arch_spin_lock(arch_spinlock_t *lock)
 272 {
 273         unsigned int val;
 274         SCOND_FAIL_RETRY_VAR_DEF;
 275
 276         smp_mb();
 277
 278         __asm__ __volatile__(
 279         "0:     mov     %[delay], 1             \n"
 280         "1:     llock   %[val], [%[slock]]      \n"
 281         "       breq    %[val], %[LOCKED], 0b   \n"     /* spin while LOCKED */
 282         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
 283         "       bz      4f                      \n"     /* done */
 284         "                                       \n"
 285         SCOND_FAIL_RETRY_ASM
 286
 287         : [val]         "=&r"   (val)
 288           SCOND_FAIL_RETRY_VARS
 289         : [slock]       "r"     (&(lock->slock)),
 290           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
 291         : "memory", "cc");
 292
 293         smp_mb();
 294 }
 295
 296 /* 1 - lock taken successfully */
 297 static inline int arch_spin_trylock(arch_spinlock_t *lock)
 298 {
 299         unsigned int val, got_it = 0;
 300         SCOND_FAIL_RETRY_VAR_DEF;
 301
 302         smp_mb();
 303
 304         __asm__ __volatile__(
 305         "0:     mov     %[delay], 1             \n"
 306         "1:     llock   %[val], [%[slock]]      \n"
 307         "       breq    %[val], %[LOCKED], 4f   \n"     /* already LOCKED, just bail */
 308         "       scond   %[LOCKED], [%[slock]]   \n"     /* acquire */
 309         "       bz.d    4f                      \n"
 310         "       mov.z   %[got_it], 1            \n"     /* got it */
 311         "                                       \n"
 312         SCOND_FAIL_RETRY_ASM
 313
 314         : [val]         "=&r"   (val),
 315           [got_it]      "+&r"   (got_it)
 316           SCOND_FAIL_RETRY_VARS
 317         : [slock]       "r"     (&(lock->slock)),
 318           [LOCKED]      "r"     (__ARCH_SPIN_LOCK_LOCKED__)
 319         : "memory", "cc");
 320
 321         smp_mb();
 322
 323         return got_it;
 324 }
 325
 326 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 327 {
 328         smp_mb();
 329
 330         lock->slock = __ARCH_SPIN_LOCK_UNLOCKED__;
 331
 332         smp_mb();
 333 }
 334
 335 /*
 336  * Read-write spinlocks, allowing multiple readers but only one writer.
 337  * Unfair locking as Writers could be starved indefinitely by Reader(s)
 338  */
 339
 340 static inline void arch_read_lock(arch_rwlock_t *rw)
 341 {
 342         unsigned int val;
 343         SCOND_FAIL_RETRY_VAR_DEF;
 344
 345         smp_mb();
 346
 347         /*
 348          * zero means writer holds the lock exclusively, deny Reader.
 349          * Otherwise grant lock to first/subseq reader
 350          *
 351          *      if (rw->counter > 0) {
 352          *              rw->counter--;
 353          *              ret = 1;
 354          *      }
 355          */
 356
 357         __asm__ __volatile__(
 358         "0:     mov     %[delay], 1             \n"
 359         "1:     llock   %[val], [%[rwlock]]     \n"
 360         "       brls    %[val], %[WR_LOCKED], 0b\n"     /* <= 0: spin while write locked */
 361         "       sub     %[val], %[val], 1       \n"     /* reader lock */
 362         "       scond   %[val], [%[rwlock]]     \n"
 363         "       bz      4f                      \n"     /* done */
 364         "                                       \n"
 365         SCOND_FAIL_RETRY_ASM
 366
 367         : [val]         "=&r"   (val)
 368           SCOND_FAIL_RETRY_VARS
 369         : [rwlock]      "r"     (&(rw->counter)),
 370           [WR_LOCKED]   "ir"    (0)
 371         : "memory", "cc");
 372
 373         smp_mb();
 374 }
 375
 376 /* 1 - lock taken successfully */
 377 static inline int arch_read_trylock(arch_rwlock_t *rw)
 378 {
 379         unsigned int val, got_it = 0;
 380         SCOND_FAIL_RETRY_VAR_DEF;
 381
 382         smp_mb();
 383
 384         __asm__ __volatile__(
 385         "0:     mov     %[delay], 1             \n"
 386         "1:     llock   %[val], [%[rwlock]]     \n"
 387         "       brls    %[val], %[WR_LOCKED], 4f\n"     /* <= 0: already write locked, bail */
 388         "       sub     %[val], %[val], 1       \n"     /* counter-- */
 389         "       scond   %[val], [%[rwlock]]     \n"
 390         "       bz.d    4f                      \n"
 391         "       mov.z   %[got_it], 1            \n"     /* got it */
 392         "                                       \n"
 393         SCOND_FAIL_RETRY_ASM
 394
 395         : [val]         "=&r"   (val),
 396           [got_it]      "+&r"   (got_it)
 397           SCOND_FAIL_RETRY_VARS
 398         : [rwlock]      "r"     (&(rw->counter)),
 399           [WR_LOCKED]   "ir"    (0)
 400         : "memory", "cc");
 401
 402         smp_mb();
 403
 404         return got_it;
 405 }
 406
 407 static inline void arch_write_lock(arch_rwlock_t *rw)
 408 {
 409         unsigned int val;
 410         SCOND_FAIL_RETRY_VAR_DEF;
 411
 412         smp_mb();
 413
 414         /*
 415          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
 416          * deny writer. Otherwise if unlocked grant to writer
 417          * Hence the claim that Linux rwlocks are unfair to writers.
 418          * (can be starved for an indefinite time by readers).
 419          *
 420          *      if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
 421          *              rw->counter = 0;
 422          *              ret = 1;
 423          *      }
 424          */
 425
 426         __asm__ __volatile__(
 427         "0:     mov     %[delay], 1             \n"
 428         "1:     llock   %[val], [%[rwlock]]     \n"
 429         "       brne    %[val], %[UNLOCKED], 0b \n"     /* while !UNLOCKED spin */
 430         "       mov     %[val], %[WR_LOCKED]    \n"
 431         "       scond   %[val], [%[rwlock]]     \n"
 432         "       bz      4f                      \n"
 433         "                                       \n"
 434         SCOND_FAIL_RETRY_ASM
 435
 436         : [val]         "=&r"   (val)
 437           SCOND_FAIL_RETRY_VARS
 438         : [rwlock]      "r"     (&(rw->counter)),
 439           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 440           [WR_LOCKED]   "ir"    (0)
 441         : "memory", "cc");
 442
 443         smp_mb();
 444 }
 445
 446 /* 1 - lock taken successfully */
 447 static inline int arch_write_trylock(arch_rwlock_t *rw)
 448 {
 449         unsigned int val, got_it = 0;
 450         SCOND_FAIL_RETRY_VAR_DEF;
 451
 452         smp_mb();
 453
 454         __asm__ __volatile__(
 455         "0:     mov     %[delay], 1             \n"
 456         "1:     llock   %[val], [%[rwlock]]     \n"
 457         "       brne    %[val], %[UNLOCKED], 4f \n"     /* !UNLOCKED, bail */
 458         "       mov     %[val], %[WR_LOCKED]    \n"
 459         "       scond   %[val], [%[rwlock]]     \n"
 460         "       bz.d    4f                      \n"
 461         "       mov.z   %[got_it], 1            \n"     /* got it */
 462         "                                       \n"
 463         SCOND_FAIL_RETRY_ASM
 464
 465         : [val]         "=&r"   (val),
 466           [got_it]      "+&r"   (got_it)
 467           SCOND_FAIL_RETRY_VARS
 468         : [rwlock]      "r"     (&(rw->counter)),
 469           [UNLOCKED]    "ir"    (__ARCH_RW_LOCK_UNLOCKED__),
 470           [WR_LOCKED]   "ir"    (0)
 471         : "memory", "cc");
 472
 473         smp_mb();
 474
 475         return got_it;
 476 }
 477
 478 static inline void arch_read_unlock(arch_rwlock_t *rw)
 479 {
 480         unsigned int val;
 481
 482         smp_mb();
 483
 484         /*
 485          * rw->counter++;
 486          */
 487         __asm__ __volatile__(
 488         "1:     llock   %[val], [%[rwlock]]     \n"
 489         "       add     %[val], %[val], 1       \n"
 490         "       scond   %[val], [%[rwlock]]     \n"
 491         "       bnz     1b                      \n"
 492         "                                       \n"
 493         : [val]         "=&r"   (val)
 494         : [rwlock]      "r"     (&(rw->counter))
 495         : "memory", "cc");
 496
 497         smp_mb();
 498 }
 499
 500 static inline void arch_write_unlock(arch_rwlock_t *rw)
 501 {
 502         unsigned int val;
 503
 504         smp_mb();
 505
 506         /*
 507          * rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
 508          */
 509         __asm__ __volatile__(
 510         "1:     llock   %[val], [%[rwlock]]     \n"
 511         "       scond   %[UNLOCKED], [%[rwlock]]\n"
 512         "       bnz     1b                      \n"
 513         "                                       \n"
 514         : [val]         "=&r"   (val)
 515         : [rwlock]      "r"     (&(rw->counter)),
 516           [UNLOCKED]    "r"     (__ARCH_RW_LOCK_UNLOCKED__)
 517         : "memory", "cc");
 518
 519         smp_mb();
 520 }
 521
 522 #undef SCOND_FAIL_RETRY_VAR_DEF
 523 #undef SCOND_FAIL_RETRY_ASM
 524 #undef SCOND_FAIL_RETRY_VARS
 525
 526 #endif  /* CONFIG_ARC_STAR_9000923308 */
 527
 528 #else   /* !CONFIG_ARC_HAS_LLSC */
 529
 530 static inline void arch_spin_lock(arch_spinlock_t *lock)
 531 {
 532         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
 533
 534         /*
 535          * This smp_mb() is technically superfluous, we only need the one
 536          * after the lock for providing the ACQUIRE semantics.
 537          * However doing the "right" thing was regressing hackbench
 538          * so keeping this, pending further investigation
 539          */
 540         smp_mb();
 541
 542         __asm__ __volatile__(
 543         "1:     ex  %0, [%1]            \n"
 544         "       breq  %0, %2, 1b        \n"
 545         : "+&r" (val)
 546         : "r"(&(lock->slock)), "ir"(__ARCH_SPIN_LOCK_LOCKED__)
 547         : "memory");
 548
 549         /*
 550          * ACQUIRE barrier to ensure load/store after taking the lock
 551          * don't "bleed-up" out of the critical section (leak-in is allowed)
 552          * http://www.spinics.net/lists/kernel/msg2010409.html
 553          *
 554          * ARCv2 only has load-load, store-store and all-all barrier
 555          * thus need the full all-all barrier
 556          */
 557         smp_mb();
 558 }
 559
 560 /* 1 - lock taken successfully */
 561 static inline int arch_spin_trylock(arch_spinlock_t *lock)
 562 {
 563         unsigned int val = __ARCH_SPIN_LOCK_LOCKED__;
 564
 565         smp_mb();
 566
 567         __asm__ __volatile__(
 568         "1:     ex  %0, [%1]            \n"
 569         : "+r" (val)
 570         : "r"(&(lock->slock))
 571         : "memory");
 572
 573         smp_mb();
 574
 575         return (val == __ARCH_SPIN_LOCK_UNLOCKED__);
 576 }
 577
 578 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 579 {
 580         unsigned int val = __ARCH_SPIN_LOCK_UNLOCKED__;
 581
 582         /*
 583          * RELEASE barrier: given the instructions avail on ARCv2, full barrier
 584          * is the only option
 585          */
 586         smp_mb();
 587
 588         __asm__ __volatile__(
 589         "       ex  %0, [%1]            \n"
 590         : "+r" (val)
 591         : "r"(&(lock->slock))
 592         : "memory");
 593
 594         /*
 595          * superfluous, but keeping for now - see pairing version in
 596          * arch_spin_lock above
 597          */
 598         smp_mb();
 599 }
 600
 601 /*
 602  * Read-write spinlocks, allowing multiple readers but only one writer.
 603  * Unfair locking as Writers could be starved indefinitely by Reader(s)
 604  *
 605  * The spinlock itself is contained in @counter and access to it is
 606  * serialized with @lock_mutex.
 607  */
 608
 609 /* 1 - lock taken successfully */
 610 static inline int arch_read_trylock(arch_rwlock_t *rw)
 611 {
 612         int ret = 0;
 613
 614         arch_spin_lock(&(rw->lock_mutex));
 615
 616         /*
 617          * zero means writer holds the lock exclusively, deny Reader.
 618          * Otherwise grant lock to first/subseq reader
 619          */
 620         if (rw->counter > 0) {
 621                 rw->counter--;
 622                 ret = 1;
 623         }
 624
 625         arch_spin_unlock(&(rw->lock_mutex));
 626
 627         smp_mb();
 628         return ret;
 629 }
 630
 631 /* 1 - lock taken successfully */
 632 static inline int arch_write_trylock(arch_rwlock_t *rw)
 633 {
 634         int ret = 0;
 635
 636         arch_spin_lock(&(rw->lock_mutex));
 637
 638         /*
 639          * If reader(s) hold lock (lock < __ARCH_RW_LOCK_UNLOCKED__),
 640          * deny writer. Otherwise if unlocked grant to writer
 641          * Hence the claim that Linux rwlocks are unfair to writers.
 642          * (can be starved for an indefinite time by readers).
 643          */
 644         if (rw->counter == __ARCH_RW_LOCK_UNLOCKED__) {
 645                 rw->counter = 0;
 646                 ret = 1;
 647         }
 648         arch_spin_unlock(&(rw->lock_mutex));
 649
 650         return ret;
 651 }
 652
 653 static inline void arch_read_lock(arch_rwlock_t *rw)
 654 {
 655         while (!arch_read_trylock(rw))
 656                 cpu_relax();
 657 }
 658
 659 static inline void arch_write_lock(arch_rwlock_t *rw)
 660 {
 661         while (!arch_write_trylock(rw))
 662                 cpu_relax();
 663 }
 664
 665 static inline void arch_read_unlock(arch_rwlock_t *rw)
 666 {
 667         arch_spin_lock(&(rw->lock_mutex));
 668         rw->counter++;
 669         arch_spin_unlock(&(rw->lock_mutex));
 670 }
 671
 672 static inline void arch_write_unlock(arch_rwlock_t *rw)
 673 {
 674         arch_spin_lock(&(rw->lock_mutex));
 675         rw->counter = __ARCH_RW_LOCK_UNLOCKED__;
 676         arch_spin_unlock(&(rw->lock_mutex));
 677 }
 678
 679 #endif
 680
 681 #define arch_read_can_lock(x)   ((x)->counter > 0)
 682 #define arch_write_can_lock(x)  ((x)->counter == __ARCH_RW_LOCK_UNLOCKED__)
 683
 684 #define arch_read_lock_flags(lock, flags)       arch_read_lock(lock)
 685 #define arch_write_lock_flags(lock, flags)      arch_write_lock(lock)
 686
 687 #define arch_spin_relax(lock)   cpu_relax()
 688 #define arch_read_relax(lock)   cpu_relax()
 689 #define arch_write_relax(lock)  cpu_relax()
 690
 691 #endif /* __ASM_SPINLOCK_H */