1 /* Copyright (C) 2002-2014 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
23 #include <lowlevellock.h>
25 #ifndef lll_trylock_elision
26 #define lll_trylock_elision(a,t) lll_trylock(a)
30 #define DO_ELISION(m) 0
33 /* We don't force elision in trylock, because this can lead to inconsistent
34 lock state if the lock was actually busy. */
37 __pthread_mutex_trylock (mutex)
38 pthread_mutex_t *mutex;
41 pid_t id = THREAD_GETMEM (THREAD_SELF, tid);
43 switch (__builtin_expect (PTHREAD_MUTEX_TYPE_ELISION (mutex),
44 PTHREAD_MUTEX_TIMED_NP))
46 /* Recursive mutex. */
47 case PTHREAD_MUTEX_RECURSIVE_NP|PTHREAD_MUTEX_ELISION_NP:
48 case PTHREAD_MUTEX_RECURSIVE_NP:
49 /* Check whether we already hold the mutex. */
50 if (mutex->__data.__owner == id)
52 /* Just bump the counter. */
53 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
54 /* Overflow of the counter. */
57 ++mutex->__data.__count;
61 if (lll_trylock (mutex->__data.__lock) == 0)
63 /* Record the ownership. */
64 mutex->__data.__owner = id;
65 mutex->__data.__count = 1;
66 ++mutex->__data.__nusers;
71 case PTHREAD_MUTEX_TIMED_ELISION_NP:
73 if (lll_trylock_elision (mutex->__data.__lock,
74 mutex->__data.__elision) != 0)
76 /* Don't record the ownership. */
79 case PTHREAD_MUTEX_TIMED_NP:
80 if (DO_ELISION (mutex))
83 case PTHREAD_MUTEX_ADAPTIVE_NP:
84 case PTHREAD_MUTEX_ERRORCHECK_NP:
85 if (lll_trylock (mutex->__data.__lock) != 0)
88 /* Record the ownership. */
89 mutex->__data.__owner = id;
90 ++mutex->__data.__nusers;
94 case PTHREAD_MUTEX_ROBUST_RECURSIVE_NP:
95 case PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP:
96 case PTHREAD_MUTEX_ROBUST_NORMAL_NP:
97 case PTHREAD_MUTEX_ROBUST_ADAPTIVE_NP:
98 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
99 &mutex->__data.__list.__next);
101 oldval = mutex->__data.__lock;
105 if ((oldval & FUTEX_OWNER_DIED) != 0)
107 /* The previous owner died. Try locking the mutex. */
108 int newval = id | (oldval & FUTEX_WAITERS);
111 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
114 if (newval != oldval)
120 /* We got the mutex. */
121 mutex->__data.__count = 1;
122 /* But it is inconsistent unless marked otherwise. */
123 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
125 ENQUEUE_MUTEX (mutex);
126 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
128 /* Note that we deliberately exist here. If we fall
129 through to the end of the function __nusers would be
130 incremented which is not correct because the old
131 owner has to be discounted. */
135 /* Check whether we already hold the mutex. */
136 if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
138 int kind = PTHREAD_MUTEX_TYPE (mutex);
139 if (kind == PTHREAD_MUTEX_ROBUST_ERRORCHECK_NP)
141 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
146 if (kind == PTHREAD_MUTEX_ROBUST_RECURSIVE_NP)
148 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
151 /* Just bump the counter. */
152 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
153 /* Overflow of the counter. */
156 ++mutex->__data.__count;
162 oldval = lll_robust_trylock (mutex->__data.__lock, id);
163 if (oldval != 0 && (oldval & FUTEX_OWNER_DIED) == 0)
165 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
170 if (__builtin_expect (mutex->__data.__owner
171 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
173 /* This mutex is now not recoverable. */
174 mutex->__data.__count = 0;
176 lll_unlock (mutex->__data.__lock,
177 PTHREAD_ROBUST_MUTEX_PSHARED (mutex));
178 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
179 return ENOTRECOVERABLE;
182 while ((oldval & FUTEX_OWNER_DIED) != 0);
184 ENQUEUE_MUTEX (mutex);
185 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
187 mutex->__data.__owner = id;
188 ++mutex->__data.__nusers;
189 mutex->__data.__count = 1;
193 case PTHREAD_MUTEX_PI_RECURSIVE_NP:
194 case PTHREAD_MUTEX_PI_ERRORCHECK_NP:
195 case PTHREAD_MUTEX_PI_NORMAL_NP:
196 case PTHREAD_MUTEX_PI_ADAPTIVE_NP:
197 case PTHREAD_MUTEX_PI_ROBUST_RECURSIVE_NP:
198 case PTHREAD_MUTEX_PI_ROBUST_ERRORCHECK_NP:
199 case PTHREAD_MUTEX_PI_ROBUST_NORMAL_NP:
200 case PTHREAD_MUTEX_PI_ROBUST_ADAPTIVE_NP:
202 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
203 int robust = mutex->__data.__kind & PTHREAD_MUTEX_ROBUST_NORMAL_NP;
206 /* Note: robust PI futexes are signaled by setting bit 0. */
207 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending,
208 (void *) (((uintptr_t) &mutex->__data.__list.__next)
211 oldval = mutex->__data.__lock;
213 /* Check whether we already hold the mutex. */
214 if (__builtin_expect ((oldval & FUTEX_TID_MASK) == id, 0))
216 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
218 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
222 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
224 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
226 /* Just bump the counter. */
227 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
228 /* Overflow of the counter. */
231 ++mutex->__data.__count;
238 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
243 if ((oldval & FUTEX_OWNER_DIED) == 0)
245 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
252 /* The mutex owner died. The kernel will now take care of
254 int private = (robust
255 ? PTHREAD_ROBUST_MUTEX_PSHARED (mutex)
256 : PTHREAD_MUTEX_PSHARED (mutex));
257 INTERNAL_SYSCALL_DECL (__err);
258 int e = INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
259 __lll_private_flag (FUTEX_TRYLOCK_PI,
262 if (INTERNAL_SYSCALL_ERROR_P (e, __err)
263 && INTERNAL_SYSCALL_ERRNO (e, __err) == EWOULDBLOCK)
265 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
270 oldval = mutex->__data.__lock;
273 if (__builtin_expect (oldval & FUTEX_OWNER_DIED, 0))
275 atomic_and (&mutex->__data.__lock, ~FUTEX_OWNER_DIED);
277 /* We got the mutex. */
278 mutex->__data.__count = 1;
279 /* But it is inconsistent unless marked otherwise. */
280 mutex->__data.__owner = PTHREAD_MUTEX_INCONSISTENT;
282 ENQUEUE_MUTEX (mutex);
283 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
285 /* Note that we deliberately exit here. If we fall
286 through to the end of the function __nusers would be
287 incremented which is not correct because the old owner
288 has to be discounted. */
293 && __builtin_expect (mutex->__data.__owner
294 == PTHREAD_MUTEX_NOTRECOVERABLE, 0))
296 /* This mutex is now not recoverable. */
297 mutex->__data.__count = 0;
299 INTERNAL_SYSCALL_DECL (__err);
300 INTERNAL_SYSCALL (futex, __err, 4, &mutex->__data.__lock,
301 __lll_private_flag (FUTEX_UNLOCK_PI,
302 PTHREAD_ROBUST_MUTEX_PSHARED (mutex)),
305 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
306 return ENOTRECOVERABLE;
311 ENQUEUE_MUTEX_PI (mutex);
312 THREAD_SETMEM (THREAD_SELF, robust_head.list_op_pending, NULL);
315 mutex->__data.__owner = id;
316 ++mutex->__data.__nusers;
317 mutex->__data.__count = 1;
322 case PTHREAD_MUTEX_PP_RECURSIVE_NP:
323 case PTHREAD_MUTEX_PP_ERRORCHECK_NP:
324 case PTHREAD_MUTEX_PP_NORMAL_NP:
325 case PTHREAD_MUTEX_PP_ADAPTIVE_NP:
327 int kind = mutex->__data.__kind & PTHREAD_MUTEX_KIND_MASK_NP;
329 oldval = mutex->__data.__lock;
331 /* Check whether we already hold the mutex. */
332 if (mutex->__data.__owner == id)
334 if (kind == PTHREAD_MUTEX_ERRORCHECK_NP)
337 if (kind == PTHREAD_MUTEX_RECURSIVE_NP)
339 /* Just bump the counter. */
340 if (__builtin_expect (mutex->__data.__count + 1 == 0, 0))
341 /* Overflow of the counter. */
344 ++mutex->__data.__count;
350 int oldprio = -1, ceilval;
353 int ceiling = (oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK)
354 >> PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
356 if (__pthread_current_priority () > ceiling)
359 __pthread_tpp_change_priority (oldprio, -1);
363 int retval = __pthread_tpp_change_priority (oldprio, ceiling);
367 ceilval = ceiling << PTHREAD_MUTEX_PRIO_CEILING_SHIFT;
371 = atomic_compare_and_exchange_val_acq (&mutex->__data.__lock,
372 ceilval | 1, ceilval);
374 if (oldval == ceilval)
377 while ((oldval & PTHREAD_MUTEX_PRIO_CEILING_MASK) != ceilval);
379 if (oldval != ceilval)
381 __pthread_tpp_change_priority (oldprio, -1);
385 assert (mutex->__data.__owner == 0);
386 /* Record the ownership. */
387 mutex->__data.__owner = id;
388 ++mutex->__data.__nusers;
389 mutex->__data.__count = 1;
396 /* Correct code cannot set any other type. */
403 #ifndef __pthread_mutex_trylock
404 #ifndef pthread_mutex_trylock
405 strong_alias (__pthread_mutex_trylock, pthread_mutex_trylock)