1 /* Copyright (C) 2002,2003,2004,2005,2006,2007 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, write to the Free
17 Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
27 #include <sys/param.h>
28 #include <dl-sysdep.h>
30 #include <lowlevellock.h>
31 #include <kernel-features.h>
34 #ifndef NEED_SEPARATE_REGISTER_STACK
36 /* Most architectures have exactly one stack pointer. Some have more. */
37 # define STACK_VARIABLES void *stackaddr = NULL
39 /* How to pass the values to the 'create_thread' function. */
40 # define STACK_VARIABLES_ARGS stackaddr
42 /* How to declare function which gets there parameters. */
43 # define STACK_VARIABLES_PARMS void *stackaddr
45 /* How to declare allocate_stack. */
46 # define ALLOCATE_STACK_PARMS void **stack
48 /* This is how the function is called. We do it this way to allow
49 other variants of the function to have more parameters. */
50 # define ALLOCATE_STACK(attr, pd) allocate_stack (attr, pd, &stackaddr)
54 /* We need two stacks. The kernel will place them but we have to tell
55 the kernel about the size of the reserved address space. */
56 # define STACK_VARIABLES void *stackaddr = NULL; size_t stacksize = 0
58 /* How to pass the values to the 'create_thread' function. */
59 # define STACK_VARIABLES_ARGS stackaddr, stacksize
61 /* How to declare function which gets there parameters. */
62 # define STACK_VARIABLES_PARMS void *stackaddr, size_t stacksize
64 /* How to declare allocate_stack. */
65 # define ALLOCATE_STACK_PARMS void **stack, size_t *stacksize
67 /* This is how the function is called. We do it this way to allow
68 other variants of the function to have more parameters. */
69 # define ALLOCATE_STACK(attr, pd) \
70 allocate_stack (attr, pd, &stackaddr, &stacksize)
75 /* Default alignment of stack. */
77 # define STACK_ALIGN __alignof__ (long double)
80 /* Default value for minimal stack size after allocating thread
81 descriptor and guard. */
82 #ifndef MINIMAL_REST_STACK
83 # define MINIMAL_REST_STACK 4096
87 /* Let the architecture add some flags to the mmap() call used to
89 #ifndef ARCH_MAP_FLAGS
90 # define ARCH_MAP_FLAGS 0
93 /* This yields the pointer that TLS support code calls the thread pointer. */
95 # define TLS_TPADJ(pd) (pd)
97 # define TLS_TPADJ(pd) ((struct pthread *)((char *) (pd) + TLS_PRE_TCB_SIZE))
100 /* Cache handling for not-yet free stacks. */
102 /* Maximum size in kB of cache. */
103 static size_t stack_cache_maxsize = 40 * 1024 * 1024; /* 40MiBi by default. */
104 static size_t stack_cache_actsize;
106 /* Mutex protecting this variable. */
107 static int stack_cache_lock = LLL_LOCK_INITIALIZER;
109 /* List of queued stack frames. */
110 static LIST_HEAD (stack_cache);
112 /* List of the stacks in use. */
113 static LIST_HEAD (stack_used);
115 /* List of the threads with user provided stacks in use. No need to
116 initialize this, since it's done in __pthread_initialize_minimal. */
117 list_t __stack_user __attribute__ ((nocommon));
118 hidden_data_def (__stack_user)
120 #if COLORING_INCREMENT != 0
121 /* Number of threads created. */
122 static unsigned int nptl_ncreated;
126 /* Check whether the stack is still used or not. */
127 #define FREE_P(descr) ((descr)->tid <= 0)
130 /* We create a double linked list of all cache entries. Double linked
131 because this allows removing entries from the end. */
134 /* Get a stack frame from the cache. We have to match by size since
135 some blocks might be too small or far too large. */
136 static struct pthread *
137 get_cached_stack (size_t *sizep, void **memp)
139 size_t size = *sizep;
140 struct pthread *result = NULL;
143 lll_lock (stack_cache_lock, LLL_PRIVATE);
145 /* Search the cache for a matching entry. We search for the
146 smallest stack which has at least the required size. Note that
147 in normal situations the size of all allocated stacks is the
148 same. As the very least there are only a few different sizes.
149 Therefore this loop will exit early most of the time with an
151 list_for_each (entry, &stack_cache)
153 struct pthread *curr;
155 curr = list_entry (entry, struct pthread, list);
156 if (FREE_P (curr) && curr->stackblock_size >= size)
158 if (curr->stackblock_size == size)
165 || result->stackblock_size > curr->stackblock_size)
170 if (__builtin_expect (result == NULL, 0)
171 /* Make sure the size difference is not too excessive. In that
172 case we do not use the block. */
173 || __builtin_expect (result->stackblock_size > 4 * size, 0))
175 /* Release the lock. */
176 lll_unlock (stack_cache_lock, LLL_PRIVATE);
181 /* Dequeue the entry. */
182 list_del (&result->list);
184 /* And add to the list of stacks in use. */
185 list_add (&result->list, &stack_used);
187 /* And decrease the cache size. */
188 stack_cache_actsize -= result->stackblock_size;
190 /* Release the lock early. */
191 lll_unlock (stack_cache_lock, LLL_PRIVATE);
193 /* Report size and location of the stack to the caller. */
194 *sizep = result->stackblock_size;
195 *memp = result->stackblock;
197 /* Cancellation handling is back to the default. */
198 result->cancelhandling = 0;
199 result->cleanup = NULL;
201 /* No pending event. */
202 result->nextevent = NULL;
205 dtv_t *dtv = GET_DTV (TLS_TPADJ (result));
206 memset (dtv, '\0', (dtv[-1].counter + 1) * sizeof (dtv_t));
208 /* Re-initialize the TLS. */
209 _dl_allocate_tls_init (TLS_TPADJ (result));
215 /* Free stacks until cache size is lower than LIMIT. */
217 free_stacks (size_t limit)
219 /* We reduce the size of the cache. Remove the last entries until
220 the size is below the limit. */
224 /* Search from the end of the list. */
225 list_for_each_prev_safe (entry, prev, &stack_cache)
227 struct pthread *curr;
229 curr = list_entry (entry, struct pthread, list);
232 /* Unlink the block. */
235 /* Account for the freed memory. */
236 stack_cache_actsize -= curr->stackblock_size;
238 /* Free the memory associated with the ELF TLS. */
239 _dl_deallocate_tls (TLS_TPADJ (curr), false);
241 /* Remove this block. This should never fail. If it does
242 something is really wrong. */
243 if (munmap (curr->stackblock, curr->stackblock_size) != 0)
246 /* Maybe we have freed enough. */
247 if (stack_cache_actsize <= limit)
254 /* Add a stack frame which is not used anymore to the stack. Must be
255 called with the cache lock held. */
257 __attribute ((always_inline))
258 queue_stack (struct pthread *stack)
260 /* We unconditionally add the stack to the list. The memory may
261 still be in use but it will not be reused until the kernel marks
262 the stack as not used anymore. */
263 list_add (&stack->list, &stack_cache);
265 stack_cache_actsize += stack->stackblock_size;
266 if (__builtin_expect (stack_cache_actsize > stack_cache_maxsize, 0))
267 free_stacks (stack_cache_maxsize);
271 /* This function is called indirectly from the freeres code in libc. */
273 __free_stack_cache (void)
281 change_stack_perm (struct pthread *pd
282 #ifdef NEED_SEPARATE_REGISTER_STACK
287 #ifdef NEED_SEPARATE_REGISTER_STACK
288 void *stack = (pd->stackblock
289 + (((((pd->stackblock_size - pd->guardsize) / 2)
290 & pagemask) + pd->guardsize) & pagemask));
291 size_t len = pd->stackblock + pd->stackblock_size - stack;
292 #elif _STACK_GROWS_DOWN
293 void *stack = pd->stackblock + pd->guardsize;
294 size_t len = pd->stackblock_size - pd->guardsize;
295 #elif _STACK_GROWS_UP
296 void *stack = pd->stackblock;
297 size_t len = (uintptr_t) pd - pd->guardsize - (uintptr_t) pd->stackblock;
299 # error "Define either _STACK_GROWS_DOWN or _STACK_GROWS_UP"
301 if (mprotect (stack, len, PROT_READ | PROT_WRITE | PROT_EXEC) != 0)
309 allocate_stack (const struct pthread_attr *attr, struct pthread **pdp,
310 ALLOCATE_STACK_PARMS)
314 size_t pagesize_m1 = __getpagesize () - 1;
317 assert (attr != NULL);
318 assert (powerof2 (pagesize_m1 + 1));
319 assert (TCB_ALIGNMENT >= STACK_ALIGN);
321 /* Get the stack size from the attribute if it is set. Otherwise we
322 use the default we determined at start time. */
323 size = attr->stacksize ?: __default_stacksize;
325 /* Get memory for the stack. */
326 if (__builtin_expect (attr->flags & ATTR_FLAG_STACKADDR, 0))
330 /* If the user also specified the size of the stack make sure it
332 if (attr->stacksize != 0
333 && attr->stacksize < (__static_tls_size + MINIMAL_REST_STACK))
336 /* Adjust stack size for alignment of the TLS block. */
338 adj = ((uintptr_t) attr->stackaddr - TLS_TCB_SIZE)
339 & __static_tls_align_m1;
340 assert (size > adj + TLS_TCB_SIZE);
342 adj = ((uintptr_t) attr->stackaddr - __static_tls_size)
343 & __static_tls_align_m1;
347 /* The user provided some memory. Let's hope it matches the
348 size... We do not allocate guard pages if the user provided
349 the stack. It is the user's responsibility to do this if it
352 pd = (struct pthread *) ((uintptr_t) attr->stackaddr
353 - TLS_TCB_SIZE - adj);
355 pd = (struct pthread *) (((uintptr_t) attr->stackaddr
356 - __static_tls_size - adj)
360 /* The user provided stack memory needs to be cleared. */
361 memset (pd, '\0', sizeof (struct pthread));
363 /* The first TSD block is included in the TCB. */
364 pd->specific[0] = pd->specific_1stblock;
366 /* Remember the stack-related values. */
367 pd->stackblock = (char *) attr->stackaddr - size;
368 pd->stackblock_size = size;
370 /* This is a user-provided stack. It will not be queued in the
371 stack cache nor will the memory (except the TLS memory) be freed. */
372 pd->user_stack = true;
374 /* This is at least the second thread. */
375 pd->header.multiple_threads = 1;
376 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
377 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
380 #ifndef __ASSUME_PRIVATE_FUTEX
381 /* The thread must know when private futexes are supported. */
382 pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
383 header.private_futex);
386 #ifdef NEED_DL_SYSINFO
387 /* Copy the sysinfo value from the parent. */
388 THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
391 /* The process ID is also the same as that of the caller. */
392 pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
394 /* Allocate the DTV for this thread. */
395 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
397 /* Something went wrong. */
398 assert (errno == ENOMEM);
403 /* Prepare to modify global data. */
404 lll_lock (stack_cache_lock, LLL_PRIVATE);
406 /* And add to the list of stacks in use. */
407 list_add (&pd->list, &__stack_user);
409 lll_unlock (stack_cache_lock, LLL_PRIVATE);
413 /* Allocate some anonymous memory. If possible use the cache. */
417 const int prot = (PROT_READ | PROT_WRITE
418 | ((GL(dl_stack_flags) & PF_X) ? PROT_EXEC : 0));
420 #if COLORING_INCREMENT != 0
421 /* Add one more page for stack coloring. Don't do it for stacks
422 with 16 times pagesize or larger. This might just cause
423 unnecessary misalignment. */
424 if (size <= 16 * pagesize_m1)
425 size += pagesize_m1 + 1;
428 /* Adjust the stack size for alignment. */
429 size &= ~__static_tls_align_m1;
432 /* Make sure the size of the stack is enough for the guard and
433 eventually the thread descriptor. */
434 guardsize = (attr->guardsize + pagesize_m1) & ~pagesize_m1;
435 if (__builtin_expect (size < ((guardsize + __static_tls_size
436 + MINIMAL_REST_STACK + pagesize_m1)
439 /* The stack is too small (or the guard too large). */
442 /* Try to get a stack from the cache. */
444 pd = get_cached_stack (&size, &mem);
447 /* To avoid aliasing effects on a larger scale than pages we
448 adjust the allocated stack size if necessary. This way
449 allocations directly following each other will not have
450 aliasing problems. */
451 #if MULTI_PAGE_ALIASING != 0
452 if ((size % MULTI_PAGE_ALIASING) == 0)
453 size += pagesize_m1 + 1;
456 mem = mmap (NULL, size, prot,
457 MAP_PRIVATE | MAP_ANONYMOUS | ARCH_MAP_FLAGS, -1, 0);
459 if (__builtin_expect (mem == MAP_FAILED, 0))
461 #ifdef ARCH_RETRY_MMAP
462 mem = ARCH_RETRY_MMAP (size);
463 if (__builtin_expect (mem == MAP_FAILED, 0))
468 /* SIZE is guaranteed to be greater than zero.
469 So we can never get a null pointer back from mmap. */
470 assert (mem != NULL);
472 #if COLORING_INCREMENT != 0
473 /* Atomically increment NCREATED. */
474 unsigned int ncreated = atomic_increment_val (&nptl_ncreated);
476 /* We chose the offset for coloring by incrementing it for
477 every new thread by a fixed amount. The offset used
478 module the page size. Even if coloring would be better
479 relative to higher alignment values it makes no sense to
480 do it since the mmap() interface does not allow us to
481 specify any alignment for the returned memory block. */
482 size_t coloring = (ncreated * COLORING_INCREMENT) & pagesize_m1;
484 /* Make sure the coloring offsets does not disturb the alignment
485 of the TCB and static TLS block. */
486 if (__builtin_expect ((coloring & __static_tls_align_m1) != 0, 0))
487 coloring = (((coloring + __static_tls_align_m1)
488 & ~(__static_tls_align_m1))
491 /* Unless specified we do not make any adjustments. */
495 /* Place the thread descriptor at the end of the stack. */
497 pd = (struct pthread *) ((char *) mem + size - coloring) - 1;
499 pd = (struct pthread *) ((((uintptr_t) mem + size - coloring
501 & ~__static_tls_align_m1)
505 /* Remember the stack-related values. */
506 pd->stackblock = mem;
507 pd->stackblock_size = size;
509 /* We allocated the first block thread-specific data array.
510 This address will not change for the lifetime of this
512 pd->specific[0] = pd->specific_1stblock;
514 /* This is at least the second thread. */
515 pd->header.multiple_threads = 1;
516 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
517 __pthread_multiple_threads = *__libc_multiple_threads_ptr = 1;
520 #ifndef __ASSUME_PRIVATE_FUTEX
521 /* The thread must know when private futexes are supported. */
522 pd->header.private_futex = THREAD_GETMEM (THREAD_SELF,
523 header.private_futex);
526 #ifdef NEED_DL_SYSINFO
527 /* Copy the sysinfo value from the parent. */
528 THREAD_SYSINFO(pd) = THREAD_SELF_SYSINFO;
531 /* The process ID is also the same as that of the caller. */
532 pd->pid = THREAD_GETMEM (THREAD_SELF, pid);
534 /* Allocate the DTV for this thread. */
535 if (_dl_allocate_tls (TLS_TPADJ (pd)) == NULL)
537 /* Something went wrong. */
538 assert (errno == ENOMEM);
540 /* Free the stack memory we just allocated. */
541 (void) munmap (mem, size);
547 /* Prepare to modify global data. */
548 lll_lock (stack_cache_lock, LLL_PRIVATE);
550 /* And add to the list of stacks in use. */
551 list_add (&pd->list, &stack_used);
553 lll_unlock (stack_cache_lock, LLL_PRIVATE);
556 /* There might have been a race. Another thread might have
557 caused the stacks to get exec permission while this new
558 stack was prepared. Detect if this was possible and
559 change the permission if necessary. */
560 if (__builtin_expect ((GL(dl_stack_flags) & PF_X) != 0
561 && (prot & PROT_EXEC) == 0, 0))
563 int err = change_stack_perm (pd
564 #ifdef NEED_SEPARATE_REGISTER_STACK
570 /* Free the stack memory we just allocated. */
571 (void) munmap (mem, size);
578 /* Note that all of the stack and the thread descriptor is
579 zeroed. This means we do not have to initialize fields
580 with initial value zero. This is specifically true for
581 the 'tid' field which is always set back to zero once the
582 stack is not used anymore and for the 'guardsize' field
583 which will be read next. */
586 /* Create or resize the guard area if necessary. */
587 if (__builtin_expect (guardsize > pd->guardsize, 0))
589 #ifdef NEED_SEPARATE_REGISTER_STACK
590 char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
591 #elif _STACK_GROWS_DOWN
593 # elif _STACK_GROWS_UP
594 char *guard = (char *) (((uintptr_t) pd - guardsize) & ~pagesize_m1);
596 if (mprotect (guard, guardsize, PROT_NONE) != 0)
602 lll_lock (stack_cache_lock, LLL_PRIVATE);
604 /* Remove the thread from the list. */
605 list_del (&pd->list);
607 lll_unlock (stack_cache_lock, LLL_PRIVATE);
609 /* Get rid of the TLS block we allocated. */
610 _dl_deallocate_tls (TLS_TPADJ (pd), false);
612 /* Free the stack memory regardless of whether the size
613 of the cache is over the limit or not. If this piece
614 of memory caused problems we better do not use it
615 anymore. Uh, and we ignore possible errors. There
616 is nothing we could do. */
617 (void) munmap (mem, size);
622 pd->guardsize = guardsize;
624 else if (__builtin_expect (pd->guardsize - guardsize > size - reqsize,
627 /* The old guard area is too large. */
629 #ifdef NEED_SEPARATE_REGISTER_STACK
630 char *guard = mem + (((size - guardsize) / 2) & ~pagesize_m1);
631 char *oldguard = mem + (((size - pd->guardsize) / 2) & ~pagesize_m1);
634 && mprotect (oldguard, guard - oldguard, prot) != 0)
637 if (mprotect (guard + guardsize,
638 oldguard + pd->guardsize - guard - guardsize,
641 #elif _STACK_GROWS_DOWN
642 if (mprotect ((char *) mem + guardsize, pd->guardsize - guardsize,
645 #elif _STACK_GROWS_UP
646 if (mprotect ((char *) pd - pd->guardsize,
647 pd->guardsize - guardsize, prot) != 0)
651 pd->guardsize = guardsize;
653 /* The pthread_getattr_np() calls need to get passed the size
654 requested in the attribute, regardless of how large the
655 actually used guardsize is. */
656 pd->reported_guardsize = guardsize;
659 /* Initialize the lock. We have to do this unconditionally since the
660 stillborn thread could be canceled while the lock is taken. */
661 pd->lock = LLL_LOCK_INITIALIZER;
663 /* The robust mutex lists also need to be initialized
664 unconditionally because the cleanup for the previous stack owner
665 might have happened in the kernel. */
666 pd->robust_head.futex_offset = (offsetof (pthread_mutex_t, __data.__lock)
667 - offsetof (pthread_mutex_t,
668 __data.__list.__next));
669 pd->robust_head.list_op_pending = NULL;
670 #ifdef __PTHREAD_MUTEX_HAVE_PREV
671 pd->robust_prev = &pd->robust_head;
673 pd->robust_head.list = &pd->robust_head;
675 /* We place the thread descriptor at the end of the stack. */
679 /* The stack begins before the TCB and the static TLS block. */
680 stacktop = ((char *) (pd + 1) - __static_tls_size);
682 stacktop = (char *) (pd - 1);
685 #ifdef NEED_SEPARATE_REGISTER_STACK
686 *stack = pd->stackblock;
687 *stacksize = stacktop - *stack;
688 #elif _STACK_GROWS_DOWN
690 #elif _STACK_GROWS_UP
691 *stack = pd->stackblock;
701 __deallocate_stack (struct pthread *pd)
703 lll_lock (stack_cache_lock, LLL_PRIVATE);
705 /* Remove the thread from the list of threads with user defined
707 list_del (&pd->list);
709 /* Not much to do. Just free the mmap()ed memory. Note that we do
710 not reset the 'used' flag in the 'tid' field. This is done by
711 the kernel. If no thread has been created yet this field is
713 if (__builtin_expect (! pd->user_stack, 1))
714 (void) queue_stack (pd);
716 /* Free the memory associated with the ELF TLS. */
717 _dl_deallocate_tls (TLS_TPADJ (pd), false);
719 lll_unlock (stack_cache_lock, LLL_PRIVATE);
725 __make_stacks_executable (void **stack_endp)
727 /* First the main thread's stack. */
728 int err = _dl_make_stack_executable (stack_endp);
732 #ifdef NEED_SEPARATE_REGISTER_STACK
733 const size_t pagemask = ~(__getpagesize () - 1);
736 lll_lock (stack_cache_lock, LLL_PRIVATE);
739 list_for_each (runp, &stack_used)
741 err = change_stack_perm (list_entry (runp, struct pthread, list)
742 #ifdef NEED_SEPARATE_REGISTER_STACK
750 /* Also change the permission for the currently unused stacks. This
751 might be wasted time but better spend it here than adding a check
754 list_for_each (runp, &stack_cache)
756 err = change_stack_perm (list_entry (runp, struct pthread, list)
757 #ifdef NEED_SEPARATE_REGISTER_STACK
765 lll_unlock (stack_cache_lock, LLL_PRIVATE);
771 /* In case of a fork() call the memory allocation in the child will be
772 the same but only one thread is running. All stacks except that of
773 the one running thread are not used anymore. We have to recycle
776 __reclaim_stacks (void)
778 struct pthread *self = (struct pthread *) THREAD_SELF;
780 /* No locking necessary. The caller is the only stack in use. */
782 /* Mark all stacks except the still running one as free. */
784 list_for_each (runp, &stack_used)
786 struct pthread *curp = list_entry (runp, struct pthread, list);
789 /* This marks the stack as free. */
792 /* The PID field must be initialized for the new process. */
793 curp->pid = self->pid;
795 /* Account for the size of the stack. */
796 stack_cache_actsize += curp->stackblock_size;
800 /* Reset the PIDs in any cached stacks. */
801 list_for_each (runp, &stack_cache)
803 struct pthread *curp = list_entry (runp, struct pthread, list);
804 curp->pid = self->pid;
807 /* Add the stack of all running threads to the cache. */
808 list_splice (&stack_used, &stack_cache);
810 /* Remove the entry for the current thread to from the cache list
811 and add it to the list of running threads. Which of the two
812 lists is decided by the user_stack flag. */
813 list_del (&self->list);
815 /* Re-initialize the lists for all the threads. */
816 INIT_LIST_HEAD (&stack_used);
817 INIT_LIST_HEAD (&__stack_user);
819 if (__builtin_expect (THREAD_GETMEM (self, user_stack), 0))
820 list_add (&self->list, &__stack_user);
822 list_add (&self->list, &stack_used);
824 /* There is one thread running. */
827 /* Initialize the lock. */
828 stack_cache_lock = LLL_LOCK_INITIALIZER;
833 # undef __find_thread_by_id
834 /* Find a thread given the thread ID. */
837 __find_thread_by_id (pid_t tid)
839 struct pthread *result = NULL;
841 lll_lock (stack_cache_lock, LLL_PRIVATE);
843 /* Iterate over the list with system-allocated threads first. */
845 list_for_each (runp, &stack_used)
847 struct pthread *curp;
849 curp = list_entry (runp, struct pthread, list);
851 if (curp->tid == tid)
858 /* Now the list with threads using user-allocated stacks. */
859 list_for_each (runp, &__stack_user)
861 struct pthread *curp;
863 curp = list_entry (runp, struct pthread, list);
865 if (curp->tid == tid)
873 lll_unlock (stack_cache_lock, LLL_PRIVATE);
882 setxid_signal_thread (struct xid_command *cmdp, struct pthread *t)
884 if (! IS_DETACHED (t))
889 ch = t->cancelhandling;
891 /* If the thread is exiting right now, ignore it. */
892 if ((ch & EXITING_BITMASK) != 0)
895 while (atomic_compare_and_exchange_bool_acq (&t->cancelhandling,
896 ch | SETXID_BITMASK, ch));
900 INTERNAL_SYSCALL_DECL (err);
902 val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
906 val = INTERNAL_SYSCALL (tgkill, err, 3, THREAD_GETMEM (THREAD_SELF, pid),
908 if (INTERNAL_SYSCALL_ERROR_P (val, err)
909 && INTERNAL_SYSCALL_ERRNO (val, err) == ENOSYS)
911 val = INTERNAL_SYSCALL (tkill, err, 2, t->tid, SIGSETXID);
914 if (!INTERNAL_SYSCALL_ERROR_P (val, err))
915 atomic_increment (&cmdp->cntr);
921 __nptl_setxid (struct xid_command *cmdp)
924 lll_lock (stack_cache_lock, LLL_PRIVATE);
929 struct pthread *self = THREAD_SELF;
931 /* Iterate over the list with system-allocated threads first. */
933 list_for_each (runp, &stack_used)
935 struct pthread *t = list_entry (runp, struct pthread, list);
939 setxid_signal_thread (cmdp, t);
942 /* Now the list with threads using user-allocated stacks. */
943 list_for_each (runp, &__stack_user)
945 struct pthread *t = list_entry (runp, struct pthread, list);
949 setxid_signal_thread (cmdp, t);
952 int cur = cmdp->cntr;
955 lll_futex_wait (&cmdp->cntr, cur, LLL_PRIVATE);
959 /* This must be last, otherwise the current thread might not have
960 permissions to send SIGSETXID syscall to the other threads. */
961 INTERNAL_SYSCALL_DECL (err);
962 result = INTERNAL_SYSCALL_NCS (cmdp->syscall_no, err, 3,
963 cmdp->id[0], cmdp->id[1], cmdp->id[2]);
964 if (INTERNAL_SYSCALL_ERROR_P (result, err))
966 __set_errno (INTERNAL_SYSCALL_ERRNO (result, err));
970 lll_unlock (stack_cache_lock, LLL_PRIVATE);
974 static inline void __attribute__((always_inline))
975 init_one_static_tls (struct pthread *curp, struct link_map *map)
977 dtv_t *dtv = GET_DTV (TLS_TPADJ (curp));
979 void *dest = (char *) curp - map->l_tls_offset;
981 void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
983 # error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
986 /* Fill in the DTV slot so that a later LD/GD access will find it. */
987 dtv[map->l_tls_modid].pointer.val = dest;
988 dtv[map->l_tls_modid].pointer.is_static = true;
990 /* Initialize the memory. */
991 memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
992 '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
997 __pthread_init_static_tls (struct link_map *map)
999 lll_lock (stack_cache_lock, LLL_PRIVATE);
1001 /* Iterate over the list with system-allocated threads first. */
1003 list_for_each (runp, &stack_used)
1004 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1006 /* Now the list with threads using user-allocated stacks. */
1007 list_for_each (runp, &__stack_user)
1008 init_one_static_tls (list_entry (runp, struct pthread, list), map);
1010 lll_unlock (stack_cache_lock, LLL_PRIVATE);
1016 __wait_lookup_done (void)
1018 lll_lock (stack_cache_lock, LLL_PRIVATE);
1020 struct pthread *self = THREAD_SELF;
1022 /* Iterate over the list with system-allocated threads first. */
1024 list_for_each (runp, &stack_used)
1026 struct pthread *t = list_entry (runp, struct pthread, list);
1027 if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1030 int *const gscope_flagp = &t->header.gscope_flag;
1032 /* We have to wait until this thread is done with the global
1033 scope. First tell the thread that we are waiting and
1034 possibly have to be woken. */
1035 if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1036 THREAD_GSCOPE_FLAG_WAIT,
1037 THREAD_GSCOPE_FLAG_USED))
1041 lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
1042 while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1045 /* Now the list with threads using user-allocated stacks. */
1046 list_for_each (runp, &__stack_user)
1048 struct pthread *t = list_entry (runp, struct pthread, list);
1049 if (t == self || t->header.gscope_flag == THREAD_GSCOPE_FLAG_UNUSED)
1052 int *const gscope_flagp = &t->header.gscope_flag;
1054 /* We have to wait until this thread is done with the global
1055 scope. First tell the thread that we are waiting and
1056 possibly have to be woken. */
1057 if (atomic_compare_and_exchange_bool_acq (gscope_flagp,
1058 THREAD_GSCOPE_FLAG_WAIT,
1059 THREAD_GSCOPE_FLAG_USED))
1063 lll_futex_wait (gscope_flagp, THREAD_GSCOPE_FLAG_WAIT, LLL_PRIVATE);
1064 while (*gscope_flagp == THREAD_GSCOPE_FLAG_WAIT);
1067 lll_unlock (stack_cache_lock, LLL_PRIVATE);