1 /* Copyright (C) 2005-2022 Free Software Foundation, Inc.
2 Contributed by Richard Henderson <rth@redhat.com>.
4 This file is part of the GNU Offloading and Multi Processing Library
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
26 /* This file contains data types and function declarations that are not
27 part of the official OpenACC or OpenMP user interfaces. There are
28 declarations in here that are part of the GNU Offloading and Multi
29 Processing ABI, in that the compiler is required to know about them
32 The convention is that the all caps prefix "GOMP" is used group items
33 that are part of the external ABI, and the lower case prefix "gomp"
34 is used group items that are completely private to the library. */
39 #ifndef _LIBGOMP_CHECKING_
40 /* Define to 1 to perform internal sanity checks. */
41 #define _LIBGOMP_CHECKING_ 0
46 #include "libgomp-plugin.h"
47 #include "gomp-constants.h"
56 /* Needed for memset in priority_queue.c. */
57 #if _LIBGOMP_CHECKING_
58 # ifdef STRING_WITH_STRINGS
65 # ifdef HAVE_STRINGS_H
72 #ifdef HAVE_ATTRIBUTE_VISIBILITY
73 # pragma GCC visibility push(hidden)
76 /* If we were a C++ library, we'd get this from <std/atomic>. */
89 #if defined(HAVE_ALIGNED_ALLOC) \
90 || defined(HAVE_POSIX_MEMALIGN) \
91 || defined(HAVE_MEMALIGN)
92 /* Defined if gomp_aligned_alloc doesn't use fallback version
93 and free can be used instead of gomp_aligned_free. */
94 #define GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC 1
97 #if defined(GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC) && !defined(__AMDGCN__)
98 #define GOMP_USE_ALIGNED_WORK_SHARES 1
101 extern void *gomp_malloc (size_t) __attribute__((malloc
));
102 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc
));
103 extern void *gomp_realloc (void *, size_t);
104 extern void *gomp_aligned_alloc (size_t, size_t)
105 __attribute__((malloc
, alloc_size (2)));
106 extern void gomp_aligned_free (void *);
108 /* Avoid conflicting prototypes of alloca() in system headers by using
109 GCC's builtin alloca(). */
110 #define gomp_alloca(x) __builtin_alloca(x)
112 /* Optimized allocators for team-specific data that will die with the team. */
115 /* The arena is initialized in config/gcn/team.c. */
116 #define TEAM_ARENA_SIZE 64*1024 /* Must match the value in plugin-gcn.c. */
117 #define TEAM_ARENA_START 16 /* LDS offset of free pointer. */
118 #define TEAM_ARENA_FREE 24 /* LDS offset of free pointer. */
119 #define TEAM_ARENA_END 32 /* LDS offset of end pointer. */
121 static inline void * __attribute__((malloc
))
122 team_malloc (size_t size
)
124 /* 4-byte align the size. */
125 size
= (size
+ 3) & ~3;
127 /* Allocate directly from the arena.
128 The compiler does not support DS atomics, yet. */
130 asm ("ds_add_rtn_u64 %0, %1, %2\n\ts_waitcnt 0"
131 : "=v"(result
) : "v"(TEAM_ARENA_FREE
), "v"(size
), "e"(1L) : "memory");
134 if (result
+ size
> *(void * __lds
*)TEAM_ARENA_END
)
136 /* While this is experimental, let's make sure we know when OOM
138 const char msg
[] = "GCN team arena exhausted\n";
139 write (2, msg
, sizeof(msg
)-1);
141 /* Fall back to using the heap (slowly). */
142 result
= gomp_malloc (size
);
147 static inline void * __attribute__((malloc
))
148 team_malloc_cleared (size_t size
)
150 char *result
= team_malloc (size
);
152 /* Clear the allocated memory. */
153 __builtin_memset (result
, 0, size
);
159 team_free (void *ptr
)
161 /* The whole arena is freed when the kernel exits.
162 However, if we fell back to using heap then we should free it.
163 It would be better if this function could be a no-op, but at least
164 LDS loads are cheap. */
165 if (ptr
< *(void * __lds
*)TEAM_ARENA_START
166 || ptr
>= *(void * __lds
*)TEAM_ARENA_END
)
170 #define team_malloc(...) gomp_malloc (__VA_ARGS__)
171 #define team_malloc_cleared(...) gomp_malloc_cleared (__VA_ARGS__)
172 #define team_free(...) free (__VA_ARGS__)
177 extern void gomp_vdebug (int, const char *, va_list);
178 extern void gomp_debug (int, const char *, ...)
179 __attribute__ ((format (printf
, 2, 3)));
180 #define gomp_vdebug(KIND, FMT, VALIST) \
182 if (__builtin_expect (gomp_debug_var, 0)) \
183 (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
185 #define gomp_debug(KIND, ...) \
187 if (__builtin_expect (gomp_debug_var, 0)) \
188 (gomp_debug) ((KIND), __VA_ARGS__); \
190 extern void gomp_verror (const char *, va_list);
191 extern void gomp_error (const char *, ...)
192 __attribute__ ((format (printf
, 1, 2)));
193 extern void gomp_vfatal (const char *, va_list)
194 __attribute__ ((noreturn
));
195 extern void gomp_fatal (const char *, ...)
196 __attribute__ ((noreturn
, format (printf
, 1, 2)));
199 struct gomp_taskgroup
;
202 #include "priority_queue.h"
206 #include "simple-bar.h"
210 /* This structure contains the data to control one work-sharing construct,
211 either a LOOP (FOR/DO) or a SECTIONS. */
213 enum gomp_schedule_type
220 GFS_MONOTONIC
= 0x80000000U
223 struct gomp_doacross_work_share
226 /* chunk_size copy, as ws->chunk_size is multiplied by incr for
229 /* Likewise, but for ull implementation. */
230 unsigned long long chunk_size_ull
;
231 /* For schedule(static,0) this is the number
232 of iterations assigned to the last thread, i.e. number of
233 iterations / number of threads. */
235 /* Likewise, but for ull implementation. */
236 unsigned long long q_ull
;
238 /* Size of each array entry (padded to cache line size). */
239 unsigned long elt_sz
;
240 /* Number of dimensions in sink vectors. */
241 unsigned int ncounts
;
242 /* True if the iterations can be flattened. */
244 /* Actual array (of elt_sz sized units), aligned to cache line size.
245 This is indexed by team_id for GFS_STATIC and outermost iteration
246 / chunk_size for other schedules. */
247 unsigned char *array
;
248 /* These two are only used for schedule(static,0). */
249 /* This one is number of iterations % number of threads. */
252 /* And this one is cached t * (q + 1). */
254 /* Likewise, but for the ull implementation. */
255 unsigned long long boundary_ull
;
257 /* Pointer to extra memory if needed for lastprivate(conditional). */
259 /* Array of shift counts for each dimension if they can be flattened. */
260 unsigned int shift_counts
[];
263 /* Like struct gomp_work_share, but only the 1st cacheline of it plus
264 flexible array at the end.
265 Keep in sync with struct gomp_work_share. */
266 struct gomp_work_share_1st_cacheline
268 enum gomp_schedule_type sched
;
272 long chunk_size
, end
, incr
;
275 unsigned long long chunk_size_ull
, end_ull
, incr_ull
;
279 unsigned *ordered_team_ids
;
280 struct gomp_doacross_work_share
*doacross
;
282 unsigned ordered_num_used
, ordered_owner
, ordered_cur
;
283 struct gomp_work_share
*next_alloc
;
287 struct gomp_work_share
289 /* This member records the SCHEDULE clause to be used for this construct.
290 The user specification of "runtime" will already have been resolved.
291 If this is a SECTIONS construct, this value will always be DYNAMIC. */
292 enum gomp_schedule_type sched
;
298 /* This is the chunk_size argument to the SCHEDULE clause. */
301 /* This is the iteration end point. If this is a SECTIONS construct,
302 this is the number of contained sections. */
305 /* This is the iteration step. If this is a SECTIONS construct, this
311 /* The same as above, but for the unsigned long long loop variants. */
312 unsigned long long chunk_size_ull
;
313 unsigned long long end_ull
;
314 unsigned long long incr_ull
;
319 /* This is a circular queue that details which threads will be allowed
320 into the ordered region and in which order. When a thread allocates
321 iterations on which it is going to work, it also registers itself at
322 the end of the array. When a thread reaches the ordered region, it
323 checks to see if it is the one at the head of the queue. If not, it
324 blocks on its RELEASE semaphore. */
325 unsigned *ordered_team_ids
;
327 /* This is a pointer to DOACROSS work share data. */
328 struct gomp_doacross_work_share
*doacross
;
331 /* This is the number of threads that have registered themselves in
332 the circular queue ordered_team_ids. */
333 unsigned ordered_num_used
;
335 /* This is the team_id of the currently acknowledged owner of the ordered
336 section, or -1u if the ordered section has not been acknowledged by
337 any thread. This is distinguished from the thread that is *allowed*
338 to take the section next. */
339 unsigned ordered_owner
;
341 /* This is the index into the circular queue ordered_team_ids of the
342 current thread that's allowed into the ordered reason. */
343 unsigned ordered_cur
;
345 /* This is a chain of allocated gomp_work_share blocks, valid only
346 in the first gomp_work_share struct in the block. */
347 struct gomp_work_share
*next_alloc
;
349 /* The above fields are written once during workshare initialization,
350 or related to ordered worksharing. Make sure the following fields
351 are in a different cache line. */
353 /* This lock protects the update of the following members. */
354 #ifdef GOMP_USE_ALIGNED_WORK_SHARES
355 gomp_mutex_t lock
__attribute__((aligned (64)));
357 char pad
[64 - offsetof (struct gomp_work_share_1st_cacheline
, pad
)];
361 /* This is the count of the number of threads that have exited the work
362 share construct. If the construct was marked nowait, they have moved on
363 to other work; otherwise they're blocked on a barrier. The last member
364 of the team to exit the work share construct must deallocate it. */
365 unsigned threads_completed
;
368 /* This is the next iteration value to be allocated. In the case of
369 GFS_STATIC loops, this the iteration start point and never changes. */
372 /* The same, but with unsigned long long type. */
373 unsigned long long next_ull
;
375 /* This is the returned data structure for SINGLE COPYPRIVATE. */
380 /* Link to gomp_work_share struct for next work sharing construct
381 encountered after this one. */
382 gomp_ptrlock_t next_ws
;
384 /* gomp_work_share structs are chained in the free work share cache
386 struct gomp_work_share
*next_free
;
389 /* Task reductions for this work-sharing construct. */
390 uintptr_t *task_reductions
;
392 /* If only few threads are in the team, ordered_team_ids can point
393 to this array which fills the padding at the end of this struct. */
394 unsigned inline_ordered_team_ids
[0];
397 extern char gomp_workshare_struct_check1
398 [offsetof (struct gomp_work_share_1st_cacheline
, next_alloc
)
399 == offsetof (struct gomp_work_share
, next_alloc
) ? 1 : -1];
400 extern char gomp_workshare_struct_check2
401 [offsetof (struct gomp_work_share
, lock
) == 64 ? 1 : -1];
403 /* This structure contains all of the thread-local data associated with
404 a thread team. This is the data that must be saved when a thread
405 encounters a nested PARALLEL construct. */
407 struct gomp_team_state
409 /* This is the team of which the thread is currently a member. */
410 struct gomp_team
*team
;
412 /* This is the work share construct which this thread is currently
413 processing. Recall that with NOWAIT, not all threads may be
414 processing the same construct. */
415 struct gomp_work_share
*work_share
;
417 /* This is the previous work share construct or NULL if there wasn't any.
418 When all threads are done with the current work sharing construct,
419 the previous one can be freed. The current one can't, as its
420 next_ws field is used. */
421 struct gomp_work_share
*last_work_share
;
423 /* This is the ID of this thread within the team. This value is
424 guaranteed to be between 0 and N-1, where N is the number of
425 threads in the team. */
431 /* Active nesting level. Only active parallel regions are counted. */
432 unsigned active_level
;
434 /* Place-partition-var, offset and length into gomp_places_list array. */
435 unsigned place_partition_off
;
436 unsigned place_partition_len
;
438 /* Def-allocator-var ICV. */
439 uintptr_t def_allocator
;
441 #ifdef HAVE_SYNC_BUILTINS
442 /* Number of single stmts encountered. */
443 unsigned long single_count
;
446 /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the
447 trip number through the loop. So first time a particular loop
448 is encountered this number is 0, the second time through the loop
449 is 1, etc. This is unused when the compiler knows in advance that
450 the loop is statically scheduled. */
451 unsigned long static_trip
;
454 struct target_mem_desc
;
456 /* These are the OpenMP 4.0 Internal Control Variables described in
457 section 2.3.1. Those described as having one copy per task are
458 stored within the structure; those described as having one copy
459 for the whole program are (naturally) global variables. */
463 unsigned long nthreads_var
;
464 enum gomp_schedule_type run_sched_var
;
465 int run_sched_chunk_size
;
466 int default_device_var
;
467 unsigned int thread_limit_var
;
469 unsigned char max_active_levels_var
;
472 struct target_mem_desc
*target_data
;
475 enum gomp_target_offload_t
477 GOMP_TARGET_OFFLOAD_DEFAULT
,
478 GOMP_TARGET_OFFLOAD_MANDATORY
,
479 GOMP_TARGET_OFFLOAD_DISABLED
482 #define gomp_supported_active_levels UCHAR_MAX
484 extern struct gomp_task_icv gomp_global_icv
;
485 #ifndef HAVE_SYNC_BUILTINS
486 extern gomp_mutex_t gomp_managed_threads_lock
;
488 extern bool gomp_cancel_var
;
489 extern enum gomp_target_offload_t gomp_target_offload_var
;
490 extern int gomp_max_task_priority_var
;
491 extern unsigned long long gomp_spin_count_var
, gomp_throttled_spin_count_var
;
492 extern unsigned long gomp_available_cpus
, gomp_managed_threads
;
493 extern unsigned long *gomp_nthreads_var_list
, gomp_nthreads_var_list_len
;
494 extern char *gomp_bind_var_list
;
495 extern unsigned long gomp_bind_var_list_len
;
496 extern void **gomp_places_list
;
497 extern unsigned long gomp_places_list_len
;
498 extern unsigned int gomp_num_teams_var
;
499 extern int gomp_nteams_var
;
500 extern int gomp_teams_thread_limit_var
;
501 extern int gomp_debug_var
;
502 extern bool gomp_display_affinity_var
;
503 extern char *gomp_affinity_format_var
;
504 extern size_t gomp_affinity_format_len
;
505 extern uintptr_t gomp_def_allocator
;
506 extern int goacc_device_num
;
507 extern char *goacc_device_type
;
508 extern int goacc_default_dims
[GOMP_DIM_MAX
];
514 /* Undeferred task. */
515 GOMP_TASK_UNDEFERRED
,
516 /* Task created by GOMP_task and waiting to be run. */
518 /* Task currently executing or scheduled and about to execute. */
520 /* Used for target tasks that have vars mapped and async run started,
521 but not yet completed. Once that completes, they will be readded
522 into the queues as GOMP_TASK_WAITING in order to perform the var
524 GOMP_TASK_ASYNC_RUNNING
,
525 /* Task that has finished executing but is waiting for its
526 completion event to be fulfilled. */
530 struct gomp_task_depend_entry
532 /* Address of dependency. */
534 struct gomp_task_depend_entry
*next
;
535 struct gomp_task_depend_entry
*prev
;
536 /* Task that provides the dependency in ADDR. */
537 struct gomp_task
*task
;
538 /* Depend entry is of type "IN" (1) or "INOUTSET" (2). */
544 struct gomp_dependers_vec
548 struct gomp_task
*elem
[];
551 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies. */
557 /* Number of tasks we are waiting for. */
559 gomp_sem_t taskwait_sem
;
562 /* This structure describes a "task" to be run by a thread. */
566 /* Parent of this task. */
567 struct gomp_task
*parent
;
568 /* Children of this task. */
569 struct priority_queue children_queue
;
570 /* Taskgroup this task belongs in. */
571 struct gomp_taskgroup
*taskgroup
;
572 /* Tasks that depend on this task. */
573 struct gomp_dependers_vec
*dependers
;
574 struct htab
*depend_hash
;
575 struct gomp_taskwait
*taskwait
;
576 /* Last depend({,in}out:omp_all_memory) child if any. */
577 struct gomp_task
*depend_all_memory
;
578 /* Number of items in DEPEND. */
580 /* Number of tasks this task depends on. Once this counter reaches
581 0, we have no unsatisfied dependencies, and this task can be put
582 into the various queues to be scheduled. */
583 size_t num_dependees
;
586 /* Valid only if deferred_p is false. */
587 gomp_sem_t
*completion_sem
;
588 /* Valid only if deferred_p is true. Set to the team that executes the
589 task if the task is detached and the completion event has yet to be
591 struct gomp_team
*detach_team
;
595 /* Priority of this task. */
597 /* The priority node for this task in each of the different queues.
598 We put this here to avoid allocating space for each priority
599 node. Then we play offsetof() games to convert between pnode[]
600 entries and the gomp_task in which they reside. */
601 struct priority_node pnode
[3];
603 struct gomp_task_icv icv
;
606 enum gomp_task_kind kind
;
609 bool copy_ctors_done
;
610 /* Set for undeferred tasks with unsatisfied dependencies which
611 block further execution of their parent until the dependencies
613 bool parent_depends_on
;
614 /* Dependencies provided and/or needed for this task. DEPEND_COUNT
615 is the number of items available. */
616 struct gomp_task_depend_entry depend
[];
619 /* This structure describes a single #pragma omp taskgroup. */
621 struct gomp_taskgroup
623 struct gomp_taskgroup
*prev
;
624 /* Queue of tasks that belong in this taskgroup. */
625 struct priority_queue taskgroup_queue
;
626 uintptr_t *reductions
;
627 bool in_taskgroup_wait
;
630 gomp_sem_t taskgroup_sem
;
634 /* Various state of OpenMP async offloading tasks. */
635 enum gomp_target_task_state
637 GOMP_TARGET_TASK_DATA
,
638 GOMP_TARGET_TASK_BEFORE_MAP
,
639 GOMP_TARGET_TASK_FALLBACK
,
640 GOMP_TARGET_TASK_READY_TO_RUN
,
641 GOMP_TARGET_TASK_RUNNING
,
642 GOMP_TARGET_TASK_FINISHED
645 /* This structure describes a target task. */
647 struct gomp_target_task
649 struct gomp_device_descr
*devicep
;
653 unsigned short *kinds
;
655 enum gomp_target_task_state state
;
656 struct target_mem_desc
*tgt
;
657 struct gomp_task
*task
;
658 struct gomp_team
*team
;
659 /* Device-specific target arguments. */
664 /* This structure describes a "team" of threads. These are the threads
665 that are spawned by a PARALLEL constructs, as well as the work sharing
666 constructs that the team encounters. */
670 /* This is the number of threads in the current team. */
673 /* This is number of gomp_work_share structs that have been allocated
674 as a block last time. */
675 unsigned work_share_chunk
;
677 /* This is the saved team state that applied to a master thread before
678 the current thread was created. */
679 struct gomp_team_state prev_ts
;
681 /* This semaphore should be used by the master thread instead of its
682 "native" semaphore in the thread structure. Required for nested
683 parallels, as the master is a member of two teams. */
684 gomp_sem_t master_release
;
686 /* This points to an array with pointers to the release semaphore
687 of the threads in the team. */
688 gomp_sem_t
**ordered_release
;
690 /* List of work shares on which gomp_fini_work_share hasn't been
691 called yet. If the team hasn't been cancelled, this should be
692 equal to each thr->ts.work_share, but otherwise it can be a possibly
693 long list of workshares. */
694 struct gomp_work_share
*work_shares_to_free
;
696 /* List of gomp_work_share structs chained through next_free fields.
697 This is populated and taken off only by the first thread in the
698 team encountering a new work sharing construct, in a critical
700 struct gomp_work_share
*work_share_list_alloc
;
702 /* List of gomp_work_share structs freed by free_work_share. New
703 entries are atomically added to the start of the list, and
704 alloc_work_share can safely only move all but the first entry
705 to work_share_list alloc, as free_work_share can happen concurrently
706 with alloc_work_share. */
707 struct gomp_work_share
*work_share_list_free
;
709 #ifdef HAVE_SYNC_BUILTINS
710 /* Number of simple single regions encountered by threads in this
712 unsigned long single_count
;
714 /* Mutex protecting addition of workshares to work_share_list_free. */
715 gomp_mutex_t work_share_list_free_lock
;
718 /* This barrier is used for most synchronization of the team. */
719 gomp_barrier_t barrier
;
721 /* Initial work shares, to avoid allocating any gomp_work_share
722 structs in the common case. */
723 struct gomp_work_share work_shares
[8];
725 gomp_mutex_t task_lock
;
726 /* Scheduled tasks. */
727 struct priority_queue task_queue
;
728 /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */
729 unsigned int task_count
;
730 /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */
731 unsigned int task_queued_count
;
732 /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
733 directly in gomp_barrier_handle_tasks; tasks spawned
734 from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
735 that is called from a task run from gomp_barrier_handle_tasks.
736 task_running_count should be always <= team->nthreads,
737 and if current task isn't in_tied_task, then it will be
738 even < team->nthreads. */
739 unsigned int task_running_count
;
740 int work_share_cancelled
;
743 /* Number of tasks waiting for their completion event to be fulfilled. */
744 unsigned int task_detach_count
;
746 /* This array contains structures for implicit tasks. */
747 struct gomp_task implicit_task
[];
750 /* This structure contains all data that is private to libgomp and is
751 allocated per thread. */
755 /* This is the function that the thread should run upon launch. */
756 void (*fn
) (void *data
);
759 /* This is the current team state for this thread. The ts.team member
760 is NULL only if the thread is idle. */
761 struct gomp_team_state ts
;
763 /* This is the task that the thread is currently executing. */
764 struct gomp_task
*task
;
766 /* This semaphore is used for ordered loops. */
769 /* Place this thread is bound to plus one, or zero if not bound
773 /* User pthread thread pool */
774 struct gomp_thread_pool
*thread_pool
;
776 #ifdef LIBGOMP_USE_PTHREADS
777 /* omp_get_num_teams () - 1. */
778 unsigned int num_teams
;
780 /* omp_get_team_num (). */
781 unsigned int team_num
;
784 #if defined(LIBGOMP_USE_PTHREADS) \
785 && (!defined(HAVE_TLS) \
786 || !defined(__GLIBC__) \
787 || !defined(USING_INITIAL_EXEC_TLS))
788 /* pthread_t of the thread containing this gomp_thread.
789 On Linux when using initial-exec TLS,
790 (typeof (pthread_t)) gomp_thread () - pthread_self ()
791 is constant in all threads, so we can optimize and not
793 #define GOMP_NEEDS_THREAD_HANDLE 1
799 struct gomp_thread_pool
801 /* This array manages threads spawned from the top level, which will
802 return to the idle loop once the current PARALLEL construct ends. */
803 struct gomp_thread
**threads
;
804 unsigned threads_size
;
805 unsigned threads_used
;
806 /* The last team is used for non-nested teams to delay their destruction to
807 make sure all the threads in the team move on to the pool's barrier before
808 the team's barrier is destroyed. */
809 struct gomp_team
*last_team
;
810 /* Number of threads running in this contention group. */
811 unsigned long threads_busy
;
813 /* This barrier holds and releases threads waiting in thread pools. */
814 gomp_simple_barrier_t threads_dock
;
817 enum gomp_cancel_kind
819 GOMP_CANCEL_PARALLEL
= 1,
820 GOMP_CANCEL_LOOP
= 2,
821 GOMP_CANCEL_FOR
= GOMP_CANCEL_LOOP
,
822 GOMP_CANCEL_DO
= GOMP_CANCEL_LOOP
,
823 GOMP_CANCEL_SECTIONS
= 4,
824 GOMP_CANCEL_TASKGROUP
= 8
827 /* ... and here is that TLS data. */
829 #if defined __nvptx__
830 extern struct gomp_thread
*nvptx_thrs
__attribute__((shared
));
831 static inline struct gomp_thread
*gomp_thread (void)
834 asm ("mov.u32 %0, %%tid.y;" : "=r" (tid
));
835 return nvptx_thrs
+ tid
;
837 #elif defined __AMDGCN__
838 static inline struct gomp_thread
*gcn_thrs (void)
840 /* The value is at the bottom of LDS. */
841 struct gomp_thread
* __lds
*thrs
= (struct gomp_thread
* __lds
*)4;
844 static inline void set_gcn_thrs (struct gomp_thread
*val
)
846 /* The value is at the bottom of LDS. */
847 struct gomp_thread
* __lds
*thrs
= (struct gomp_thread
* __lds
*)4;
850 static inline struct gomp_thread
*gomp_thread (void)
852 int tid
= __builtin_gcn_dim_pos(1);
853 return gcn_thrs () + tid
;
855 #elif defined HAVE_TLS || defined USE_EMUTLS
856 extern __thread
struct gomp_thread gomp_tls_data
;
857 static inline struct gomp_thread
*gomp_thread (void)
859 return &gomp_tls_data
;
862 extern pthread_key_t gomp_tls_key
;
863 static inline struct gomp_thread
*gomp_thread (void)
865 return pthread_getspecific (gomp_tls_key
);
869 extern struct gomp_task_icv
*gomp_new_icv (void);
871 /* Here's how to access the current copy of the ICVs. */
873 static inline struct gomp_task_icv
*gomp_icv (bool write
)
875 struct gomp_task
*task
= gomp_thread ()->task
;
879 return gomp_new_icv ();
881 return &gomp_global_icv
;
884 #ifdef LIBGOMP_USE_PTHREADS
885 /* The attributes to be used during thread creation. */
886 extern pthread_attr_t gomp_thread_attr
;
888 extern pthread_key_t gomp_thread_destructor
;
891 /* Function prototypes. */
895 extern void gomp_init_affinity (void);
896 #ifdef LIBGOMP_USE_PTHREADS
897 extern void gomp_init_thread_affinity (pthread_attr_t
*, unsigned int);
899 extern void **gomp_affinity_alloc (unsigned long, bool);
900 extern void gomp_affinity_init_place (void *);
901 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
903 extern bool gomp_affinity_remove_cpu (void *, unsigned long);
904 extern bool gomp_affinity_copy_place (void *, void *, long);
905 extern bool gomp_affinity_same_place (void *, void *);
906 extern bool gomp_affinity_finalize_place_list (bool);
907 extern bool gomp_affinity_init_level (int, unsigned long, bool);
908 extern void gomp_affinity_print_place (void *);
909 extern void gomp_get_place_proc_ids_8 (int, int64_t *);
910 extern void gomp_display_affinity_place (char *, size_t, size_t *, int);
914 extern bool gomp_print_string (const char *str
, size_t len
);
915 extern void gomp_set_affinity_format (const char *, size_t);
916 extern void gomp_display_string (char *, size_t, size_t *, const char *,
918 #ifdef LIBGOMP_USE_PTHREADS
919 typedef pthread_t gomp_thread_handle
;
921 typedef struct {} gomp_thread_handle
;
923 extern size_t gomp_display_affinity (char *, size_t, const char *,
925 struct gomp_team_state
*, unsigned int);
926 extern void gomp_display_affinity_thread (gomp_thread_handle
,
927 struct gomp_team_state
*,
928 unsigned int) __attribute__((cold
));
932 extern int gomp_iter_static_next (long *, long *);
933 extern bool gomp_iter_dynamic_next_locked (long *, long *);
934 extern bool gomp_iter_guided_next_locked (long *, long *);
936 #ifdef HAVE_SYNC_BUILTINS
937 extern bool gomp_iter_dynamic_next (long *, long *);
938 extern bool gomp_iter_guided_next (long *, long *);
943 extern int gomp_iter_ull_static_next (unsigned long long *,
944 unsigned long long *);
945 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *,
946 unsigned long long *);
947 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *,
948 unsigned long long *);
950 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
951 extern bool gomp_iter_ull_dynamic_next (unsigned long long *,
952 unsigned long long *);
953 extern bool gomp_iter_ull_guided_next (unsigned long long *,
954 unsigned long long *);
959 extern void gomp_ordered_first (void);
960 extern void gomp_ordered_last (void);
961 extern void gomp_ordered_next (void);
962 extern void gomp_ordered_static_init (void);
963 extern void gomp_ordered_static_next (void);
964 extern void gomp_ordered_sync (void);
965 extern void gomp_doacross_init (unsigned, long *, long, size_t);
966 extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
967 unsigned long long, size_t);
971 extern unsigned gomp_resolve_num_threads (unsigned, unsigned);
973 /* proc.c (in config/) */
975 extern void gomp_init_num_threads (void);
976 extern unsigned gomp_dynamic_max_threads (void);
980 extern void gomp_init_task (struct gomp_task
*, struct gomp_task
*,
981 struct gomp_task_icv
*);
982 extern void gomp_end_task (void);
983 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t
);
984 extern void gomp_task_maybe_wait_for_dependencies (void **);
985 extern bool gomp_create_target_task (struct gomp_device_descr
*,
986 void (*) (void *), size_t, void **,
987 size_t *, unsigned short *, unsigned int,
989 enum gomp_target_task_state
);
990 extern struct gomp_taskgroup
*gomp_parallel_reduction_register (uintptr_t *,
992 extern void gomp_workshare_taskgroup_start (void);
993 extern void gomp_workshare_task_reduction_register (uintptr_t *, uintptr_t *);
996 gomp_finish_task (struct gomp_task
*task
)
998 if (__builtin_expect (task
->depend_hash
!= NULL
, 0))
999 free (task
->depend_hash
);
1004 extern struct gomp_team
*gomp_new_team (unsigned);
1005 extern void gomp_team_start (void (*) (void *), void *, unsigned,
1006 unsigned, struct gomp_team
*,
1007 struct gomp_taskgroup
*);
1008 extern void gomp_team_end (void);
1009 extern void gomp_free_thread (void *);
1010 extern int gomp_pause_host (void);
1014 extern void gomp_init_targets_once (void);
1015 extern int gomp_get_num_devices (void);
1016 extern bool gomp_target_task_fn (void *);
1018 /* Splay tree definitions. */
1019 typedef struct splay_tree_node_s
*splay_tree_node
;
1020 typedef struct splay_tree_s
*splay_tree
;
1021 typedef struct splay_tree_key_s
*splay_tree_key
;
1023 struct target_var_desc
{
1026 /* True if data should be copied from device to host at the end. */
1028 /* True if data always should be copied from device to host at the end. */
1029 bool always_copy_from
;
1030 /* True if this is for OpenACC 'attach'. */
1032 /* If GOMP_MAP_TO_PSET had a NULL pointer; used for Fortran descriptors,
1033 which were initially unallocated. */
1034 bool has_null_ptr_assoc
;
1035 /* Relative offset against key host_start. */
1037 /* Actual length. */
1041 struct target_mem_desc
{
1042 /* Reference count. */
1044 /* All the splay nodes allocated together. */
1045 splay_tree_node array
;
1046 /* Start of the target region. */
1047 uintptr_t tgt_start
;
1048 /* End of the targer region. */
1050 /* Handle to free. */
1052 /* Previous target_mem_desc. */
1053 struct target_mem_desc
*prev
;
1054 /* Number of items in following list. */
1057 /* Corresponding target device descriptor. */
1058 struct gomp_device_descr
*device_descr
;
1060 /* List of target items to remove (or decrease refcount)
1061 at the end of region. */
1062 struct target_var_desc list
[];
1065 /* Special value for refcount - mask to indicate existence of special
1066 values. Right now we allocate 3 bits. */
1067 #define REFCOUNT_SPECIAL (~(uintptr_t) 0x7)
1069 /* Special value for refcount - infinity. */
1070 #define REFCOUNT_INFINITY (REFCOUNT_SPECIAL | 0)
1071 /* Special value for refcount - tgt_offset contains target address of the
1072 artificial pointer to "omp declare target link" object. */
1073 #define REFCOUNT_LINK (REFCOUNT_SPECIAL | 1)
1075 /* Special value for refcount - structure element sibling list items.
1076 All such key refounts have REFCOUNT_STRUCTELEM bits set, with _FLAG_FIRST
1077 and _FLAG_LAST indicating first and last in the created sibling sequence. */
1078 #define REFCOUNT_STRUCTELEM (REFCOUNT_SPECIAL | 4)
1079 #define REFCOUNT_STRUCTELEM_P(V) \
1080 (((V) & REFCOUNT_STRUCTELEM) == REFCOUNT_STRUCTELEM)
1081 /* The first leading key with _FLAG_FIRST set houses the actual reference count
1082 in the structelem_refcount field. Other siblings point to this counter value
1083 through its structelem_refcount_ptr field. */
1084 #define REFCOUNT_STRUCTELEM_FLAG_FIRST (1)
1085 /* The last key in the sibling sequence has this set. This is required to
1086 indicate the sequence boundary, when we remove the structure sibling list
1088 #define REFCOUNT_STRUCTELEM_FLAG_LAST (2)
1090 #define REFCOUNT_STRUCTELEM_FIRST_P(V) \
1091 (REFCOUNT_STRUCTELEM_P (V) && ((V) & REFCOUNT_STRUCTELEM_FLAG_FIRST))
1092 #define REFCOUNT_STRUCTELEM_LAST_P(V) \
1093 (REFCOUNT_STRUCTELEM_P (V) && ((V) & REFCOUNT_STRUCTELEM_FLAG_LAST))
1095 /* Special offset values. */
1096 #define OFFSET_INLINED (~(uintptr_t) 0)
1097 #define OFFSET_POINTER (~(uintptr_t) 1)
1098 #define OFFSET_STRUCT (~(uintptr_t) 2)
1100 /* Auxiliary structure for infrequently-used or API-specific data. */
1102 struct splay_tree_aux
{
1103 /* Pointer to the original mapping of "omp declare target link" object. */
1104 splay_tree_key link_key
;
1105 /* For a block with attached pointers, the attachment counters for each.
1106 Only used for OpenACC. */
1107 uintptr_t *attach_count
;
1110 struct splay_tree_key_s
{
1111 /* Address of the host object. */
1112 uintptr_t host_start
;
1113 /* Address immediately after the host object. */
1115 /* Descriptor of the target memory. */
1116 struct target_mem_desc
*tgt
;
1117 /* Offset from tgt->tgt_start to the start of the target object. */
1118 uintptr_t tgt_offset
;
1119 /* Reference count. */
1122 /* Dynamic reference count. */
1123 uintptr_t dynamic_refcount
;
1125 /* Unified reference count for structure element siblings, this is used
1126 when REFCOUNT_STRUCTELEM_FIRST_P(k->refcount) == true, the first sibling
1127 in a structure element sibling list item sequence. */
1128 uintptr_t structelem_refcount
;
1130 /* When REFCOUNT_STRUCTELEM_P (k->refcount) == true, this field points
1131 into the (above) structelem_refcount field of the _FIRST splay_tree_key,
1132 the first key in the created sequence. All structure element siblings
1133 share a single refcount in this manner. Since these two fields won't be
1134 used at the same time, they are stashed in a union. */
1135 uintptr_t *structelem_refcount_ptr
;
1137 struct splay_tree_aux
*aux
;
1140 /* The comparison function. */
1143 splay_compare (splay_tree_key x
, splay_tree_key y
)
1145 if (x
->host_start
== x
->host_end
1146 && y
->host_start
== y
->host_end
)
1148 if (x
->host_end
<= y
->host_start
)
1150 if (x
->host_start
>= y
->host_end
)
1155 #include "splay-tree.h"
1157 typedef struct acc_dispatch_t
1160 __typeof (GOMP_OFFLOAD_openacc_exec
) *exec_func
;
1162 /* Create/destroy TLS data. */
1163 __typeof (GOMP_OFFLOAD_openacc_create_thread_data
) *create_thread_data_func
;
1164 __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data
)
1165 *destroy_thread_data_func
;
1168 /* Once created and put into the "active" list, asyncqueues are then never
1169 destructed and removed from the "active" list, other than if the TODO
1170 device is shut down. */
1173 struct goacc_asyncqueue
**asyncqueue
;
1174 struct goacc_asyncqueue_list
*active
;
1176 __typeof (GOMP_OFFLOAD_openacc_async_construct
) *construct_func
;
1177 __typeof (GOMP_OFFLOAD_openacc_async_destruct
) *destruct_func
;
1178 __typeof (GOMP_OFFLOAD_openacc_async_test
) *test_func
;
1179 __typeof (GOMP_OFFLOAD_openacc_async_synchronize
) *synchronize_func
;
1180 __typeof (GOMP_OFFLOAD_openacc_async_serialize
) *serialize_func
;
1181 __typeof (GOMP_OFFLOAD_openacc_async_queue_callback
) *queue_callback_func
;
1183 __typeof (GOMP_OFFLOAD_openacc_async_exec
) *exec_func
;
1184 __typeof (GOMP_OFFLOAD_openacc_async_dev2host
) *dev2host_func
;
1185 __typeof (GOMP_OFFLOAD_openacc_async_host2dev
) *host2dev_func
;
1188 __typeof (GOMP_OFFLOAD_openacc_get_property
) *get_property_func
;
1190 /* NVIDIA target specific routines. */
1192 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device
)
1193 *get_current_device_func
;
1194 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context
)
1195 *get_current_context_func
;
1196 __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream
) *get_stream_func
;
1197 __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream
) *set_stream_func
;
1201 /* Various state of the accelerator device. */
1202 enum gomp_device_state
1204 GOMP_DEVICE_UNINITIALIZED
,
1205 GOMP_DEVICE_INITIALIZED
,
1206 GOMP_DEVICE_FINALIZED
1209 /* This structure describes accelerator device.
1210 It contains name of the corresponding libgomp plugin, function handlers for
1211 interaction with the device, ID-number of the device, and information about
1213 struct gomp_device_descr
1215 /* Immutable data, which is only set during initialization, and which is not
1216 guarded by the lock. */
1218 /* The name of the device. */
1221 /* Capabilities of device (supports OpenACC, OpenMP). */
1222 unsigned int capabilities
;
1224 /* This is the ID number of device among devices of the same type. */
1227 /* This is the TYPE of device. */
1228 enum offload_target_type type
;
1230 /* Function handlers. */
1231 __typeof (GOMP_OFFLOAD_get_name
) *get_name_func
;
1232 __typeof (GOMP_OFFLOAD_get_caps
) *get_caps_func
;
1233 __typeof (GOMP_OFFLOAD_get_type
) *get_type_func
;
1234 __typeof (GOMP_OFFLOAD_get_num_devices
) *get_num_devices_func
;
1235 __typeof (GOMP_OFFLOAD_init_device
) *init_device_func
;
1236 __typeof (GOMP_OFFLOAD_fini_device
) *fini_device_func
;
1237 __typeof (GOMP_OFFLOAD_version
) *version_func
;
1238 __typeof (GOMP_OFFLOAD_load_image
) *load_image_func
;
1239 __typeof (GOMP_OFFLOAD_unload_image
) *unload_image_func
;
1240 __typeof (GOMP_OFFLOAD_alloc
) *alloc_func
;
1241 __typeof (GOMP_OFFLOAD_free
) *free_func
;
1242 __typeof (GOMP_OFFLOAD_dev2host
) *dev2host_func
;
1243 __typeof (GOMP_OFFLOAD_host2dev
) *host2dev_func
;
1244 __typeof (GOMP_OFFLOAD_dev2dev
) *dev2dev_func
;
1245 __typeof (GOMP_OFFLOAD_can_run
) *can_run_func
;
1246 __typeof (GOMP_OFFLOAD_run
) *run_func
;
1247 __typeof (GOMP_OFFLOAD_async_run
) *async_run_func
;
1249 /* Splay tree containing information about mapped memory regions. */
1250 struct splay_tree_s mem_map
;
1252 /* Mutex for the mutable data. */
1255 /* Current state of the device. OpenACC allows to move from INITIALIZED state
1256 back to UNINITIALIZED state. OpenMP allows only to move from INITIALIZED
1257 to FINALIZED state (at program shutdown). */
1258 enum gomp_device_state state
;
1260 /* OpenACC-specific data and functions. */
1261 /* This is mutable because of its mutable target_data member. */
1262 acc_dispatch_t openacc
;
1265 /* Kind of the pragma, for which gomp_map_vars () is called. */
1266 enum gomp_map_vars_kind
1268 GOMP_MAP_VARS_OPENACC
= 1,
1269 GOMP_MAP_VARS_TARGET
= 2,
1270 GOMP_MAP_VARS_DATA
= 4,
1271 GOMP_MAP_VARS_ENTER_DATA
= 8
1274 extern void gomp_acc_declare_allocate (bool, size_t, void **, size_t *,
1276 struct gomp_coalesce_buf
;
1277 extern void gomp_copy_host2dev (struct gomp_device_descr
*,
1278 struct goacc_asyncqueue
*, void *, const void *,
1279 size_t, bool, struct gomp_coalesce_buf
*);
1280 extern void gomp_copy_dev2host (struct gomp_device_descr
*,
1281 struct goacc_asyncqueue
*, void *, const void *,
1283 extern uintptr_t gomp_map_val (struct target_mem_desc
*, void **, size_t);
1284 extern void gomp_attach_pointer (struct gomp_device_descr
*,
1285 struct goacc_asyncqueue
*, splay_tree
,
1286 splay_tree_key
, uintptr_t, size_t,
1287 struct gomp_coalesce_buf
*, bool);
1288 extern void gomp_detach_pointer (struct gomp_device_descr
*,
1289 struct goacc_asyncqueue
*, splay_tree_key
,
1290 uintptr_t, bool, struct gomp_coalesce_buf
*);
1291 extern struct target_mem_desc
*goacc_map_vars (struct gomp_device_descr
*,
1292 struct goacc_asyncqueue
*,
1293 size_t, void **, void **,
1294 size_t *, void *, bool,
1295 enum gomp_map_vars_kind
);
1296 extern void goacc_unmap_vars (struct target_mem_desc
*, bool,
1297 struct goacc_asyncqueue
*);
1298 extern void gomp_init_device (struct gomp_device_descr
*);
1299 extern bool gomp_fini_device (struct gomp_device_descr
*);
1300 extern void gomp_unload_device (struct gomp_device_descr
*);
1301 extern bool gomp_remove_var (struct gomp_device_descr
*, splay_tree_key
);
1302 extern void gomp_remove_var_async (struct gomp_device_descr
*, splay_tree_key
,
1303 struct goacc_asyncqueue
*);
1307 extern void gomp_init_work_share (struct gomp_work_share
*, size_t, unsigned);
1308 extern void gomp_fini_work_share (struct gomp_work_share
*);
1309 extern bool gomp_work_share_start (size_t);
1310 extern void gomp_work_share_end (void);
1311 extern bool gomp_work_share_end_cancel (void);
1312 extern void gomp_work_share_end_nowait (void);
1315 gomp_work_share_init_done (void)
1317 struct gomp_thread
*thr
= gomp_thread ();
1318 if (__builtin_expect (thr
->ts
.last_work_share
!= NULL
, 1))
1319 gomp_ptrlock_set (&thr
->ts
.last_work_share
->next_ws
, thr
->ts
.work_share
);
1322 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1323 # pragma GCC visibility pop
1326 /* Now that we're back to default visibility, include the globals. */
1327 #include "libgomp_g.h"
1329 /* Include omp.h by parts. */
1330 #include "omp-lock.h"
1331 #define _LIBGOMP_OMP_LOCK_DEFINED 1
1334 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \
1335 || !defined (HAVE_ATTRIBUTE_ALIAS) \
1336 || !defined (HAVE_AS_SYMVER_DIRECTIVE) \
1338 || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT)
1339 # undef LIBGOMP_GNU_SYMBOL_VERSIONING
1342 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING
1343 extern void gomp_init_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1344 extern void gomp_destroy_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1345 extern void gomp_set_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1346 extern void gomp_unset_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1347 extern int gomp_test_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1348 extern void gomp_init_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1349 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1350 extern void gomp_set_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1351 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1352 extern int gomp_test_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1354 extern void gomp_init_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1355 extern void gomp_destroy_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1356 extern void gomp_set_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1357 extern void gomp_unset_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1358 extern int gomp_test_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1359 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1360 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1361 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1362 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1363 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1365 # define omp_lock_symver(fn) \
1366 __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
1367 __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
1369 # define gomp_init_lock_30 omp_init_lock
1370 # define gomp_destroy_lock_30 omp_destroy_lock
1371 # define gomp_set_lock_30 omp_set_lock
1372 # define gomp_unset_lock_30 omp_unset_lock
1373 # define gomp_test_lock_30 omp_test_lock
1374 # define gomp_init_nest_lock_30 omp_init_nest_lock
1375 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock
1376 # define gomp_set_nest_lock_30 omp_set_nest_lock
1377 # define gomp_unset_nest_lock_30 omp_unset_nest_lock
1378 # define gomp_test_nest_lock_30 omp_test_nest_lock
1381 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1382 # define attribute_hidden __attribute__ ((visibility ("hidden")))
1384 # define attribute_hidden
1388 # define HAVE_ATTRIBUTE_COPY
1391 #ifdef HAVE_ATTRIBUTE_COPY
1392 # define attribute_copy(arg) __attribute__ ((copy (arg)))
1394 # define attribute_copy(arg)
1397 #ifdef HAVE_ATTRIBUTE_ALIAS
1398 # define strong_alias(fn, al) \
1399 extern __typeof (fn) al __attribute__ ((alias (#fn))) attribute_copy (fn);
1401 # define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__)
1402 # define ialias_str1(x) ialias_str2(x)
1403 # define ialias_str2(x) #x
1404 # define ialias(fn) \
1405 extern __typeof (fn) gomp_ialias_##fn \
1406 __attribute__ ((alias (#fn))) attribute_hidden attribute_copy (fn);
1407 # define ialias_redirect(fn) \
1408 extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
1409 # define ialias_call(fn) gomp_ialias_ ## fn
1412 # define ialias_redirect(fn)
1413 # define ialias_call(fn) fn
1416 /* Helper function for priority_node_to_task() and
1417 task_to_priority_node().
1419 Return the offset from a task to its priority_node entry. The
1420 priority_node entry is has a type of TYPE. */
1422 static inline size_t
1423 priority_queue_offset (enum priority_queue_type type
)
1425 return offsetof (struct gomp_task
, pnode
[(int) type
]);
1428 /* Return the task associated with a priority NODE of type TYPE. */
1430 static inline struct gomp_task
*
1431 priority_node_to_task (enum priority_queue_type type
,
1432 struct priority_node
*node
)
1434 return (struct gomp_task
*) ((char *) node
- priority_queue_offset (type
));
1437 /* Return the priority node of type TYPE for a given TASK. */
1439 static inline struct priority_node
*
1440 task_to_priority_node (enum priority_queue_type type
,
1441 struct gomp_task
*task
)
1443 return (struct priority_node
*) ((char *) task
1444 + priority_queue_offset (type
));
1447 #ifdef LIBGOMP_USE_PTHREADS
1448 static inline gomp_thread_handle
1449 gomp_thread_self (void)
1451 return pthread_self ();
1454 static inline gomp_thread_handle
1455 gomp_thread_to_pthread_t (struct gomp_thread
*thr
)
1457 struct gomp_thread
*this_thr
= gomp_thread ();
1458 if (thr
== this_thr
)
1459 return pthread_self ();
1460 #ifdef GOMP_NEEDS_THREAD_HANDLE
1463 /* On Linux with initial-exec TLS, the pthread_t of the thread containing
1464 thr can be computed from thr, this_thr and pthread_self (),
1465 as the distance between this_thr and pthread_self () is constant. */
1466 return pthread_self () + ((uintptr_t) thr
- (uintptr_t) this_thr
);
1470 static inline gomp_thread_handle
1471 gomp_thread_self (void)
1473 return (gomp_thread_handle
) {};
1476 static inline gomp_thread_handle
1477 gomp_thread_to_pthread_t (struct gomp_thread
*thr
)
1480 return gomp_thread_self ();
1484 #endif /* LIBGOMP_H */