1 /* Copyright (C) 2005-2023 Free Software Foundation, Inc.
2 Contributed by Richard Henderson <rth@redhat.com>.
4 This file is part of the GNU Offloading and Multi Processing Library
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
26 /* This file contains data types and function declarations that are not
27 part of the official OpenACC or OpenMP user interfaces. There are
28 declarations in here that are part of the GNU Offloading and Multi
29 Processing ABI, in that the compiler is required to know about them
32 The convention is that the all caps prefix "GOMP" is used group items
33 that are part of the external ABI, and the lower case prefix "gomp"
34 is used group items that are completely private to the library. */
39 #ifndef _LIBGOMP_CHECKING_
40 /* Define to 1 to perform internal sanity checks. */
41 #define _LIBGOMP_CHECKING_ 0
46 #include "libgomp-plugin.h"
47 #include "gomp-constants.h"
56 /* Needed for memset in priority_queue.c. */
57 #if _LIBGOMP_CHECKING_
58 # ifdef STRING_WITH_STRINGS
65 # ifdef HAVE_STRINGS_H
72 #ifdef HAVE_ATTRIBUTE_VISIBILITY
73 # pragma GCC visibility push(hidden)
76 /* If we were a C++ library, we'd get this from <std/atomic>. */
89 #if defined(HAVE_ALIGNED_ALLOC) \
90 || defined(HAVE_POSIX_MEMALIGN) \
91 || defined(HAVE_MEMALIGN)
92 /* Defined if gomp_aligned_alloc doesn't use fallback version
93 and free can be used instead of gomp_aligned_free. */
94 #define GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC 1
97 #if defined(GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC) && !defined(__AMDGCN__)
98 #define GOMP_USE_ALIGNED_WORK_SHARES 1
101 extern void *gomp_malloc (size_t) __attribute__((malloc
));
102 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc
));
103 extern void *gomp_realloc (void *, size_t);
104 extern void *gomp_aligned_alloc (size_t, size_t)
105 __attribute__((malloc
, alloc_size (2)));
106 extern void gomp_aligned_free (void *);
108 /* Avoid conflicting prototypes of alloca() in system headers by using
109 GCC's builtin alloca(). */
110 #define gomp_alloca(x) __builtin_alloca(x)
112 /* Optimized allocators for team-specific data that will die with the team. */
115 #include "libgomp-gcn.h"
116 /* The arena is initialized in config/gcn/team.c. */
117 #define TEAM_ARENA_START 16 /* LDS offset of free pointer. */
118 #define TEAM_ARENA_FREE 24 /* LDS offset of free pointer. */
119 #define TEAM_ARENA_END 32 /* LDS offset of end pointer. */
121 static inline void * __attribute__((malloc
))
122 team_malloc (size_t size
)
124 /* 4-byte align the size. */
125 size
= (size
+ 3) & ~3;
127 /* Allocate directly from the arena.
128 The compiler does not support DS atomics, yet. */
130 asm ("ds_add_rtn_u64 %0, %1, %2\n\ts_waitcnt 0"
131 : "=v"(result
) : "v"(TEAM_ARENA_FREE
), "v"(size
), "e"(1L) : "memory");
134 if (result
+ size
> *(void * __lds
*)TEAM_ARENA_END
)
136 /* While this is experimental, let's make sure we know when OOM
138 const char msg
[] = "GCN team arena exhausted;"
139 " configure with GCN_TEAM_ARENA_SIZE=bytes\n";
140 write (2, msg
, sizeof(msg
)-1);
142 /* Fall back to using the heap (slowly). */
143 result
= gomp_malloc (size
);
148 static inline void * __attribute__((malloc
))
149 team_malloc_cleared (size_t size
)
151 char *result
= team_malloc (size
);
153 /* Clear the allocated memory. */
154 __builtin_memset (result
, 0, size
);
160 team_free (void *ptr
)
162 /* The whole arena is freed when the kernel exits.
163 However, if we fell back to using heap then we should free it.
164 It would be better if this function could be a no-op, but at least
165 LDS loads are cheap. */
166 if (ptr
< *(void * __lds
*)TEAM_ARENA_START
167 || ptr
>= *(void * __lds
*)TEAM_ARENA_END
)
171 #define team_malloc(...) gomp_malloc (__VA_ARGS__)
172 #define team_malloc_cleared(...) gomp_malloc_cleared (__VA_ARGS__)
173 #define team_free(...) free (__VA_ARGS__)
178 extern void gomp_vdebug (int, const char *, va_list);
179 extern void gomp_debug (int, const char *, ...)
180 __attribute__ ((format (printf
, 2, 3)));
181 #define gomp_vdebug(KIND, FMT, VALIST) \
183 if (__builtin_expect (gomp_debug_var, 0)) \
184 (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
186 #define gomp_debug(KIND, ...) \
188 if (__builtin_expect (gomp_debug_var, 0)) \
189 (gomp_debug) ((KIND), __VA_ARGS__); \
191 extern void gomp_verror (const char *, va_list);
192 extern void gomp_error (const char *, ...)
193 __attribute__ ((format (printf
, 1, 2)));
194 extern void gomp_vfatal (const char *, va_list)
195 __attribute__ ((noreturn
));
196 extern void gomp_fatal (const char *, ...)
197 __attribute__ ((noreturn
, format (printf
, 1, 2)));
200 struct gomp_taskgroup
;
203 #include "priority_queue.h"
207 #include "simple-bar.h"
211 /* This structure contains the data to control one work-sharing construct,
212 either a LOOP (FOR/DO) or a SECTIONS. */
214 enum gomp_schedule_type
221 GFS_MONOTONIC
= 0x80000000U
224 struct gomp_doacross_work_share
227 /* chunk_size copy, as ws->chunk_size is multiplied by incr for
230 /* Likewise, but for ull implementation. */
231 unsigned long long chunk_size_ull
;
232 /* For schedule(static,0) this is the number
233 of iterations assigned to the last thread, i.e. number of
234 iterations / number of threads. */
236 /* Likewise, but for ull implementation. */
237 unsigned long long q_ull
;
239 /* Size of each array entry (padded to cache line size). */
240 unsigned long elt_sz
;
241 /* Number of dimensions in sink vectors. */
242 unsigned int ncounts
;
243 /* True if the iterations can be flattened. */
245 /* Actual array (of elt_sz sized units), aligned to cache line size.
246 This is indexed by team_id for GFS_STATIC and outermost iteration
247 / chunk_size for other schedules. */
248 unsigned char *array
;
249 /* These two are only used for schedule(static,0). */
250 /* This one is number of iterations % number of threads. */
253 /* And this one is cached t * (q + 1). */
255 /* Likewise, but for the ull implementation. */
256 unsigned long long boundary_ull
;
258 /* Pointer to extra memory if needed for lastprivate(conditional). */
260 /* Array of shift counts for each dimension if they can be flattened. */
261 unsigned int shift_counts
[];
264 /* Like struct gomp_work_share, but only the 1st cacheline of it plus
265 flexible array at the end.
266 Keep in sync with struct gomp_work_share. */
267 struct gomp_work_share_1st_cacheline
269 enum gomp_schedule_type sched
;
273 long chunk_size
, end
, incr
;
276 unsigned long long chunk_size_ull
, end_ull
, incr_ull
;
280 unsigned *ordered_team_ids
;
281 struct gomp_doacross_work_share
*doacross
;
283 unsigned ordered_num_used
, ordered_owner
, ordered_cur
;
284 struct gomp_work_share
*next_alloc
;
288 struct gomp_work_share
290 /* This member records the SCHEDULE clause to be used for this construct.
291 The user specification of "runtime" will already have been resolved.
292 If this is a SECTIONS construct, this value will always be DYNAMIC. */
293 enum gomp_schedule_type sched
;
299 /* This is the chunk_size argument to the SCHEDULE clause. */
302 /* This is the iteration end point. If this is a SECTIONS construct,
303 this is the number of contained sections. */
306 /* This is the iteration step. If this is a SECTIONS construct, this
312 /* The same as above, but for the unsigned long long loop variants. */
313 unsigned long long chunk_size_ull
;
314 unsigned long long end_ull
;
315 unsigned long long incr_ull
;
320 /* This is a circular queue that details which threads will be allowed
321 into the ordered region and in which order. When a thread allocates
322 iterations on which it is going to work, it also registers itself at
323 the end of the array. When a thread reaches the ordered region, it
324 checks to see if it is the one at the head of the queue. If not, it
325 blocks on its RELEASE semaphore. */
326 unsigned *ordered_team_ids
;
328 /* This is a pointer to DOACROSS work share data. */
329 struct gomp_doacross_work_share
*doacross
;
332 /* This is the number of threads that have registered themselves in
333 the circular queue ordered_team_ids. */
334 unsigned ordered_num_used
;
336 /* This is the team_id of the currently acknowledged owner of the ordered
337 section, or -1u if the ordered section has not been acknowledged by
338 any thread. This is distinguished from the thread that is *allowed*
339 to take the section next. */
340 unsigned ordered_owner
;
342 /* This is the index into the circular queue ordered_team_ids of the
343 current thread that's allowed into the ordered reason. */
344 unsigned ordered_cur
;
346 /* This is a chain of allocated gomp_work_share blocks, valid only
347 in the first gomp_work_share struct in the block. */
348 struct gomp_work_share
*next_alloc
;
350 /* The above fields are written once during workshare initialization,
351 or related to ordered worksharing. Make sure the following fields
352 are in a different cache line. */
354 /* This lock protects the update of the following members. */
355 #ifdef GOMP_USE_ALIGNED_WORK_SHARES
356 gomp_mutex_t lock
__attribute__((aligned (64)));
358 char pad
[64 - offsetof (struct gomp_work_share_1st_cacheline
, pad
)];
362 /* This is the count of the number of threads that have exited the work
363 share construct. If the construct was marked nowait, they have moved on
364 to other work; otherwise they're blocked on a barrier. The last member
365 of the team to exit the work share construct must deallocate it. */
366 unsigned threads_completed
;
369 /* This is the next iteration value to be allocated. In the case of
370 GFS_STATIC loops, this the iteration start point and never changes. */
373 /* The same, but with unsigned long long type. */
374 unsigned long long next_ull
;
376 /* This is the returned data structure for SINGLE COPYPRIVATE. */
381 /* Link to gomp_work_share struct for next work sharing construct
382 encountered after this one. */
383 gomp_ptrlock_t next_ws
;
385 /* gomp_work_share structs are chained in the free work share cache
387 struct gomp_work_share
*next_free
;
390 /* Task reductions for this work-sharing construct. */
391 uintptr_t *task_reductions
;
393 /* If only few threads are in the team, ordered_team_ids can point
394 to this array which fills the padding at the end of this struct. */
395 unsigned inline_ordered_team_ids
[0];
398 extern char gomp_workshare_struct_check1
399 [offsetof (struct gomp_work_share_1st_cacheline
, next_alloc
)
400 == offsetof (struct gomp_work_share
, next_alloc
) ? 1 : -1];
401 extern char gomp_workshare_struct_check2
402 [offsetof (struct gomp_work_share
, lock
) == 64 ? 1 : -1];
404 /* This structure contains all of the thread-local data associated with
405 a thread team. This is the data that must be saved when a thread
406 encounters a nested PARALLEL construct. */
408 struct gomp_team_state
410 /* This is the team of which the thread is currently a member. */
411 struct gomp_team
*team
;
413 /* This is the work share construct which this thread is currently
414 processing. Recall that with NOWAIT, not all threads may be
415 processing the same construct. */
416 struct gomp_work_share
*work_share
;
418 /* This is the previous work share construct or NULL if there wasn't any.
419 When all threads are done with the current work sharing construct,
420 the previous one can be freed. The current one can't, as its
421 next_ws field is used. */
422 struct gomp_work_share
*last_work_share
;
424 /* This is the ID of this thread within the team. This value is
425 guaranteed to be between 0 and N-1, where N is the number of
426 threads in the team. */
432 /* Active nesting level. Only active parallel regions are counted. */
433 unsigned active_level
;
435 /* Place-partition-var, offset and length into gomp_places_list array. */
436 unsigned place_partition_off
;
437 unsigned place_partition_len
;
439 /* Def-allocator-var ICV. */
440 uintptr_t def_allocator
;
442 #ifdef HAVE_SYNC_BUILTINS
443 /* Number of single stmts encountered. */
444 unsigned long single_count
;
447 /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the
448 trip number through the loop. So first time a particular loop
449 is encountered this number is 0, the second time through the loop
450 is 1, etc. This is unused when the compiler knows in advance that
451 the loop is statically scheduled. */
452 unsigned long static_trip
;
455 struct target_mem_desc
;
460 GOMP_ICV_SCHEDULE
= 2,
461 GOMP_ICV_SCHEDULE_CHUNK_SIZE
= 3,
462 GOMP_ICV_DYNAMIC
= 4,
463 GOMP_ICV_TEAMS_THREAD_LIMIT
= 5,
464 GOMP_ICV_THREAD_LIMIT
= 6,
465 GOMP_ICV_NTHREADS
= 7,
466 GOMP_ICV_NTHREADS_LIST
= 8,
467 GOMP_ICV_NTHREADS_LIST_LEN
= 9,
469 GOMP_ICV_BIND_LIST
= 11,
470 GOMP_ICV_BIND_LIST_LEN
= 12,
471 GOMP_ICV_MAX_ACTIVE_LEVELS
= 13,
472 GOMP_ICV_WAIT_POLICY
= 14,
473 GOMP_ICV_STACKSIZE
= 15,
474 GOMP_ICV_DEFAULT_DEVICE
= 16,
475 GOMP_ICV_CANCELLATION
= 17,
476 GOMP_ICV_DISPLAY_AFFINITY
= 18,
477 GOMP_ICV_TARGET_OFFLOAD
= 19,
478 GOMP_ICV_MAX_TASK_PRIORITY
= 20,
479 GOMP_ICV_ALLOCATOR
= 21
484 GOMP_DEVICE_NUM_FOR_DEV
= -1,
485 GOMP_DEVICE_NUM_FOR_ALL
= -2,
486 GOMP_DEVICE_NUM_FOR_NO_SUFFIX
= -3
489 /* These are the OpenMP 4.0 Internal Control Variables described in
490 section 2.3.1. Those described as having one copy per task are
491 stored within the structure; those described as having one copy
492 for the whole program are (naturally) global variables. */
496 unsigned long nthreads_var
;
497 enum gomp_schedule_type run_sched_var
;
498 int run_sched_chunk_size
;
499 int default_device_var
;
500 unsigned int thread_limit_var
;
502 unsigned char max_active_levels_var
;
505 struct target_mem_desc
*target_data
;
510 GOMP_ENV_SUFFIX_UNKNOWN
= 0,
511 GOMP_ENV_SUFFIX_NONE
= 1,
512 GOMP_ENV_SUFFIX_DEV
= 2,
513 GOMP_ENV_SUFFIX_ALL
= 4,
514 GOMP_ENV_SUFFIX_DEV_X
= 8
517 /* Struct that contains all ICVs for which we need to store initial values.
518 Keeping the initial values is needed for omp_display_env. Moreover initial
519 _DEV and _ALL variants of environment variables are also used to determine
520 actually used values for devices and for the host. */
521 struct gomp_initial_icvs
523 unsigned long *nthreads_var_list
;
525 unsigned long nthreads_var
;
526 unsigned long nthreads_var_list_len
;
527 unsigned long bind_var_list_len
;
528 unsigned long stacksize
;
529 int run_sched_chunk_size
;
530 int default_device_var
;
532 int teams_thread_limit_var
;
534 unsigned int thread_limit_var
;
535 enum gomp_schedule_type run_sched_var
;
537 unsigned char max_active_levels_var
;
541 struct gomp_default_icv
543 unsigned long nthreads_var
;
544 enum gomp_schedule_type run_sched_var
;
545 int run_sched_chunk_size
;
546 int default_device_var
;
547 unsigned int thread_limit_var
;
549 int teams_thread_limit_var
;
551 unsigned char max_active_levels_var
;
555 /* DEVICE_NUM "-1" is reserved for "_DEV" icvs.
556 DEVICE_NUM "-2" is reserved for "_ALL" icvs.
557 DEVICE_NUM "-3" is reserved for ICVs without suffix.
558 Non-negative DEVICE_NUM is for "_DEV_X" icvs. */
563 struct gomp_initial_icvs icvs
;
564 struct gomp_icv_list
*next
;
567 struct gomp_offload_icvs
572 int teams_thread_limit
;
575 struct gomp_offload_icv_list
578 struct gomp_offload_icvs icvs
;
579 struct gomp_offload_icv_list
*next
;
582 enum gomp_target_offload_t
584 GOMP_TARGET_OFFLOAD_DEFAULT
,
585 GOMP_TARGET_OFFLOAD_MANDATORY
,
586 GOMP_TARGET_OFFLOAD_DISABLED
589 #define gomp_supported_active_levels UCHAR_MAX
591 extern struct gomp_task_icv gomp_global_icv
;
592 #ifndef HAVE_SYNC_BUILTINS
593 extern gomp_mutex_t gomp_managed_threads_lock
;
595 extern bool gomp_cancel_var
;
596 extern enum gomp_target_offload_t gomp_target_offload_var
;
597 extern int gomp_max_task_priority_var
;
598 extern unsigned long long gomp_spin_count_var
, gomp_throttled_spin_count_var
;
599 extern unsigned long gomp_available_cpus
, gomp_managed_threads
;
600 extern unsigned long *gomp_nthreads_var_list
, gomp_nthreads_var_list_len
;
601 extern char *gomp_bind_var_list
;
602 extern unsigned long gomp_bind_var_list_len
;
603 extern void **gomp_places_list
;
604 extern unsigned long gomp_places_list_len
;
605 extern unsigned int gomp_num_teams_var
;
606 extern int gomp_nteams_var
;
607 extern int gomp_teams_thread_limit_var
;
608 extern int gomp_debug_var
;
609 extern bool gomp_display_affinity_var
;
610 extern char *gomp_affinity_format_var
;
611 extern size_t gomp_affinity_format_len
;
612 extern uintptr_t gomp_def_allocator
;
613 extern const struct gomp_default_icv gomp_default_icv_values
;
614 extern struct gomp_icv_list
*gomp_initial_icv_list
;
615 extern struct gomp_offload_icv_list
*gomp_offload_icv_list
;
616 extern int goacc_device_num
;
617 extern char *goacc_device_type
;
618 extern int goacc_default_dims
[GOMP_DIM_MAX
];
624 /* Undeferred task. */
625 GOMP_TASK_UNDEFERRED
,
626 /* Task created by GOMP_task and waiting to be run. */
628 /* Task currently executing or scheduled and about to execute. */
630 /* Used for target tasks that have vars mapped and async run started,
631 but not yet completed. Once that completes, they will be readded
632 into the queues as GOMP_TASK_WAITING in order to perform the var
634 GOMP_TASK_ASYNC_RUNNING
,
635 /* Task that has finished executing but is waiting for its
636 completion event to be fulfilled. */
640 struct gomp_task_depend_entry
642 /* Address of dependency. */
644 struct gomp_task_depend_entry
*next
;
645 struct gomp_task_depend_entry
*prev
;
646 /* Task that provides the dependency in ADDR. */
647 struct gomp_task
*task
;
648 /* Depend entry is of type "IN" (1) or "INOUTSET" (2). */
654 struct gomp_dependers_vec
658 struct gomp_task
*elem
[];
661 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies. */
667 /* Number of tasks we are waiting for. */
669 gomp_sem_t taskwait_sem
;
672 /* This structure describes a "task" to be run by a thread. */
676 /* Parent of this task. */
677 struct gomp_task
*parent
;
678 /* Children of this task. */
679 struct priority_queue children_queue
;
680 /* Taskgroup this task belongs in. */
681 struct gomp_taskgroup
*taskgroup
;
682 /* Tasks that depend on this task. */
683 struct gomp_dependers_vec
*dependers
;
684 struct htab
*depend_hash
;
685 struct gomp_taskwait
*taskwait
;
686 /* Last depend({,in}out:omp_all_memory) child if any. */
687 struct gomp_task
*depend_all_memory
;
688 /* Number of items in DEPEND. */
690 /* Number of tasks this task depends on. Once this counter reaches
691 0, we have no unsatisfied dependencies, and this task can be put
692 into the various queues to be scheduled. */
693 size_t num_dependees
;
696 /* Valid only if deferred_p is false. */
697 gomp_sem_t
*completion_sem
;
698 /* Valid only if deferred_p is true. Set to the team that executes the
699 task if the task is detached and the completion event has yet to be
701 struct gomp_team
*detach_team
;
705 /* Priority of this task. */
707 /* The priority node for this task in each of the different queues.
708 We put this here to avoid allocating space for each priority
709 node. Then we play offsetof() games to convert between pnode[]
710 entries and the gomp_task in which they reside. */
711 struct priority_node pnode
[3];
713 struct gomp_task_icv icv
;
716 enum gomp_task_kind kind
;
719 bool copy_ctors_done
;
720 /* Set for undeferred tasks with unsatisfied dependencies which
721 block further execution of their parent until the dependencies
723 bool parent_depends_on
;
724 /* Dependencies provided and/or needed for this task. DEPEND_COUNT
725 is the number of items available. */
726 struct gomp_task_depend_entry depend
[];
729 /* This structure describes a single #pragma omp taskgroup. */
731 struct gomp_taskgroup
733 struct gomp_taskgroup
*prev
;
734 /* Queue of tasks that belong in this taskgroup. */
735 struct priority_queue taskgroup_queue
;
736 uintptr_t *reductions
;
737 bool in_taskgroup_wait
;
740 gomp_sem_t taskgroup_sem
;
744 /* Various state of OpenMP async offloading tasks. */
745 enum gomp_target_task_state
747 GOMP_TARGET_TASK_DATA
,
748 GOMP_TARGET_TASK_BEFORE_MAP
,
749 GOMP_TARGET_TASK_FALLBACK
,
750 GOMP_TARGET_TASK_READY_TO_RUN
,
751 GOMP_TARGET_TASK_RUNNING
,
752 GOMP_TARGET_TASK_FINISHED
755 /* This structure describes a target task. */
757 struct gomp_target_task
759 struct gomp_device_descr
*devicep
;
763 unsigned short *kinds
;
765 enum gomp_target_task_state state
;
766 struct target_mem_desc
*tgt
;
767 struct gomp_task
*task
;
768 struct gomp_team
*team
;
769 /* Device-specific target arguments. */
774 /* This structure describes a "team" of threads. These are the threads
775 that are spawned by a PARALLEL constructs, as well as the work sharing
776 constructs that the team encounters. */
780 /* This is the number of threads in the current team. */
783 /* This is number of gomp_work_share structs that have been allocated
784 as a block last time. */
785 unsigned work_share_chunk
;
787 /* This is the saved team state that applied to a master thread before
788 the current thread was created. */
789 struct gomp_team_state prev_ts
;
791 /* This semaphore should be used by the master thread instead of its
792 "native" semaphore in the thread structure. Required for nested
793 parallels, as the master is a member of two teams. */
794 gomp_sem_t master_release
;
796 /* This points to an array with pointers to the release semaphore
797 of the threads in the team. */
798 gomp_sem_t
**ordered_release
;
800 /* List of work shares on which gomp_fini_work_share hasn't been
801 called yet. If the team hasn't been cancelled, this should be
802 equal to each thr->ts.work_share, but otherwise it can be a possibly
803 long list of workshares. */
804 struct gomp_work_share
*work_shares_to_free
;
806 /* List of gomp_work_share structs chained through next_free fields.
807 This is populated and taken off only by the first thread in the
808 team encountering a new work sharing construct, in a critical
810 struct gomp_work_share
*work_share_list_alloc
;
812 /* List of gomp_work_share structs freed by free_work_share. New
813 entries are atomically added to the start of the list, and
814 alloc_work_share can safely only move all but the first entry
815 to work_share_list alloc, as free_work_share can happen concurrently
816 with alloc_work_share. */
817 struct gomp_work_share
*work_share_list_free
;
819 #ifdef HAVE_SYNC_BUILTINS
820 /* Number of simple single regions encountered by threads in this
822 unsigned long single_count
;
824 /* Mutex protecting addition of workshares to work_share_list_free. */
825 gomp_mutex_t work_share_list_free_lock
;
828 /* This barrier is used for most synchronization of the team. */
829 gomp_barrier_t barrier
;
831 /* Initial work shares, to avoid allocating any gomp_work_share
832 structs in the common case. */
833 struct gomp_work_share work_shares
[8];
835 gomp_mutex_t task_lock
;
836 /* Scheduled tasks. */
837 struct priority_queue task_queue
;
838 /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */
839 unsigned int task_count
;
840 /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */
841 unsigned int task_queued_count
;
842 /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
843 directly in gomp_barrier_handle_tasks; tasks spawned
844 from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
845 that is called from a task run from gomp_barrier_handle_tasks.
846 task_running_count should be always <= team->nthreads,
847 and if current task isn't in_tied_task, then it will be
848 even < team->nthreads. */
849 unsigned int task_running_count
;
850 int work_share_cancelled
;
853 /* Number of tasks waiting for their completion event to be fulfilled. */
854 unsigned int task_detach_count
;
856 /* This array contains structures for implicit tasks. */
857 struct gomp_task implicit_task
[];
860 /* This structure contains all data that is private to libgomp and is
861 allocated per thread. */
865 /* This is the function that the thread should run upon launch. */
866 void (*fn
) (void *data
);
869 /* This is the current team state for this thread. The ts.team member
870 is NULL only if the thread is idle. */
871 struct gomp_team_state ts
;
873 /* This is the task that the thread is currently executing. */
874 struct gomp_task
*task
;
876 /* This semaphore is used for ordered loops. */
879 /* Place this thread is bound to plus one, or zero if not bound
883 /* User pthread thread pool */
884 struct gomp_thread_pool
*thread_pool
;
886 #ifdef LIBGOMP_USE_PTHREADS
887 /* omp_get_num_teams () - 1. */
888 unsigned int num_teams
;
890 /* omp_get_team_num (). */
891 unsigned int team_num
;
894 #if defined(LIBGOMP_USE_PTHREADS) \
895 && (!defined(HAVE_TLS) \
896 || !defined(__GLIBC__) \
897 || !defined(USING_INITIAL_EXEC_TLS))
898 /* pthread_t of the thread containing this gomp_thread.
899 On Linux when using initial-exec TLS,
900 (typeof (pthread_t)) gomp_thread () - pthread_self ()
901 is constant in all threads, so we can optimize and not
903 #define GOMP_NEEDS_THREAD_HANDLE 1
909 struct gomp_thread_pool
911 /* This array manages threads spawned from the top level, which will
912 return to the idle loop once the current PARALLEL construct ends. */
913 struct gomp_thread
**threads
;
914 unsigned threads_size
;
915 unsigned threads_used
;
916 /* The last team is used for non-nested teams to delay their destruction to
917 make sure all the threads in the team move on to the pool's barrier before
918 the team's barrier is destroyed. */
919 struct gomp_team
*last_team
;
920 /* Number of threads running in this contention group. */
921 unsigned long threads_busy
;
923 /* This barrier holds and releases threads waiting in thread pools. */
924 gomp_simple_barrier_t threads_dock
;
927 enum gomp_cancel_kind
929 GOMP_CANCEL_PARALLEL
= 1,
930 GOMP_CANCEL_LOOP
= 2,
931 GOMP_CANCEL_FOR
= GOMP_CANCEL_LOOP
,
932 GOMP_CANCEL_DO
= GOMP_CANCEL_LOOP
,
933 GOMP_CANCEL_SECTIONS
= 4,
934 GOMP_CANCEL_TASKGROUP
= 8
937 /* ... and here is that TLS data. */
939 #if defined __nvptx__
940 extern struct gomp_thread
*nvptx_thrs
__attribute__((shared
));
941 static inline struct gomp_thread
*gomp_thread (void)
944 asm ("mov.u32 %0, %%tid.y;" : "=r" (tid
));
945 return nvptx_thrs
+ tid
;
947 #elif defined __AMDGCN__
948 static inline struct gomp_thread
*gcn_thrs (void)
950 /* The value is at the bottom of LDS. */
951 struct gomp_thread
* __lds
*thrs
= (struct gomp_thread
* __lds
*)4;
954 static inline void set_gcn_thrs (struct gomp_thread
*val
)
956 /* The value is at the bottom of LDS. */
957 struct gomp_thread
* __lds
*thrs
= (struct gomp_thread
* __lds
*)4;
960 static inline struct gomp_thread
*gomp_thread (void)
962 int tid
= __builtin_gcn_dim_pos(1);
963 return gcn_thrs () + tid
;
965 #elif defined HAVE_TLS || defined USE_EMUTLS
966 extern __thread
struct gomp_thread gomp_tls_data
;
967 static inline struct gomp_thread
*gomp_thread (void)
969 return &gomp_tls_data
;
972 extern pthread_key_t gomp_tls_key
;
973 static inline struct gomp_thread
*gomp_thread (void)
975 return pthread_getspecific (gomp_tls_key
);
979 extern struct gomp_task_icv
*gomp_new_icv (void);
981 /* Here's how to access the current copy of the ICVs. */
983 static inline struct gomp_task_icv
*gomp_icv (bool write
)
985 struct gomp_task
*task
= gomp_thread ()->task
;
989 return gomp_new_icv ();
991 return &gomp_global_icv
;
994 #ifdef LIBGOMP_USE_PTHREADS
995 /* The attributes to be used during thread creation. */
996 extern pthread_attr_t gomp_thread_attr
;
998 extern pthread_key_t gomp_thread_destructor
;
1001 /* Function prototypes. */
1005 extern void gomp_init_affinity (void);
1006 #ifdef LIBGOMP_USE_PTHREADS
1007 extern void gomp_init_thread_affinity (pthread_attr_t
*, unsigned int);
1009 extern void **gomp_affinity_alloc (unsigned long, bool);
1010 extern void gomp_affinity_init_place (void *);
1011 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
1013 extern bool gomp_affinity_remove_cpu (void *, unsigned long);
1014 extern bool gomp_affinity_copy_place (void *, void *, long);
1015 extern bool gomp_affinity_same_place (void *, void *);
1016 extern bool gomp_affinity_finalize_place_list (bool);
1017 extern bool gomp_affinity_init_level (int, unsigned long, bool);
1018 extern void gomp_affinity_print_place (void *);
1019 extern void gomp_get_place_proc_ids_8 (int, int64_t *);
1020 extern void gomp_display_affinity_place (char *, size_t, size_t *, int);
1022 /* affinity-fmt.c */
1024 extern bool gomp_print_string (const char *str
, size_t len
);
1025 extern void gomp_set_affinity_format (const char *, size_t);
1026 extern void gomp_display_string (char *, size_t, size_t *, const char *,
1028 #ifdef LIBGOMP_USE_PTHREADS
1029 typedef pthread_t gomp_thread_handle
;
1031 typedef struct {} gomp_thread_handle
;
1033 extern size_t gomp_display_affinity (char *, size_t, const char *,
1035 struct gomp_team_state
*, unsigned int);
1036 extern void gomp_display_affinity_thread (gomp_thread_handle
,
1037 struct gomp_team_state
*,
1038 unsigned int) __attribute__((cold
));
1042 extern struct gomp_icv_list
*gomp_get_initial_icv_item (int dev_num
);
1043 extern bool gomp_get_icv_flag (uint32_t value
, enum gomp_icvs icv
);
1047 extern int gomp_iter_static_next (long *, long *);
1048 extern bool gomp_iter_dynamic_next_locked (long *, long *);
1049 extern bool gomp_iter_guided_next_locked (long *, long *);
1051 #ifdef HAVE_SYNC_BUILTINS
1052 extern bool gomp_iter_dynamic_next (long *, long *);
1053 extern bool gomp_iter_guided_next (long *, long *);
1058 extern int gomp_iter_ull_static_next (unsigned long long *,
1059 unsigned long long *);
1060 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *,
1061 unsigned long long *);
1062 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *,
1063 unsigned long long *);
1065 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
1066 extern bool gomp_iter_ull_dynamic_next (unsigned long long *,
1067 unsigned long long *);
1068 extern bool gomp_iter_ull_guided_next (unsigned long long *,
1069 unsigned long long *);
1074 extern void gomp_ordered_first (void);
1075 extern void gomp_ordered_last (void);
1076 extern void gomp_ordered_next (void);
1077 extern void gomp_ordered_static_init (void);
1078 extern void gomp_ordered_static_next (void);
1079 extern void gomp_ordered_sync (void);
1080 extern void gomp_doacross_init (unsigned, long *, long, size_t);
1081 extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
1082 unsigned long long, size_t);
1086 extern unsigned gomp_resolve_num_threads (unsigned, unsigned);
1088 /* proc.c (in config/) */
1090 extern void gomp_init_num_threads (void);
1091 extern unsigned gomp_dynamic_max_threads (void);
1095 extern void gomp_init_task (struct gomp_task
*, struct gomp_task
*,
1096 struct gomp_task_icv
*);
1097 extern void gomp_end_task (void);
1098 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t
);
1099 extern void gomp_task_maybe_wait_for_dependencies (void **);
1100 extern bool gomp_create_target_task (struct gomp_device_descr
*,
1101 void (*) (void *), size_t, void **,
1102 size_t *, unsigned short *, unsigned int,
1104 enum gomp_target_task_state
);
1105 extern struct gomp_taskgroup
*gomp_parallel_reduction_register (uintptr_t *,
1107 extern void gomp_workshare_taskgroup_start (void);
1108 extern void gomp_workshare_task_reduction_register (uintptr_t *, uintptr_t *);
1111 gomp_finish_task (struct gomp_task
*task
)
1113 if (__builtin_expect (task
->depend_hash
!= NULL
, 0))
1114 free (task
->depend_hash
);
1119 extern struct gomp_team
*gomp_new_team (unsigned);
1120 extern void gomp_team_start (void (*) (void *), void *, unsigned,
1121 unsigned, struct gomp_team
*,
1122 struct gomp_taskgroup
*);
1123 extern void gomp_team_end (void);
1124 extern void gomp_free_thread (void *);
1125 extern int gomp_pause_host (void);
1129 extern void gomp_init_targets_once (void);
1130 extern int gomp_get_num_devices (void);
1131 extern bool gomp_target_task_fn (void *);
1132 extern void gomp_target_rev (uint64_t, uint64_t, uint64_t, uint64_t, uint64_t,
1133 int, struct goacc_asyncqueue
*);
1135 /* Splay tree definitions. */
1136 typedef struct splay_tree_node_s
*splay_tree_node
;
1137 typedef struct splay_tree_s
*splay_tree
;
1138 typedef struct splay_tree_key_s
*splay_tree_key
;
1140 struct target_var_desc
{
1143 /* True if data should be copied from device to host at the end. */
1145 /* True if data always should be copied from device to host at the end. */
1146 bool always_copy_from
;
1147 /* True if this is for OpenACC 'attach'. */
1149 /* If GOMP_MAP_TO_PSET had a NULL pointer; used for Fortran descriptors,
1150 which were initially unallocated. */
1151 bool has_null_ptr_assoc
;
1152 /* Relative offset against key host_start. */
1154 /* Actual length. */
1158 struct target_mem_desc
;
1160 /* Special value for refcount - mask to indicate existence of special
1161 values. Right now we allocate 3 bits. */
1162 #define REFCOUNT_SPECIAL (~(uintptr_t) 0x7)
1164 /* Special value for refcount - infinity. */
1165 #define REFCOUNT_INFINITY (REFCOUNT_SPECIAL | 0)
1166 /* Special value for refcount - tgt_offset contains target address of the
1167 artificial pointer to "omp declare target link" object. */
1168 #define REFCOUNT_LINK (REFCOUNT_SPECIAL | 1)
1170 /* Special value for refcount - structure element sibling list items.
1171 All such key refounts have REFCOUNT_STRUCTELEM bits set, with _FLAG_FIRST
1172 and _FLAG_LAST indicating first and last in the created sibling sequence. */
1173 #define REFCOUNT_STRUCTELEM (REFCOUNT_SPECIAL | 4)
1174 #define REFCOUNT_STRUCTELEM_P(V) \
1175 (((V) & REFCOUNT_STRUCTELEM) == REFCOUNT_STRUCTELEM)
1176 /* The first leading key with _FLAG_FIRST set houses the actual reference count
1177 in the structelem_refcount field. Other siblings point to this counter value
1178 through its structelem_refcount_ptr field. */
1179 #define REFCOUNT_STRUCTELEM_FLAG_FIRST (1)
1180 /* The last key in the sibling sequence has this set. This is required to
1181 indicate the sequence boundary, when we remove the structure sibling list
1183 #define REFCOUNT_STRUCTELEM_FLAG_LAST (2)
1185 #define REFCOUNT_STRUCTELEM_FIRST_P(V) \
1186 (REFCOUNT_STRUCTELEM_P (V) && ((V) & REFCOUNT_STRUCTELEM_FLAG_FIRST))
1187 #define REFCOUNT_STRUCTELEM_LAST_P(V) \
1188 (REFCOUNT_STRUCTELEM_P (V) && ((V) & REFCOUNT_STRUCTELEM_FLAG_LAST))
1190 /* Special offset values. */
1191 #define OFFSET_INLINED (~(uintptr_t) 0)
1192 #define OFFSET_POINTER (~(uintptr_t) 1)
1193 #define OFFSET_STRUCT (~(uintptr_t) 2)
1195 /* Auxiliary structure for infrequently-used or API-specific data. */
1197 struct splay_tree_aux
{
1198 /* Pointer to the original mapping of "omp declare target link" object. */
1199 splay_tree_key link_key
;
1200 /* For a block with attached pointers, the attachment counters for each.
1201 Only used for OpenACC. */
1202 uintptr_t *attach_count
;
1205 struct splay_tree_key_s
{
1206 /* Address of the host object. */
1207 uintptr_t host_start
;
1208 /* Address immediately after the host object. */
1210 /* Descriptor of the target memory. */
1211 struct target_mem_desc
*tgt
;
1212 /* Offset from tgt->tgt_start to the start of the target object. */
1213 uintptr_t tgt_offset
;
1214 /* Reference count. */
1217 /* Dynamic reference count. */
1218 uintptr_t dynamic_refcount
;
1220 /* Unified reference count for structure element siblings, this is used
1221 when REFCOUNT_STRUCTELEM_FIRST_P(k->refcount) == true, the first sibling
1222 in a structure element sibling list item sequence. */
1223 uintptr_t structelem_refcount
;
1225 /* When REFCOUNT_STRUCTELEM_P (k->refcount) == true, this field points
1226 into the (above) structelem_refcount field of the _FIRST splay_tree_key,
1227 the first key in the created sequence. All structure element siblings
1228 share a single refcount in this manner. Since these two fields won't be
1229 used at the same time, they are stashed in a union. */
1230 uintptr_t *structelem_refcount_ptr
;
1232 struct splay_tree_aux
*aux
;
1235 /* The comparison function. */
1238 splay_compare (splay_tree_key x
, splay_tree_key y
)
1240 if (x
->host_start
== x
->host_end
1241 && y
->host_start
== y
->host_end
)
1243 if (x
->host_end
<= y
->host_start
)
1245 if (x
->host_start
>= y
->host_end
)
1250 #include "splay-tree.h"
1252 /* Reverse offload splay-tree handling (functions only). */
1254 struct reverse_splay_tree_key_s
{
1255 /* Address of the device object. */
1260 typedef struct reverse_splay_tree_node_s
*reverse_splay_tree_node
;
1261 typedef struct reverse_splay_tree_s
*reverse_splay_tree
;
1262 typedef struct reverse_splay_tree_key_s
*reverse_splay_tree_key
;
1265 reverse_splay_compare (reverse_splay_tree_key x
, reverse_splay_tree_key y
)
1267 if (x
->dev
< y
->dev
)
1269 if (x
->dev
> y
->dev
)
1274 #define splay_tree_prefix reverse
1275 #include "splay-tree.h"
1277 struct target_mem_desc
{
1278 /* Reference count. */
1280 /* All the splay nodes allocated together. */
1281 splay_tree_node array
;
1282 /* Likewise for the reverse lookup device->host for reverse offload. */
1283 reverse_splay_tree_node rev_array
;
1284 /* Start of the target region. */
1285 uintptr_t tgt_start
;
1286 /* End of the targer region. */
1288 /* Handle to free. */
1290 /* Previous target_mem_desc. */
1291 struct target_mem_desc
*prev
;
1292 /* Number of items in following list. */
1295 /* Corresponding target device descriptor. */
1296 struct gomp_device_descr
*device_descr
;
1298 /* List of target items to remove (or decrease refcount)
1299 at the end of region. */
1300 struct target_var_desc list
[];
1304 typedef struct acc_dispatch_t
1307 __typeof (GOMP_OFFLOAD_openacc_exec
) *exec_func
;
1309 /* Create/destroy TLS data. */
1310 __typeof (GOMP_OFFLOAD_openacc_create_thread_data
) *create_thread_data_func
;
1311 __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data
)
1312 *destroy_thread_data_func
;
1315 /* Once created and put into the "active" list, asyncqueues are then never
1316 destructed and removed from the "active" list, other than if the TODO
1317 device is shut down. */
1320 struct goacc_asyncqueue
**asyncqueue
;
1321 struct goacc_asyncqueue_list
*active
;
1323 __typeof (GOMP_OFFLOAD_openacc_async_construct
) *construct_func
;
1324 __typeof (GOMP_OFFLOAD_openacc_async_destruct
) *destruct_func
;
1325 __typeof (GOMP_OFFLOAD_openacc_async_test
) *test_func
;
1326 __typeof (GOMP_OFFLOAD_openacc_async_synchronize
) *synchronize_func
;
1327 __typeof (GOMP_OFFLOAD_openacc_async_serialize
) *serialize_func
;
1328 __typeof (GOMP_OFFLOAD_openacc_async_queue_callback
) *queue_callback_func
;
1330 __typeof (GOMP_OFFLOAD_openacc_async_exec
) *exec_func
;
1331 __typeof (GOMP_OFFLOAD_openacc_async_dev2host
) *dev2host_func
;
1332 __typeof (GOMP_OFFLOAD_openacc_async_host2dev
) *host2dev_func
;
1335 __typeof (GOMP_OFFLOAD_openacc_get_property
) *get_property_func
;
1337 /* NVIDIA target specific routines. */
1339 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device
)
1340 *get_current_device_func
;
1341 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context
)
1342 *get_current_context_func
;
1343 __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream
) *get_stream_func
;
1344 __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream
) *set_stream_func
;
1348 /* Various state of the accelerator device. */
1349 enum gomp_device_state
1351 GOMP_DEVICE_UNINITIALIZED
,
1352 GOMP_DEVICE_INITIALIZED
,
1353 GOMP_DEVICE_FINALIZED
1356 /* This structure describes accelerator device.
1357 It contains name of the corresponding libgomp plugin, function handlers for
1358 interaction with the device, ID-number of the device, and information about
1360 struct gomp_device_descr
1362 /* Immutable data, which is only set during initialization, and which is not
1363 guarded by the lock. */
1365 /* The name of the device. */
1368 /* Capabilities of device (supports OpenACC, OpenMP). */
1369 unsigned int capabilities
;
1371 /* This is the ID number of device among devices of the same type. */
1374 /* This is the TYPE of device. */
1375 enum offload_target_type type
;
1377 /* Function handlers. */
1378 __typeof (GOMP_OFFLOAD_get_name
) *get_name_func
;
1379 __typeof (GOMP_OFFLOAD_get_caps
) *get_caps_func
;
1380 __typeof (GOMP_OFFLOAD_get_type
) *get_type_func
;
1381 __typeof (GOMP_OFFLOAD_get_num_devices
) *get_num_devices_func
;
1382 __typeof (GOMP_OFFLOAD_init_device
) *init_device_func
;
1383 __typeof (GOMP_OFFLOAD_fini_device
) *fini_device_func
;
1384 __typeof (GOMP_OFFLOAD_version
) *version_func
;
1385 __typeof (GOMP_OFFLOAD_load_image
) *load_image_func
;
1386 __typeof (GOMP_OFFLOAD_unload_image
) *unload_image_func
;
1387 __typeof (GOMP_OFFLOAD_alloc
) *alloc_func
;
1388 __typeof (GOMP_OFFLOAD_free
) *free_func
;
1389 __typeof (GOMP_OFFLOAD_dev2host
) *dev2host_func
;
1390 __typeof (GOMP_OFFLOAD_host2dev
) *host2dev_func
;
1391 __typeof (GOMP_OFFLOAD_dev2dev
) *dev2dev_func
;
1392 __typeof (GOMP_OFFLOAD_can_run
) *can_run_func
;
1393 __typeof (GOMP_OFFLOAD_run
) *run_func
;
1394 __typeof (GOMP_OFFLOAD_async_run
) *async_run_func
;
1396 /* Splay tree containing information about mapped memory regions. */
1397 struct splay_tree_s mem_map
;
1398 struct reverse_splay_tree_s mem_map_rev
;
1400 /* Mutex for the mutable data. */
1403 /* Current state of the device. OpenACC allows to move from INITIALIZED state
1404 back to UNINITIALIZED state. OpenMP allows only to move from INITIALIZED
1405 to FINALIZED state (at program shutdown). */
1406 enum gomp_device_state state
;
1408 /* OpenACC-specific data and functions. */
1409 /* This is mutable because of its mutable target_data member. */
1410 acc_dispatch_t openacc
;
1413 /* Kind of the pragma, for which gomp_map_vars () is called. */
1414 enum gomp_map_vars_kind
1416 GOMP_MAP_VARS_OPENACC
= 1,
1417 GOMP_MAP_VARS_TARGET
= 2,
1418 GOMP_MAP_VARS_DATA
= 4,
1419 GOMP_MAP_VARS_ENTER_DATA
= 8
1422 extern void gomp_acc_declare_allocate (bool, size_t, void **, size_t *,
1424 struct gomp_coalesce_buf
;
1425 extern void gomp_copy_host2dev (struct gomp_device_descr
*,
1426 struct goacc_asyncqueue
*, void *, const void *,
1427 size_t, bool, struct gomp_coalesce_buf
*);
1428 extern void gomp_copy_dev2host (struct gomp_device_descr
*,
1429 struct goacc_asyncqueue
*, void *, const void *,
1431 extern uintptr_t gomp_map_val (struct target_mem_desc
*, void **, size_t);
1432 extern void gomp_attach_pointer (struct gomp_device_descr
*,
1433 struct goacc_asyncqueue
*, splay_tree
,
1434 splay_tree_key
, uintptr_t, size_t,
1435 struct gomp_coalesce_buf
*, bool);
1436 extern void gomp_detach_pointer (struct gomp_device_descr
*,
1437 struct goacc_asyncqueue
*, splay_tree_key
,
1438 uintptr_t, bool, struct gomp_coalesce_buf
*);
1439 extern struct target_mem_desc
*goacc_map_vars (struct gomp_device_descr
*,
1440 struct goacc_asyncqueue
*,
1441 size_t, void **, void **,
1442 size_t *, void *, bool,
1443 enum gomp_map_vars_kind
);
1444 extern void goacc_unmap_vars (struct target_mem_desc
*, bool,
1445 struct goacc_asyncqueue
*);
1446 extern void gomp_init_device (struct gomp_device_descr
*);
1447 extern bool gomp_fini_device (struct gomp_device_descr
*);
1448 extern void gomp_unload_device (struct gomp_device_descr
*);
1449 extern bool gomp_remove_var (struct gomp_device_descr
*, splay_tree_key
);
1450 extern void gomp_remove_var_async (struct gomp_device_descr
*, splay_tree_key
,
1451 struct goacc_asyncqueue
*);
1455 extern void gomp_init_work_share (struct gomp_work_share
*, size_t, unsigned);
1456 extern void gomp_fini_work_share (struct gomp_work_share
*);
1457 extern bool gomp_work_share_start (size_t);
1458 extern void gomp_work_share_end (void);
1459 extern bool gomp_work_share_end_cancel (void);
1460 extern void gomp_work_share_end_nowait (void);
1463 gomp_work_share_init_done (void)
1465 struct gomp_thread
*thr
= gomp_thread ();
1466 if (__builtin_expect (thr
->ts
.last_work_share
!= NULL
, 1))
1467 gomp_ptrlock_set (&thr
->ts
.last_work_share
->next_ws
, thr
->ts
.work_share
);
1470 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1471 # pragma GCC visibility pop
1474 /* Now that we're back to default visibility, include the globals. */
1475 #include "libgomp_g.h"
1477 /* Include omp.h by parts. */
1478 #include "omp-lock.h"
1479 #define _LIBGOMP_OMP_LOCK_DEFINED 1
1482 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \
1483 || !defined (HAVE_ATTRIBUTE_ALIAS) \
1484 || !defined (HAVE_AS_SYMVER_DIRECTIVE) \
1486 || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT)
1487 # undef LIBGOMP_GNU_SYMBOL_VERSIONING
1490 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING
1491 extern void gomp_init_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1492 extern void gomp_destroy_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1493 extern void gomp_set_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1494 extern void gomp_unset_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1495 extern int gomp_test_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1496 extern void gomp_init_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1497 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1498 extern void gomp_set_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1499 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1500 extern int gomp_test_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1502 extern void gomp_init_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1503 extern void gomp_destroy_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1504 extern void gomp_set_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1505 extern void gomp_unset_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1506 extern int gomp_test_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1507 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1508 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1509 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1510 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1511 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1513 # define omp_lock_symver(fn) \
1514 __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
1515 __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
1517 # define gomp_init_lock_30 omp_init_lock
1518 # define gomp_destroy_lock_30 omp_destroy_lock
1519 # define gomp_set_lock_30 omp_set_lock
1520 # define gomp_unset_lock_30 omp_unset_lock
1521 # define gomp_test_lock_30 omp_test_lock
1522 # define gomp_init_nest_lock_30 omp_init_nest_lock
1523 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock
1524 # define gomp_set_nest_lock_30 omp_set_nest_lock
1525 # define gomp_unset_nest_lock_30 omp_unset_nest_lock
1526 # define gomp_test_nest_lock_30 omp_test_nest_lock
1529 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1530 # define attribute_hidden __attribute__ ((visibility ("hidden")))
1532 # define attribute_hidden
1536 # define HAVE_ATTRIBUTE_COPY
1539 #ifdef HAVE_ATTRIBUTE_COPY
1540 # define attribute_copy(arg) __attribute__ ((copy (arg)))
1542 # define attribute_copy(arg)
1545 #ifdef HAVE_ATTRIBUTE_ALIAS
1546 # define strong_alias(fn, al) \
1547 extern __typeof (fn) al __attribute__ ((alias (#fn))) attribute_copy (fn);
1549 # define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__)
1550 # define ialias_str1(x) ialias_str2(x)
1551 # define ialias_str2(x) #x
1552 # define ialias(fn) \
1553 extern __typeof (fn) gomp_ialias_##fn \
1554 __attribute__ ((alias (#fn))) attribute_hidden attribute_copy (fn);
1555 # define ialias_redirect(fn) \
1556 extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
1557 # define ialias_call(fn) gomp_ialias_ ## fn
1560 # define ialias_redirect(fn)
1561 # define ialias_call(fn) fn
1564 /* Helper function for priority_node_to_task() and
1565 task_to_priority_node().
1567 Return the offset from a task to its priority_node entry. The
1568 priority_node entry is has a type of TYPE. */
1570 static inline size_t
1571 priority_queue_offset (enum priority_queue_type type
)
1573 return offsetof (struct gomp_task
, pnode
[(int) type
]);
1576 /* Return the task associated with a priority NODE of type TYPE. */
1578 static inline struct gomp_task
*
1579 priority_node_to_task (enum priority_queue_type type
,
1580 struct priority_node
*node
)
1582 return (struct gomp_task
*) ((char *) node
- priority_queue_offset (type
));
1585 /* Return the priority node of type TYPE for a given TASK. */
1587 static inline struct priority_node
*
1588 task_to_priority_node (enum priority_queue_type type
,
1589 struct gomp_task
*task
)
1591 return (struct priority_node
*) ((char *) task
1592 + priority_queue_offset (type
));
1595 #ifdef LIBGOMP_USE_PTHREADS
1596 static inline gomp_thread_handle
1597 gomp_thread_self (void)
1599 return pthread_self ();
1602 static inline gomp_thread_handle
1603 gomp_thread_to_pthread_t (struct gomp_thread
*thr
)
1605 struct gomp_thread
*this_thr
= gomp_thread ();
1606 if (thr
== this_thr
)
1607 return pthread_self ();
1608 #ifdef GOMP_NEEDS_THREAD_HANDLE
1611 /* On Linux with initial-exec TLS, the pthread_t of the thread containing
1612 thr can be computed from thr, this_thr and pthread_self (),
1613 as the distance between this_thr and pthread_self () is constant. */
1614 return pthread_self () + ((uintptr_t) thr
- (uintptr_t) this_thr
);
1618 static inline gomp_thread_handle
1619 gomp_thread_self (void)
1621 return (gomp_thread_handle
) {};
1624 static inline gomp_thread_handle
1625 gomp_thread_to_pthread_t (struct gomp_thread
*thr
)
1628 return gomp_thread_self ();
1632 #endif /* LIBGOMP_H */