1 /* Copyright (C) 2005-2015 Free Software Foundation, Inc.
2 Contributed by Richard Henderson <rth@redhat.com>.
4 This file is part of the GNU Offloading and Multi Processing Library
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
26 /* This file contains data types and function declarations that are not
27 part of the official OpenACC or OpenMP user interfaces. There are
28 declarations in here that are part of the GNU Offloading and Multi
29 Processing ABI, in that the compiler is required to know about them
32 The convention is that the all caps prefix "GOMP" is used group items
33 that are part of the external ABI, and the lower case prefix "gomp"
34 is used group items that are completely private to the library. */
39 #ifndef _LIBGOMP_CHECKING_
40 /* Define to 1 to perform internal sanity checks. */
41 #define _LIBGOMP_CHECKING_ 0
46 #include "libgomp-plugin.h"
53 /* Needed for memset in priority_queue.c. */
54 #if _LIBGOMP_CHECKING_
55 # ifdef STRING_WITH_STRINGS
62 # ifdef HAVE_STRINGS_H
69 #ifdef HAVE_ATTRIBUTE_VISIBILITY
70 # pragma GCC visibility push(hidden)
73 /* If we were a C++ library, we'd get this from <std/atomic>. */
86 extern void *gomp_malloc (size_t) __attribute__((malloc
));
87 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc
));
88 extern void *gomp_realloc (void *, size_t);
90 /* Avoid conflicting prototypes of alloca() in system headers by using
91 GCC's builtin alloca(). */
92 #define gomp_alloca(x) __builtin_alloca(x)
96 extern void gomp_vdebug (int, const char *, va_list);
97 extern void gomp_debug (int, const char *, ...)
98 __attribute__ ((format (printf
, 2, 3)));
99 #define gomp_vdebug(KIND, FMT, VALIST) \
101 if (__builtin_expect (gomp_debug_var, 0)) \
102 (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
104 #define gomp_debug(KIND, ...) \
106 if (__builtin_expect (gomp_debug_var, 0)) \
107 (gomp_debug) ((KIND), __VA_ARGS__); \
109 extern void gomp_verror (const char *, va_list);
110 extern void gomp_error (const char *, ...)
111 __attribute__ ((format (printf
, 1, 2)));
112 extern void gomp_vfatal (const char *, va_list)
113 __attribute__ ((noreturn
));
114 extern void gomp_fatal (const char *, ...)
115 __attribute__ ((noreturn
, format (printf
, 1, 2)));
118 struct gomp_taskgroup
;
121 #include "priority_queue.h"
128 /* This structure contains the data to control one work-sharing construct,
129 either a LOOP (FOR/DO) or a SECTIONS. */
131 enum gomp_schedule_type
140 struct gomp_doacross_work_share
143 /* chunk_size copy, as ws->chunk_size is multiplied by incr for
146 /* Likewise, but for ull implementation. */
147 unsigned long long chunk_size_ull
;
148 /* For schedule(static,0) this is the number
149 of iterations assigned to the last thread, i.e. number of
150 iterations / number of threads. */
152 /* Likewise, but for ull implementation. */
153 unsigned long long q_ull
;
155 /* Size of each array entry (padded to cache line size). */
156 unsigned long elt_sz
;
157 /* Number of dimensions in sink vectors. */
158 unsigned int ncounts
;
159 /* True if the iterations can be flattened. */
161 /* Actual array (of elt_sz sized units), aligned to cache line size.
162 This is indexed by team_id for GFS_STATIC and outermost iteration
163 / chunk_size for other schedules. */
164 unsigned char *array
;
165 /* These two are only used for schedule(static,0). */
166 /* This one is number of iterations % number of threads. */
169 /* And this one is cached t * (q + 1). */
171 /* Likewise, but for the ull implementation. */
172 unsigned long long boundary_ull
;
174 /* Array of shift counts for each dimension if they can be flattened. */
175 unsigned int shift_counts
[];
178 struct gomp_work_share
180 /* This member records the SCHEDULE clause to be used for this construct.
181 The user specification of "runtime" will already have been resolved.
182 If this is a SECTIONS construct, this value will always be DYNAMIC. */
183 enum gomp_schedule_type sched
;
189 /* This is the chunk_size argument to the SCHEDULE clause. */
192 /* This is the iteration end point. If this is a SECTIONS construct,
193 this is the number of contained sections. */
196 /* This is the iteration step. If this is a SECTIONS construct, this
202 /* The same as above, but for the unsigned long long loop variants. */
203 unsigned long long chunk_size_ull
;
204 unsigned long long end_ull
;
205 unsigned long long incr_ull
;
210 /* This is a circular queue that details which threads will be allowed
211 into the ordered region and in which order. When a thread allocates
212 iterations on which it is going to work, it also registers itself at
213 the end of the array. When a thread reaches the ordered region, it
214 checks to see if it is the one at the head of the queue. If not, it
215 blocks on its RELEASE semaphore. */
216 unsigned *ordered_team_ids
;
218 /* This is a pointer to DOACROSS work share data. */
219 struct gomp_doacross_work_share
*doacross
;
222 /* This is the number of threads that have registered themselves in
223 the circular queue ordered_team_ids. */
224 unsigned ordered_num_used
;
226 /* This is the team_id of the currently acknowledged owner of the ordered
227 section, or -1u if the ordered section has not been acknowledged by
228 any thread. This is distinguished from the thread that is *allowed*
229 to take the section next. */
230 unsigned ordered_owner
;
232 /* This is the index into the circular queue ordered_team_ids of the
233 current thread that's allowed into the ordered reason. */
234 unsigned ordered_cur
;
236 /* This is a chain of allocated gomp_work_share blocks, valid only
237 in the first gomp_work_share struct in the block. */
238 struct gomp_work_share
*next_alloc
;
240 /* The above fields are written once during workshare initialization,
241 or related to ordered worksharing. Make sure the following fields
242 are in a different cache line. */
244 /* This lock protects the update of the following members. */
245 gomp_mutex_t lock
__attribute__((aligned (64)));
247 /* This is the count of the number of threads that have exited the work
248 share construct. If the construct was marked nowait, they have moved on
249 to other work; otherwise they're blocked on a barrier. The last member
250 of the team to exit the work share construct must deallocate it. */
251 unsigned threads_completed
;
254 /* This is the next iteration value to be allocated. In the case of
255 GFS_STATIC loops, this the iteration start point and never changes. */
258 /* The same, but with unsigned long long type. */
259 unsigned long long next_ull
;
261 /* This is the returned data structure for SINGLE COPYPRIVATE. */
266 /* Link to gomp_work_share struct for next work sharing construct
267 encountered after this one. */
268 gomp_ptrlock_t next_ws
;
270 /* gomp_work_share structs are chained in the free work share cache
272 struct gomp_work_share
*next_free
;
275 /* If only few threads are in the team, ordered_team_ids can point
276 to this array which fills the padding at the end of this struct. */
277 unsigned inline_ordered_team_ids
[0];
280 /* This structure contains all of the thread-local data associated with
281 a thread team. This is the data that must be saved when a thread
282 encounters a nested PARALLEL construct. */
284 struct gomp_team_state
286 /* This is the team of which the thread is currently a member. */
287 struct gomp_team
*team
;
289 /* This is the work share construct which this thread is currently
290 processing. Recall that with NOWAIT, not all threads may be
291 processing the same construct. */
292 struct gomp_work_share
*work_share
;
294 /* This is the previous work share construct or NULL if there wasn't any.
295 When all threads are done with the current work sharing construct,
296 the previous one can be freed. The current one can't, as its
297 next_ws field is used. */
298 struct gomp_work_share
*last_work_share
;
300 /* This is the ID of this thread within the team. This value is
301 guaranteed to be between 0 and N-1, where N is the number of
302 threads in the team. */
308 /* Active nesting level. Only active parallel regions are counted. */
309 unsigned active_level
;
311 /* Place-partition-var, offset and length into gomp_places_list array. */
312 unsigned place_partition_off
;
313 unsigned place_partition_len
;
315 #ifdef HAVE_SYNC_BUILTINS
316 /* Number of single stmts encountered. */
317 unsigned long single_count
;
320 /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the
321 trip number through the loop. So first time a particular loop
322 is encountered this number is 0, the second time through the loop
323 is 1, etc. This is unused when the compiler knows in advance that
324 the loop is statically scheduled. */
325 unsigned long static_trip
;
328 struct target_mem_desc
;
330 /* These are the OpenMP 4.0 Internal Control Variables described in
331 section 2.3.1. Those described as having one copy per task are
332 stored within the structure; those described as having one copy
333 for the whole program are (naturally) global variables. */
337 unsigned long nthreads_var
;
338 enum gomp_schedule_type run_sched_var
;
339 int run_sched_chunk_size
;
340 int default_device_var
;
341 unsigned int thread_limit_var
;
346 struct target_mem_desc
*target_data
;
349 extern struct gomp_task_icv gomp_global_icv
;
350 #ifndef HAVE_SYNC_BUILTINS
351 extern gomp_mutex_t gomp_managed_threads_lock
;
353 extern unsigned long gomp_max_active_levels_var
;
354 extern bool gomp_cancel_var
;
355 extern int gomp_max_task_priority_var
;
356 extern unsigned long long gomp_spin_count_var
, gomp_throttled_spin_count_var
;
357 extern unsigned long gomp_available_cpus
, gomp_managed_threads
;
358 extern unsigned long *gomp_nthreads_var_list
, gomp_nthreads_var_list_len
;
359 extern char *gomp_bind_var_list
;
360 extern unsigned long gomp_bind_var_list_len
;
361 extern void **gomp_places_list
;
362 extern unsigned long gomp_places_list_len
;
363 extern int gomp_debug_var
;
364 extern int goacc_device_num
;
365 extern char *goacc_device_type
;
371 /* Undeferred task. */
372 GOMP_TASK_UNDEFERRED
,
373 /* Task created by GOMP_task and waiting to be run. */
375 /* Task currently executing or scheduled and about to execute. */
377 /* Used for target tasks that have vars mapped and async run started,
378 but not yet completed. Once that completes, they will be readded
379 into the queues as GOMP_TASK_WAITING in order to perform the var
381 GOMP_TASK_ASYNC_RUNNING
384 struct gomp_task_depend_entry
386 /* Address of dependency. */
388 struct gomp_task_depend_entry
*next
;
389 struct gomp_task_depend_entry
*prev
;
390 /* Task that provides the dependency in ADDR. */
391 struct gomp_task
*task
;
392 /* Depend entry is of type "IN". */
398 struct gomp_dependers_vec
402 struct gomp_task
*elem
[];
405 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies. */
411 /* Number of tasks we are waiting for. */
413 gomp_sem_t taskwait_sem
;
416 /* This structure describes a "task" to be run by a thread. */
420 /* Parent of this task. */
421 struct gomp_task
*parent
;
422 /* Children of this task. */
423 struct priority_queue children_queue
;
424 /* Taskgroup this task belongs in. */
425 struct gomp_taskgroup
*taskgroup
;
426 /* Tasks that depend on this task. */
427 struct gomp_dependers_vec
*dependers
;
428 struct htab
*depend_hash
;
429 struct gomp_taskwait
*taskwait
;
430 /* Number of items in DEPEND. */
432 /* Number of tasks this task depends on. Once this counter reaches
433 0, we have no unsatisfied dependencies, and this task can be put
434 into the various queues to be scheduled. */
435 size_t num_dependees
;
437 /* Priority of this task. */
439 /* The priority node for this task in each of the different queues.
440 We put this here to avoid allocating space for each priority
441 node. Then we play offsetof() games to convert between pnode[]
442 entries and the gomp_task in which they reside. */
443 struct priority_node pnode
[3];
445 struct gomp_task_icv icv
;
448 enum gomp_task_kind kind
;
451 bool copy_ctors_done
;
452 /* Set for undeferred tasks with unsatisfied dependencies which
453 block further execution of their parent until the dependencies
455 bool parent_depends_on
;
456 /* Dependencies provided and/or needed for this task. DEPEND_COUNT
457 is the number of items available. */
458 struct gomp_task_depend_entry depend
[];
461 /* This structure describes a single #pragma omp taskgroup. */
463 struct gomp_taskgroup
465 struct gomp_taskgroup
*prev
;
466 /* Queue of tasks that belong in this taskgroup. */
467 struct priority_queue taskgroup_queue
;
468 bool in_taskgroup_wait
;
470 gomp_sem_t taskgroup_sem
;
474 /* Various state of OpenMP async offloading tasks. */
475 enum gomp_target_task_state
477 GOMP_TARGET_TASK_DATA
,
478 GOMP_TARGET_TASK_BEFORE_MAP
,
479 GOMP_TARGET_TASK_FALLBACK
,
480 GOMP_TARGET_TASK_READY_TO_RUN
,
481 GOMP_TARGET_TASK_RUNNING
,
482 GOMP_TARGET_TASK_FINISHED
485 /* This structure describes a target task. */
487 struct gomp_target_task
489 struct gomp_device_descr
*devicep
;
493 unsigned short *kinds
;
495 enum gomp_target_task_state state
;
496 struct target_mem_desc
*tgt
;
497 struct gomp_task
*task
;
498 struct gomp_team
*team
;
502 /* This structure describes a "team" of threads. These are the threads
503 that are spawned by a PARALLEL constructs, as well as the work sharing
504 constructs that the team encounters. */
508 /* This is the number of threads in the current team. */
511 /* This is number of gomp_work_share structs that have been allocated
512 as a block last time. */
513 unsigned work_share_chunk
;
515 /* This is the saved team state that applied to a master thread before
516 the current thread was created. */
517 struct gomp_team_state prev_ts
;
519 /* This semaphore should be used by the master thread instead of its
520 "native" semaphore in the thread structure. Required for nested
521 parallels, as the master is a member of two teams. */
522 gomp_sem_t master_release
;
524 /* This points to an array with pointers to the release semaphore
525 of the threads in the team. */
526 gomp_sem_t
**ordered_release
;
528 /* List of work shares on which gomp_fini_work_share hasn't been
529 called yet. If the team hasn't been cancelled, this should be
530 equal to each thr->ts.work_share, but otherwise it can be a possibly
531 long list of workshares. */
532 struct gomp_work_share
*work_shares_to_free
;
534 /* List of gomp_work_share structs chained through next_free fields.
535 This is populated and taken off only by the first thread in the
536 team encountering a new work sharing construct, in a critical
538 struct gomp_work_share
*work_share_list_alloc
;
540 /* List of gomp_work_share structs freed by free_work_share. New
541 entries are atomically added to the start of the list, and
542 alloc_work_share can safely only move all but the first entry
543 to work_share_list alloc, as free_work_share can happen concurrently
544 with alloc_work_share. */
545 struct gomp_work_share
*work_share_list_free
;
547 #ifdef HAVE_SYNC_BUILTINS
548 /* Number of simple single regions encountered by threads in this
550 unsigned long single_count
;
552 /* Mutex protecting addition of workshares to work_share_list_free. */
553 gomp_mutex_t work_share_list_free_lock
;
556 /* This barrier is used for most synchronization of the team. */
557 gomp_barrier_t barrier
;
559 /* Initial work shares, to avoid allocating any gomp_work_share
560 structs in the common case. */
561 struct gomp_work_share work_shares
[8];
563 gomp_mutex_t task_lock
;
564 /* Scheduled tasks. */
565 struct priority_queue task_queue
;
566 /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */
567 unsigned int task_count
;
568 /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */
569 unsigned int task_queued_count
;
570 /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
571 directly in gomp_barrier_handle_tasks; tasks spawned
572 from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
573 that is called from a task run from gomp_barrier_handle_tasks.
574 task_running_count should be always <= team->nthreads,
575 and if current task isn't in_tied_task, then it will be
576 even < team->nthreads. */
577 unsigned int task_running_count
;
578 int work_share_cancelled
;
581 /* This array contains structures for implicit tasks. */
582 struct gomp_task implicit_task
[];
585 /* This structure contains all data that is private to libgomp and is
586 allocated per thread. */
590 /* This is the function that the thread should run upon launch. */
591 void (*fn
) (void *data
);
594 /* This is the current team state for this thread. The ts.team member
595 is NULL only if the thread is idle. */
596 struct gomp_team_state ts
;
598 /* This is the task that the thread is currently executing. */
599 struct gomp_task
*task
;
601 /* This semaphore is used for ordered loops. */
604 /* Place this thread is bound to plus one, or zero if not bound
608 /* User pthread thread pool */
609 struct gomp_thread_pool
*thread_pool
;
613 struct gomp_thread_pool
615 /* This array manages threads spawned from the top level, which will
616 return to the idle loop once the current PARALLEL construct ends. */
617 struct gomp_thread
**threads
;
618 unsigned threads_size
;
619 unsigned threads_used
;
620 /* The last team is used for non-nested teams to delay their destruction to
621 make sure all the threads in the team move on to the pool's barrier before
622 the team's barrier is destroyed. */
623 struct gomp_team
*last_team
;
624 /* Number of threads running in this contention group. */
625 unsigned long threads_busy
;
627 /* This barrier holds and releases threads waiting in threads. */
628 gomp_barrier_t threads_dock
;
631 enum gomp_cancel_kind
633 GOMP_CANCEL_PARALLEL
= 1,
634 GOMP_CANCEL_LOOP
= 2,
635 GOMP_CANCEL_FOR
= GOMP_CANCEL_LOOP
,
636 GOMP_CANCEL_DO
= GOMP_CANCEL_LOOP
,
637 GOMP_CANCEL_SECTIONS
= 4,
638 GOMP_CANCEL_TASKGROUP
= 8
641 /* ... and here is that TLS data. */
643 #if defined HAVE_TLS || defined USE_EMUTLS
644 extern __thread
struct gomp_thread gomp_tls_data
;
645 static inline struct gomp_thread
*gomp_thread (void)
647 return &gomp_tls_data
;
650 extern pthread_key_t gomp_tls_key
;
651 static inline struct gomp_thread
*gomp_thread (void)
653 return pthread_getspecific (gomp_tls_key
);
657 extern struct gomp_task_icv
*gomp_new_icv (void);
659 /* Here's how to access the current copy of the ICVs. */
661 static inline struct gomp_task_icv
*gomp_icv (bool write
)
663 struct gomp_task
*task
= gomp_thread ()->task
;
667 return gomp_new_icv ();
669 return &gomp_global_icv
;
672 /* The attributes to be used during thread creation. */
673 extern pthread_attr_t gomp_thread_attr
;
675 extern pthread_key_t gomp_thread_destructor
;
677 /* Function prototypes. */
681 extern void gomp_init_affinity (void);
682 extern void gomp_init_thread_affinity (pthread_attr_t
*, unsigned int);
683 extern void **gomp_affinity_alloc (unsigned long, bool);
684 extern void gomp_affinity_init_place (void *);
685 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
687 extern bool gomp_affinity_remove_cpu (void *, unsigned long);
688 extern bool gomp_affinity_copy_place (void *, void *, long);
689 extern bool gomp_affinity_same_place (void *, void *);
690 extern bool gomp_affinity_finalize_place_list (bool);
691 extern bool gomp_affinity_init_level (int, unsigned long, bool);
692 extern void gomp_affinity_print_place (void *);
693 extern void gomp_get_place_proc_ids_8 (int, int64_t *);
697 extern int gomp_iter_static_next (long *, long *);
698 extern bool gomp_iter_dynamic_next_locked (long *, long *);
699 extern bool gomp_iter_guided_next_locked (long *, long *);
701 #ifdef HAVE_SYNC_BUILTINS
702 extern bool gomp_iter_dynamic_next (long *, long *);
703 extern bool gomp_iter_guided_next (long *, long *);
708 extern int gomp_iter_ull_static_next (unsigned long long *,
709 unsigned long long *);
710 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *,
711 unsigned long long *);
712 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *,
713 unsigned long long *);
715 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
716 extern bool gomp_iter_ull_dynamic_next (unsigned long long *,
717 unsigned long long *);
718 extern bool gomp_iter_ull_guided_next (unsigned long long *,
719 unsigned long long *);
724 extern void gomp_ordered_first (void);
725 extern void gomp_ordered_last (void);
726 extern void gomp_ordered_next (void);
727 extern void gomp_ordered_static_init (void);
728 extern void gomp_ordered_static_next (void);
729 extern void gomp_ordered_sync (void);
730 extern void gomp_doacross_init (unsigned, long *, long);
731 extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
736 extern unsigned gomp_resolve_num_threads (unsigned, unsigned);
738 /* proc.c (in config/) */
740 extern void gomp_init_num_threads (void);
741 extern unsigned gomp_dynamic_max_threads (void);
745 extern void gomp_init_task (struct gomp_task
*, struct gomp_task
*,
746 struct gomp_task_icv
*);
747 extern void gomp_end_task (void);
748 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t
);
749 extern void gomp_task_maybe_wait_for_dependencies (void **);
750 extern bool gomp_create_target_task (struct gomp_device_descr
*,
751 void (*) (void *), size_t, void **,
752 size_t *, unsigned short *, unsigned int,
753 void **, enum gomp_target_task_state
);
756 gomp_finish_task (struct gomp_task
*task
)
758 if (__builtin_expect (task
->depend_hash
!= NULL
, 0))
759 free (task
->depend_hash
);
764 extern struct gomp_team
*gomp_new_team (unsigned);
765 extern void gomp_team_start (void (*) (void *), void *, unsigned,
766 unsigned, struct gomp_team
*);
767 extern void gomp_team_end (void);
768 extern void gomp_free_thread (void *);
772 extern void gomp_init_targets_once (void);
773 extern int gomp_get_num_devices (void);
774 extern bool gomp_target_task_fn (void *);
776 /* Splay tree definitions. */
777 typedef struct splay_tree_node_s
*splay_tree_node
;
778 typedef struct splay_tree_s
*splay_tree
;
779 typedef struct splay_tree_key_s
*splay_tree_key
;
781 struct target_var_desc
{
784 /* True if data should be copied from device to host at the end. */
786 /* True if data always should be copied from device to host at the end. */
787 bool always_copy_from
;
788 /* Relative offset against key host_start. */
794 struct target_mem_desc
{
795 /* Reference count. */
797 /* All the splay nodes allocated together. */
798 splay_tree_node array
;
799 /* Start of the target region. */
801 /* End of the targer region. */
803 /* Handle to free. */
805 /* Previous target_mem_desc. */
806 struct target_mem_desc
*prev
;
807 /* Number of items in following list. */
810 /* Corresponding target device descriptor. */
811 struct gomp_device_descr
*device_descr
;
813 /* List of target items to remove (or decrease refcount)
814 at the end of region. */
815 struct target_var_desc list
[];
818 /* Special value for refcount - infinity. */
819 #define REFCOUNT_INFINITY (~(uintptr_t) 0)
820 /* Special value for refcount - tgt_offset contains target address of the
821 artificial pointer to "omp declare target link" object. */
822 #define REFCOUNT_LINK (~(uintptr_t) 1)
824 struct splay_tree_key_s
{
825 /* Address of the host object. */
826 uintptr_t host_start
;
827 /* Address immediately after the host object. */
829 /* Descriptor of the target memory. */
830 struct target_mem_desc
*tgt
;
831 /* Offset from tgt->tgt_start to the start of the target object. */
832 uintptr_t tgt_offset
;
833 /* Reference count. */
835 /* Asynchronous reference count. */
836 uintptr_t async_refcount
;
837 /* Pointer to the original mapping of "omp declare target link" object. */
838 splay_tree_key link_key
;
841 /* The comparison function. */
844 splay_compare (splay_tree_key x
, splay_tree_key y
)
846 if (x
->host_start
== x
->host_end
847 && y
->host_start
== y
->host_end
)
849 if (x
->host_end
<= y
->host_start
)
851 if (x
->host_start
>= y
->host_end
)
856 #include "splay-tree.h"
858 typedef struct acc_dispatch_t
860 /* This is a linked list of data mapped using the
861 acc_map_data/acc_unmap_data or "acc enter data"/"acc exit data" pragmas.
862 Unlike mapped_data in the goacc_thread struct, unmapping can
863 happen out-of-order with respect to mapping. */
864 /* This is guarded by the lock in the "outer" struct gomp_device_descr. */
865 struct target_mem_desc
*data_environ
;
868 void (*exec_func
) (void (*) (void *), size_t, void **, void **, int,
871 /* Async cleanup callback registration. */
872 void (*register_async_cleanup_func
) (void *);
874 /* Asynchronous routines. */
875 int (*async_test_func
) (int);
876 int (*async_test_all_func
) (void);
877 void (*async_wait_func
) (int);
878 void (*async_wait_async_func
) (int, int);
879 void (*async_wait_all_func
) (void);
880 void (*async_wait_all_async_func
) (int);
881 void (*async_set_async_func
) (int);
883 /* Create/destroy TLS data. */
884 void *(*create_thread_data_func
) (int);
885 void (*destroy_thread_data_func
) (void *);
887 /* NVIDIA target specific routines. */
889 void *(*get_current_device_func
) (void);
890 void *(*get_current_context_func
) (void);
891 void *(*get_stream_func
) (int);
892 int (*set_stream_func
) (int, void *);
896 /* Various state of the accelerator device. */
897 enum gomp_device_state
899 GOMP_DEVICE_UNINITIALIZED
,
900 GOMP_DEVICE_INITIALIZED
,
901 GOMP_DEVICE_FINALIZED
904 /* This structure describes accelerator device.
905 It contains name of the corresponding libgomp plugin, function handlers for
906 interaction with the device, ID-number of the device, and information about
908 struct gomp_device_descr
910 /* Immutable data, which is only set during initialization, and which is not
911 guarded by the lock. */
913 /* The name of the device. */
916 /* Capabilities of device (supports OpenACC, OpenMP). */
917 unsigned int capabilities
;
919 /* This is the ID number of device among devices of the same type. */
922 /* This is the TYPE of device. */
923 enum offload_target_type type
;
925 /* Function handlers. */
926 const char *(*get_name_func
) (void);
927 unsigned int (*get_caps_func
) (void);
928 int (*get_type_func
) (void);
929 int (*get_num_devices_func
) (void);
930 void (*init_device_func
) (int);
931 void (*fini_device_func
) (int);
932 unsigned (*version_func
) (void);
933 int (*load_image_func
) (int, unsigned, const void *, struct addr_pair
**);
934 void (*unload_image_func
) (int, unsigned, const void *);
935 void *(*alloc_func
) (int, size_t);
936 void (*free_func
) (int, void *);
937 void *(*dev2host_func
) (int, void *, const void *, size_t);
938 void *(*host2dev_func
) (int, void *, const void *, size_t);
939 void *(*dev2dev_func
) (int, void *, const void *, size_t);
940 void (*run_func
) (int, void *, void *);
941 void (*async_run_func
) (int, void *, void *, void *);
943 /* Splay tree containing information about mapped memory regions. */
944 struct splay_tree_s mem_map
;
946 /* Mutex for the mutable data. */
949 /* Current state of the device. OpenACC allows to move from INITIALIZED state
950 back to UNINITIALIZED state. OpenMP allows only to move from INITIALIZED
951 to FINALIZED state (at program shutdown). */
952 enum gomp_device_state state
;
954 /* OpenACC-specific data and functions. */
955 /* This is mutable because of its mutable data_environ and target_data
957 acc_dispatch_t openacc
;
960 /* Kind of the pragma, for which gomp_map_vars () is called. */
961 enum gomp_map_vars_kind
963 GOMP_MAP_VARS_OPENACC
,
964 GOMP_MAP_VARS_TARGET
,
966 GOMP_MAP_VARS_ENTER_DATA
969 extern void gomp_acc_insert_pointer (size_t, void **, size_t *, void *);
970 extern void gomp_acc_remove_pointer (void *, bool, int, int);
972 extern struct target_mem_desc
*gomp_map_vars (struct gomp_device_descr
*,
973 size_t, void **, void **,
974 size_t *, void *, bool,
975 enum gomp_map_vars_kind
);
976 extern void gomp_copy_from_async (struct target_mem_desc
*);
977 extern void gomp_unmap_vars (struct target_mem_desc
*, bool);
978 extern void gomp_init_device (struct gomp_device_descr
*);
979 extern void gomp_free_memmap (struct splay_tree_s
*);
980 extern void gomp_unload_device (struct gomp_device_descr
*);
984 extern void gomp_init_work_share (struct gomp_work_share
*, bool, unsigned);
985 extern void gomp_fini_work_share (struct gomp_work_share
*);
986 extern bool gomp_work_share_start (bool);
987 extern void gomp_work_share_end (void);
988 extern bool gomp_work_share_end_cancel (void);
989 extern void gomp_work_share_end_nowait (void);
992 gomp_work_share_init_done (void)
994 struct gomp_thread
*thr
= gomp_thread ();
995 if (__builtin_expect (thr
->ts
.last_work_share
!= NULL
, 1))
996 gomp_ptrlock_set (&thr
->ts
.last_work_share
->next_ws
, thr
->ts
.work_share
);
999 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1000 # pragma GCC visibility pop
1003 /* Now that we're back to default visibility, include the globals. */
1004 #include "libgomp_g.h"
1006 /* Include omp.h by parts. */
1007 #include "omp-lock.h"
1008 #define _LIBGOMP_OMP_LOCK_DEFINED 1
1011 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \
1012 || !defined (HAVE_ATTRIBUTE_ALIAS) \
1013 || !defined (HAVE_AS_SYMVER_DIRECTIVE) \
1015 || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT)
1016 # undef LIBGOMP_GNU_SYMBOL_VERSIONING
1019 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING
1020 extern void gomp_init_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1021 extern void gomp_destroy_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1022 extern void gomp_set_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1023 extern void gomp_unset_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1024 extern int gomp_test_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1025 extern void gomp_init_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1026 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1027 extern void gomp_set_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1028 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1029 extern int gomp_test_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1031 extern void gomp_init_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1032 extern void gomp_destroy_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1033 extern void gomp_set_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1034 extern void gomp_unset_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1035 extern int gomp_test_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1036 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1037 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1038 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1039 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1040 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1042 # define strong_alias(fn, al) \
1043 extern __typeof (fn) al __attribute__ ((alias (#fn)));
1044 # define omp_lock_symver(fn) \
1045 __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
1046 __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
1048 # define gomp_init_lock_30 omp_init_lock
1049 # define gomp_destroy_lock_30 omp_destroy_lock
1050 # define gomp_set_lock_30 omp_set_lock
1051 # define gomp_unset_lock_30 omp_unset_lock
1052 # define gomp_test_lock_30 omp_test_lock
1053 # define gomp_init_nest_lock_30 omp_init_nest_lock
1054 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock
1055 # define gomp_set_nest_lock_30 omp_set_nest_lock
1056 # define gomp_unset_nest_lock_30 omp_unset_nest_lock
1057 # define gomp_test_nest_lock_30 omp_test_nest_lock
1060 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1061 # define attribute_hidden __attribute__ ((visibility ("hidden")))
1063 # define attribute_hidden
1066 #ifdef HAVE_ATTRIBUTE_ALIAS
1067 # define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__)
1068 # define ialias_str1(x) ialias_str2(x)
1069 # define ialias_str2(x) #x
1070 # define ialias(fn) \
1071 extern __typeof (fn) gomp_ialias_##fn \
1072 __attribute__ ((alias (#fn))) attribute_hidden;
1073 # define ialias_redirect(fn) \
1074 extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
1075 # define ialias_call(fn) gomp_ialias_ ## fn
1078 # define ialias_redirect(fn)
1079 # define ialias_call(fn) fn
1082 /* Helper function for priority_node_to_task() and
1083 task_to_priority_node().
1085 Return the offset from a task to its priority_node entry. The
1086 priority_node entry is has a type of TYPE. */
1088 static inline size_t
1089 priority_queue_offset (enum priority_queue_type type
)
1091 return offsetof (struct gomp_task
, pnode
[(int) type
]);
1094 /* Return the task associated with a priority NODE of type TYPE. */
1096 static inline struct gomp_task
*
1097 priority_node_to_task (enum priority_queue_type type
,
1098 struct priority_node
*node
)
1100 return (struct gomp_task
*) ((char *) node
- priority_queue_offset (type
));
1103 /* Return the priority node of type TYPE for a given TASK. */
1105 static inline struct priority_node
*
1106 task_to_priority_node (enum priority_queue_type type
,
1107 struct gomp_task
*task
)
1109 return (struct priority_node
*) ((char *) task
1110 + priority_queue_offset (type
));
1112 #endif /* LIBGOMP_H */