re PR target/90530 (Invalid SUBREG insn generated by reload)
[official-gcc.git] / libgomp / libgomp.h
blob9f433160ab5921755118b36fb9c8e79d4e03c1dc
1 /* Copyright (C) 2005-2019 Free Software Foundation, Inc.
2 Contributed by Richard Henderson <rth@redhat.com>.
4 This file is part of the GNU Offloading and Multi Processing Library
5 (libgomp).
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 more details.
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
26 /* This file contains data types and function declarations that are not
27 part of the official OpenACC or OpenMP user interfaces. There are
28 declarations in here that are part of the GNU Offloading and Multi
29 Processing ABI, in that the compiler is required to know about them
30 and use them.
32 The convention is that the all caps prefix "GOMP" is used group items
33 that are part of the external ABI, and the lower case prefix "gomp"
34 is used group items that are completely private to the library. */
36 #ifndef LIBGOMP_H
37 #define LIBGOMP_H 1
39 #ifndef _LIBGOMP_CHECKING_
40 /* Define to 1 to perform internal sanity checks. */
41 #define _LIBGOMP_CHECKING_ 0
42 #endif
44 #include "config.h"
45 #include "gstdint.h"
46 #include "libgomp-plugin.h"
47 #include "gomp-constants.h"
49 #ifdef HAVE_PTHREAD_H
50 #include <pthread.h>
51 #endif
52 #include <stdbool.h>
53 #include <stdlib.h>
54 #include <stdarg.h>
56 /* Needed for memset in priority_queue.c. */
57 #if _LIBGOMP_CHECKING_
58 # ifdef STRING_WITH_STRINGS
59 # include <string.h>
60 # include <strings.h>
61 # else
62 # ifdef HAVE_STRING_H
63 # include <string.h>
64 # else
65 # ifdef HAVE_STRINGS_H
66 # include <strings.h>
67 # endif
68 # endif
69 # endif
70 #endif
72 #ifdef HAVE_ATTRIBUTE_VISIBILITY
73 # pragma GCC visibility push(hidden)
74 #endif
76 /* If we were a C++ library, we'd get this from <std/atomic>. */
77 enum memmodel
79 MEMMODEL_RELAXED = 0,
80 MEMMODEL_CONSUME = 1,
81 MEMMODEL_ACQUIRE = 2,
82 MEMMODEL_RELEASE = 3,
83 MEMMODEL_ACQ_REL = 4,
84 MEMMODEL_SEQ_CST = 5
87 /* alloc.c */
89 #if defined(HAVE_ALIGNED_ALLOC) \
90 || defined(HAVE__ALIGNED_MALLOC) \
91 || defined(HAVE_POSIX_MEMALIGN) \
92 || defined(HAVE_MEMALIGN)
93 /* Defined if gomp_aligned_alloc doesn't use fallback version
94 and free can be used instead of gomp_aligned_free. */
95 #define GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC 1
96 #endif
98 extern void *gomp_malloc (size_t) __attribute__((malloc));
99 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc));
100 extern void *gomp_realloc (void *, size_t);
101 extern void *gomp_aligned_alloc (size_t, size_t)
102 __attribute__((malloc, alloc_size (2)));
103 extern void gomp_aligned_free (void *);
105 /* Avoid conflicting prototypes of alloca() in system headers by using
106 GCC's builtin alloca(). */
107 #define gomp_alloca(x) __builtin_alloca(x)
109 /* error.c */
111 extern void gomp_vdebug (int, const char *, va_list);
112 extern void gomp_debug (int, const char *, ...)
113 __attribute__ ((format (printf, 2, 3)));
114 #define gomp_vdebug(KIND, FMT, VALIST) \
115 do { \
116 if (__builtin_expect (gomp_debug_var, 0)) \
117 (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
118 } while (0)
119 #define gomp_debug(KIND, ...) \
120 do { \
121 if (__builtin_expect (gomp_debug_var, 0)) \
122 (gomp_debug) ((KIND), __VA_ARGS__); \
123 } while (0)
124 extern void gomp_verror (const char *, va_list);
125 extern void gomp_error (const char *, ...)
126 __attribute__ ((format (printf, 1, 2)));
127 extern void gomp_vfatal (const char *, va_list)
128 __attribute__ ((noreturn));
129 extern void gomp_fatal (const char *, ...)
130 __attribute__ ((noreturn, format (printf, 1, 2)));
132 struct gomp_task;
133 struct gomp_taskgroup;
134 struct htab;
136 #include "priority_queue.h"
137 #include "sem.h"
138 #include "mutex.h"
139 #include "bar.h"
140 #include "simple-bar.h"
141 #include "ptrlock.h"
144 /* This structure contains the data to control one work-sharing construct,
145 either a LOOP (FOR/DO) or a SECTIONS. */
147 enum gomp_schedule_type
149 GFS_RUNTIME,
150 GFS_STATIC,
151 GFS_DYNAMIC,
152 GFS_GUIDED,
153 GFS_AUTO,
154 GFS_MONOTONIC = 0x80000000U
157 struct gomp_doacross_work_share
159 union {
160 /* chunk_size copy, as ws->chunk_size is multiplied by incr for
161 GFS_DYNAMIC. */
162 long chunk_size;
163 /* Likewise, but for ull implementation. */
164 unsigned long long chunk_size_ull;
165 /* For schedule(static,0) this is the number
166 of iterations assigned to the last thread, i.e. number of
167 iterations / number of threads. */
168 long q;
169 /* Likewise, but for ull implementation. */
170 unsigned long long q_ull;
172 /* Size of each array entry (padded to cache line size). */
173 unsigned long elt_sz;
174 /* Number of dimensions in sink vectors. */
175 unsigned int ncounts;
176 /* True if the iterations can be flattened. */
177 bool flattened;
178 /* Actual array (of elt_sz sized units), aligned to cache line size.
179 This is indexed by team_id for GFS_STATIC and outermost iteration
180 / chunk_size for other schedules. */
181 unsigned char *array;
182 /* These two are only used for schedule(static,0). */
183 /* This one is number of iterations % number of threads. */
184 long t;
185 union {
186 /* And this one is cached t * (q + 1). */
187 long boundary;
188 /* Likewise, but for the ull implementation. */
189 unsigned long long boundary_ull;
191 /* Pointer to extra memory if needed for lastprivate(conditional). */
192 void *extra;
193 /* Array of shift counts for each dimension if they can be flattened. */
194 unsigned int shift_counts[];
197 struct gomp_work_share
199 /* This member records the SCHEDULE clause to be used for this construct.
200 The user specification of "runtime" will already have been resolved.
201 If this is a SECTIONS construct, this value will always be DYNAMIC. */
202 enum gomp_schedule_type sched;
204 int mode;
206 union {
207 struct {
208 /* This is the chunk_size argument to the SCHEDULE clause. */
209 long chunk_size;
211 /* This is the iteration end point. If this is a SECTIONS construct,
212 this is the number of contained sections. */
213 long end;
215 /* This is the iteration step. If this is a SECTIONS construct, this
216 is always 1. */
217 long incr;
220 struct {
221 /* The same as above, but for the unsigned long long loop variants. */
222 unsigned long long chunk_size_ull;
223 unsigned long long end_ull;
224 unsigned long long incr_ull;
228 union {
229 /* This is a circular queue that details which threads will be allowed
230 into the ordered region and in which order. When a thread allocates
231 iterations on which it is going to work, it also registers itself at
232 the end of the array. When a thread reaches the ordered region, it
233 checks to see if it is the one at the head of the queue. If not, it
234 blocks on its RELEASE semaphore. */
235 unsigned *ordered_team_ids;
237 /* This is a pointer to DOACROSS work share data. */
238 struct gomp_doacross_work_share *doacross;
241 /* This is the number of threads that have registered themselves in
242 the circular queue ordered_team_ids. */
243 unsigned ordered_num_used;
245 /* This is the team_id of the currently acknowledged owner of the ordered
246 section, or -1u if the ordered section has not been acknowledged by
247 any thread. This is distinguished from the thread that is *allowed*
248 to take the section next. */
249 unsigned ordered_owner;
251 /* This is the index into the circular queue ordered_team_ids of the
252 current thread that's allowed into the ordered reason. */
253 unsigned ordered_cur;
255 /* This is a chain of allocated gomp_work_share blocks, valid only
256 in the first gomp_work_share struct in the block. */
257 struct gomp_work_share *next_alloc;
259 /* The above fields are written once during workshare initialization,
260 or related to ordered worksharing. Make sure the following fields
261 are in a different cache line. */
263 /* This lock protects the update of the following members. */
264 gomp_mutex_t lock __attribute__((aligned (64)));
266 /* This is the count of the number of threads that have exited the work
267 share construct. If the construct was marked nowait, they have moved on
268 to other work; otherwise they're blocked on a barrier. The last member
269 of the team to exit the work share construct must deallocate it. */
270 unsigned threads_completed;
272 union {
273 /* This is the next iteration value to be allocated. In the case of
274 GFS_STATIC loops, this the iteration start point and never changes. */
275 long next;
277 /* The same, but with unsigned long long type. */
278 unsigned long long next_ull;
280 /* This is the returned data structure for SINGLE COPYPRIVATE. */
281 void *copyprivate;
284 union {
285 /* Link to gomp_work_share struct for next work sharing construct
286 encountered after this one. */
287 gomp_ptrlock_t next_ws;
289 /* gomp_work_share structs are chained in the free work share cache
290 through this. */
291 struct gomp_work_share *next_free;
294 /* Task reductions for this work-sharing construct. */
295 uintptr_t *task_reductions;
297 /* If only few threads are in the team, ordered_team_ids can point
298 to this array which fills the padding at the end of this struct. */
299 unsigned inline_ordered_team_ids[0];
302 /* This structure contains all of the thread-local data associated with
303 a thread team. This is the data that must be saved when a thread
304 encounters a nested PARALLEL construct. */
306 struct gomp_team_state
308 /* This is the team of which the thread is currently a member. */
309 struct gomp_team *team;
311 /* This is the work share construct which this thread is currently
312 processing. Recall that with NOWAIT, not all threads may be
313 processing the same construct. */
314 struct gomp_work_share *work_share;
316 /* This is the previous work share construct or NULL if there wasn't any.
317 When all threads are done with the current work sharing construct,
318 the previous one can be freed. The current one can't, as its
319 next_ws field is used. */
320 struct gomp_work_share *last_work_share;
322 /* This is the ID of this thread within the team. This value is
323 guaranteed to be between 0 and N-1, where N is the number of
324 threads in the team. */
325 unsigned team_id;
327 /* Nesting level. */
328 unsigned level;
330 /* Active nesting level. Only active parallel regions are counted. */
331 unsigned active_level;
333 /* Place-partition-var, offset and length into gomp_places_list array. */
334 unsigned place_partition_off;
335 unsigned place_partition_len;
337 #ifdef HAVE_SYNC_BUILTINS
338 /* Number of single stmts encountered. */
339 unsigned long single_count;
340 #endif
342 /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the
343 trip number through the loop. So first time a particular loop
344 is encountered this number is 0, the second time through the loop
345 is 1, etc. This is unused when the compiler knows in advance that
346 the loop is statically scheduled. */
347 unsigned long static_trip;
350 struct target_mem_desc;
352 /* These are the OpenMP 4.0 Internal Control Variables described in
353 section 2.3.1. Those described as having one copy per task are
354 stored within the structure; those described as having one copy
355 for the whole program are (naturally) global variables. */
357 struct gomp_task_icv
359 unsigned long nthreads_var;
360 enum gomp_schedule_type run_sched_var;
361 int run_sched_chunk_size;
362 int default_device_var;
363 unsigned int thread_limit_var;
364 bool dyn_var;
365 bool nest_var;
366 char bind_var;
367 /* Internal ICV. */
368 struct target_mem_desc *target_data;
371 extern struct gomp_task_icv gomp_global_icv;
372 #ifndef HAVE_SYNC_BUILTINS
373 extern gomp_mutex_t gomp_managed_threads_lock;
374 #endif
375 extern unsigned long gomp_max_active_levels_var;
376 extern bool gomp_cancel_var;
377 extern int gomp_max_task_priority_var;
378 extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var;
379 extern unsigned long gomp_available_cpus, gomp_managed_threads;
380 extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len;
381 extern char *gomp_bind_var_list;
382 extern unsigned long gomp_bind_var_list_len;
383 extern void **gomp_places_list;
384 extern unsigned long gomp_places_list_len;
385 extern unsigned int gomp_num_teams_var;
386 extern int gomp_debug_var;
387 extern bool gomp_display_affinity_var;
388 extern char *gomp_affinity_format_var;
389 extern size_t gomp_affinity_format_len;
390 extern int goacc_device_num;
391 extern char *goacc_device_type;
392 extern int goacc_default_dims[GOMP_DIM_MAX];
394 enum gomp_task_kind
396 /* Implicit task. */
397 GOMP_TASK_IMPLICIT,
398 /* Undeferred task. */
399 GOMP_TASK_UNDEFERRED,
400 /* Task created by GOMP_task and waiting to be run. */
401 GOMP_TASK_WAITING,
402 /* Task currently executing or scheduled and about to execute. */
403 GOMP_TASK_TIED,
404 /* Used for target tasks that have vars mapped and async run started,
405 but not yet completed. Once that completes, they will be readded
406 into the queues as GOMP_TASK_WAITING in order to perform the var
407 unmapping. */
408 GOMP_TASK_ASYNC_RUNNING
411 struct gomp_task_depend_entry
413 /* Address of dependency. */
414 void *addr;
415 struct gomp_task_depend_entry *next;
416 struct gomp_task_depend_entry *prev;
417 /* Task that provides the dependency in ADDR. */
418 struct gomp_task *task;
419 /* Depend entry is of type "IN". */
420 bool is_in;
421 bool redundant;
422 bool redundant_out;
425 struct gomp_dependers_vec
427 size_t n_elem;
428 size_t allocated;
429 struct gomp_task *elem[];
432 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies. */
434 struct gomp_taskwait
436 bool in_taskwait;
437 bool in_depend_wait;
438 /* Number of tasks we are waiting for. */
439 size_t n_depend;
440 gomp_sem_t taskwait_sem;
443 /* This structure describes a "task" to be run by a thread. */
445 struct gomp_task
447 /* Parent of this task. */
448 struct gomp_task *parent;
449 /* Children of this task. */
450 struct priority_queue children_queue;
451 /* Taskgroup this task belongs in. */
452 struct gomp_taskgroup *taskgroup;
453 /* Tasks that depend on this task. */
454 struct gomp_dependers_vec *dependers;
455 struct htab *depend_hash;
456 struct gomp_taskwait *taskwait;
457 /* Number of items in DEPEND. */
458 size_t depend_count;
459 /* Number of tasks this task depends on. Once this counter reaches
460 0, we have no unsatisfied dependencies, and this task can be put
461 into the various queues to be scheduled. */
462 size_t num_dependees;
464 /* Priority of this task. */
465 int priority;
466 /* The priority node for this task in each of the different queues.
467 We put this here to avoid allocating space for each priority
468 node. Then we play offsetof() games to convert between pnode[]
469 entries and the gomp_task in which they reside. */
470 struct priority_node pnode[3];
472 struct gomp_task_icv icv;
473 void (*fn) (void *);
474 void *fn_data;
475 enum gomp_task_kind kind;
476 bool in_tied_task;
477 bool final_task;
478 bool copy_ctors_done;
479 /* Set for undeferred tasks with unsatisfied dependencies which
480 block further execution of their parent until the dependencies
481 are satisfied. */
482 bool parent_depends_on;
483 /* Dependencies provided and/or needed for this task. DEPEND_COUNT
484 is the number of items available. */
485 struct gomp_task_depend_entry depend[];
488 /* This structure describes a single #pragma omp taskgroup. */
490 struct gomp_taskgroup
492 struct gomp_taskgroup *prev;
493 /* Queue of tasks that belong in this taskgroup. */
494 struct priority_queue taskgroup_queue;
495 uintptr_t *reductions;
496 bool in_taskgroup_wait;
497 bool cancelled;
498 bool workshare;
499 gomp_sem_t taskgroup_sem;
500 size_t num_children;
503 /* Various state of OpenMP async offloading tasks. */
504 enum gomp_target_task_state
506 GOMP_TARGET_TASK_DATA,
507 GOMP_TARGET_TASK_BEFORE_MAP,
508 GOMP_TARGET_TASK_FALLBACK,
509 GOMP_TARGET_TASK_READY_TO_RUN,
510 GOMP_TARGET_TASK_RUNNING,
511 GOMP_TARGET_TASK_FINISHED
514 /* This structure describes a target task. */
516 struct gomp_target_task
518 struct gomp_device_descr *devicep;
519 void (*fn) (void *);
520 size_t mapnum;
521 size_t *sizes;
522 unsigned short *kinds;
523 unsigned int flags;
524 enum gomp_target_task_state state;
525 struct target_mem_desc *tgt;
526 struct gomp_task *task;
527 struct gomp_team *team;
528 /* Device-specific target arguments. */
529 void **args;
530 void *hostaddrs[];
533 /* This structure describes a "team" of threads. These are the threads
534 that are spawned by a PARALLEL constructs, as well as the work sharing
535 constructs that the team encounters. */
537 struct gomp_team
539 /* This is the number of threads in the current team. */
540 unsigned nthreads;
542 /* This is number of gomp_work_share structs that have been allocated
543 as a block last time. */
544 unsigned work_share_chunk;
546 /* This is the saved team state that applied to a master thread before
547 the current thread was created. */
548 struct gomp_team_state prev_ts;
550 /* This semaphore should be used by the master thread instead of its
551 "native" semaphore in the thread structure. Required for nested
552 parallels, as the master is a member of two teams. */
553 gomp_sem_t master_release;
555 /* This points to an array with pointers to the release semaphore
556 of the threads in the team. */
557 gomp_sem_t **ordered_release;
559 /* List of work shares on which gomp_fini_work_share hasn't been
560 called yet. If the team hasn't been cancelled, this should be
561 equal to each thr->ts.work_share, but otherwise it can be a possibly
562 long list of workshares. */
563 struct gomp_work_share *work_shares_to_free;
565 /* List of gomp_work_share structs chained through next_free fields.
566 This is populated and taken off only by the first thread in the
567 team encountering a new work sharing construct, in a critical
568 section. */
569 struct gomp_work_share *work_share_list_alloc;
571 /* List of gomp_work_share structs freed by free_work_share. New
572 entries are atomically added to the start of the list, and
573 alloc_work_share can safely only move all but the first entry
574 to work_share_list alloc, as free_work_share can happen concurrently
575 with alloc_work_share. */
576 struct gomp_work_share *work_share_list_free;
578 #ifdef HAVE_SYNC_BUILTINS
579 /* Number of simple single regions encountered by threads in this
580 team. */
581 unsigned long single_count;
582 #else
583 /* Mutex protecting addition of workshares to work_share_list_free. */
584 gomp_mutex_t work_share_list_free_lock;
585 #endif
587 /* This barrier is used for most synchronization of the team. */
588 gomp_barrier_t barrier;
590 /* Initial work shares, to avoid allocating any gomp_work_share
591 structs in the common case. */
592 struct gomp_work_share work_shares[8];
594 gomp_mutex_t task_lock;
595 /* Scheduled tasks. */
596 struct priority_queue task_queue;
597 /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */
598 unsigned int task_count;
599 /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */
600 unsigned int task_queued_count;
601 /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
602 directly in gomp_barrier_handle_tasks; tasks spawned
603 from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
604 that is called from a task run from gomp_barrier_handle_tasks.
605 task_running_count should be always <= team->nthreads,
606 and if current task isn't in_tied_task, then it will be
607 even < team->nthreads. */
608 unsigned int task_running_count;
609 int work_share_cancelled;
610 int team_cancelled;
612 /* This array contains structures for implicit tasks. */
613 struct gomp_task implicit_task[];
616 /* This structure contains all data that is private to libgomp and is
617 allocated per thread. */
619 struct gomp_thread
621 /* This is the function that the thread should run upon launch. */
622 void (*fn) (void *data);
623 void *data;
625 /* This is the current team state for this thread. The ts.team member
626 is NULL only if the thread is idle. */
627 struct gomp_team_state ts;
629 /* This is the task that the thread is currently executing. */
630 struct gomp_task *task;
632 /* This semaphore is used for ordered loops. */
633 gomp_sem_t release;
635 /* Place this thread is bound to plus one, or zero if not bound
636 to any place. */
637 unsigned int place;
639 /* User pthread thread pool */
640 struct gomp_thread_pool *thread_pool;
642 #if defined(LIBGOMP_USE_PTHREADS) \
643 && (!defined(HAVE_TLS) \
644 || !defined(__GLIBC__) \
645 || !defined(USING_INITIAL_EXEC_TLS))
646 /* pthread_t of the thread containing this gomp_thread.
647 On Linux when using initial-exec TLS,
648 (typeof (pthread_t)) gomp_thread () - pthread_self ()
649 is constant in all threads, so we can optimize and not
650 store it. */
651 #define GOMP_NEEDS_THREAD_HANDLE 1
652 pthread_t handle;
653 #endif
657 struct gomp_thread_pool
659 /* This array manages threads spawned from the top level, which will
660 return to the idle loop once the current PARALLEL construct ends. */
661 struct gomp_thread **threads;
662 unsigned threads_size;
663 unsigned threads_used;
664 /* The last team is used for non-nested teams to delay their destruction to
665 make sure all the threads in the team move on to the pool's barrier before
666 the team's barrier is destroyed. */
667 struct gomp_team *last_team;
668 /* Number of threads running in this contention group. */
669 unsigned long threads_busy;
671 /* This barrier holds and releases threads waiting in thread pools. */
672 gomp_simple_barrier_t threads_dock;
675 enum gomp_cancel_kind
677 GOMP_CANCEL_PARALLEL = 1,
678 GOMP_CANCEL_LOOP = 2,
679 GOMP_CANCEL_FOR = GOMP_CANCEL_LOOP,
680 GOMP_CANCEL_DO = GOMP_CANCEL_LOOP,
681 GOMP_CANCEL_SECTIONS = 4,
682 GOMP_CANCEL_TASKGROUP = 8
685 /* ... and here is that TLS data. */
687 #if defined __nvptx__
688 extern struct gomp_thread *nvptx_thrs __attribute__((shared));
689 static inline struct gomp_thread *gomp_thread (void)
691 int tid;
692 asm ("mov.u32 %0, %%tid.y;" : "=r" (tid));
693 return nvptx_thrs + tid;
695 #elif defined HAVE_TLS || defined USE_EMUTLS
696 extern __thread struct gomp_thread gomp_tls_data;
697 static inline struct gomp_thread *gomp_thread (void)
699 return &gomp_tls_data;
701 #else
702 extern pthread_key_t gomp_tls_key;
703 static inline struct gomp_thread *gomp_thread (void)
705 return pthread_getspecific (gomp_tls_key);
707 #endif
709 extern struct gomp_task_icv *gomp_new_icv (void);
711 /* Here's how to access the current copy of the ICVs. */
713 static inline struct gomp_task_icv *gomp_icv (bool write)
715 struct gomp_task *task = gomp_thread ()->task;
716 if (task)
717 return &task->icv;
718 else if (write)
719 return gomp_new_icv ();
720 else
721 return &gomp_global_icv;
724 #ifdef LIBGOMP_USE_PTHREADS
725 /* The attributes to be used during thread creation. */
726 extern pthread_attr_t gomp_thread_attr;
728 extern pthread_key_t gomp_thread_destructor;
729 #endif
731 /* Function prototypes. */
733 /* affinity.c */
735 extern void gomp_init_affinity (void);
736 #ifdef LIBGOMP_USE_PTHREADS
737 extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int);
738 #endif
739 extern void **gomp_affinity_alloc (unsigned long, bool);
740 extern void gomp_affinity_init_place (void *);
741 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
742 long, bool);
743 extern bool gomp_affinity_remove_cpu (void *, unsigned long);
744 extern bool gomp_affinity_copy_place (void *, void *, long);
745 extern bool gomp_affinity_same_place (void *, void *);
746 extern bool gomp_affinity_finalize_place_list (bool);
747 extern bool gomp_affinity_init_level (int, unsigned long, bool);
748 extern void gomp_affinity_print_place (void *);
749 extern void gomp_get_place_proc_ids_8 (int, int64_t *);
750 extern void gomp_display_affinity_place (char *, size_t, size_t *, int);
752 /* affinity-fmt.c */
754 extern void gomp_print_string (const char *str, size_t len);
755 extern void gomp_set_affinity_format (const char *, size_t);
756 extern void gomp_display_string (char *, size_t, size_t *, const char *,
757 size_t);
758 #ifdef LIBGOMP_USE_PTHREADS
759 typedef pthread_t gomp_thread_handle;
760 #else
761 typedef struct {} gomp_thread_handle;
762 #endif
763 extern size_t gomp_display_affinity (char *, size_t, const char *,
764 gomp_thread_handle,
765 struct gomp_team_state *, unsigned int);
766 extern void gomp_display_affinity_thread (gomp_thread_handle,
767 struct gomp_team_state *,
768 unsigned int) __attribute__((cold));
770 /* iter.c */
772 extern int gomp_iter_static_next (long *, long *);
773 extern bool gomp_iter_dynamic_next_locked (long *, long *);
774 extern bool gomp_iter_guided_next_locked (long *, long *);
776 #ifdef HAVE_SYNC_BUILTINS
777 extern bool gomp_iter_dynamic_next (long *, long *);
778 extern bool gomp_iter_guided_next (long *, long *);
779 #endif
781 /* iter_ull.c */
783 extern int gomp_iter_ull_static_next (unsigned long long *,
784 unsigned long long *);
785 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *,
786 unsigned long long *);
787 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *,
788 unsigned long long *);
790 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
791 extern bool gomp_iter_ull_dynamic_next (unsigned long long *,
792 unsigned long long *);
793 extern bool gomp_iter_ull_guided_next (unsigned long long *,
794 unsigned long long *);
795 #endif
797 /* ordered.c */
799 extern void gomp_ordered_first (void);
800 extern void gomp_ordered_last (void);
801 extern void gomp_ordered_next (void);
802 extern void gomp_ordered_static_init (void);
803 extern void gomp_ordered_static_next (void);
804 extern void gomp_ordered_sync (void);
805 extern void gomp_doacross_init (unsigned, long *, long, size_t);
806 extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
807 unsigned long long, size_t);
809 /* parallel.c */
811 extern unsigned gomp_resolve_num_threads (unsigned, unsigned);
813 /* proc.c (in config/) */
815 extern void gomp_init_num_threads (void);
816 extern unsigned gomp_dynamic_max_threads (void);
818 /* task.c */
820 extern void gomp_init_task (struct gomp_task *, struct gomp_task *,
821 struct gomp_task_icv *);
822 extern void gomp_end_task (void);
823 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t);
824 extern void gomp_task_maybe_wait_for_dependencies (void **);
825 extern bool gomp_create_target_task (struct gomp_device_descr *,
826 void (*) (void *), size_t, void **,
827 size_t *, unsigned short *, unsigned int,
828 void **, void **,
829 enum gomp_target_task_state);
830 extern struct gomp_taskgroup *gomp_parallel_reduction_register (uintptr_t *,
831 unsigned);
832 extern void gomp_workshare_taskgroup_start (void);
833 extern void gomp_workshare_task_reduction_register (uintptr_t *, uintptr_t *);
835 static void inline
836 gomp_finish_task (struct gomp_task *task)
838 if (__builtin_expect (task->depend_hash != NULL, 0))
839 free (task->depend_hash);
842 /* team.c */
844 extern struct gomp_team *gomp_new_team (unsigned);
845 extern void gomp_team_start (void (*) (void *), void *, unsigned,
846 unsigned, struct gomp_team *,
847 struct gomp_taskgroup *);
848 extern void gomp_team_end (void);
849 extern void gomp_free_thread (void *);
850 extern int gomp_pause_host (void);
852 /* target.c */
854 extern void gomp_init_targets_once (void);
855 extern int gomp_get_num_devices (void);
856 extern bool gomp_target_task_fn (void *);
858 /* Splay tree definitions. */
859 typedef struct splay_tree_node_s *splay_tree_node;
860 typedef struct splay_tree_s *splay_tree;
861 typedef struct splay_tree_key_s *splay_tree_key;
863 struct target_var_desc {
864 /* Splay key. */
865 splay_tree_key key;
866 /* True if data should be copied from device to host at the end. */
867 bool copy_from;
868 /* True if data always should be copied from device to host at the end. */
869 bool always_copy_from;
870 /* Relative offset against key host_start. */
871 uintptr_t offset;
872 /* Actual length. */
873 uintptr_t length;
876 struct target_mem_desc {
877 /* Reference count. */
878 uintptr_t refcount;
879 /* All the splay nodes allocated together. */
880 splay_tree_node array;
881 /* Start of the target region. */
882 uintptr_t tgt_start;
883 /* End of the targer region. */
884 uintptr_t tgt_end;
885 /* Handle to free. */
886 void *to_free;
887 /* Previous target_mem_desc. */
888 struct target_mem_desc *prev;
889 /* Number of items in following list. */
890 size_t list_count;
892 /* Corresponding target device descriptor. */
893 struct gomp_device_descr *device_descr;
895 /* List of target items to remove (or decrease refcount)
896 at the end of region. */
897 struct target_var_desc list[];
900 /* Special value for refcount - infinity. */
901 #define REFCOUNT_INFINITY (~(uintptr_t) 0)
902 /* Special value for refcount - tgt_offset contains target address of the
903 artificial pointer to "omp declare target link" object. */
904 #define REFCOUNT_LINK (~(uintptr_t) 1)
906 struct splay_tree_key_s {
907 /* Address of the host object. */
908 uintptr_t host_start;
909 /* Address immediately after the host object. */
910 uintptr_t host_end;
911 /* Descriptor of the target memory. */
912 struct target_mem_desc *tgt;
913 /* Offset from tgt->tgt_start to the start of the target object. */
914 uintptr_t tgt_offset;
915 /* Reference count. */
916 uintptr_t refcount;
917 /* Dynamic reference count. */
918 uintptr_t dynamic_refcount;
919 /* Pointer to the original mapping of "omp declare target link" object. */
920 splay_tree_key link_key;
923 /* The comparison function. */
925 static inline int
926 splay_compare (splay_tree_key x, splay_tree_key y)
928 if (x->host_start == x->host_end
929 && y->host_start == y->host_end)
930 return 0;
931 if (x->host_end <= y->host_start)
932 return -1;
933 if (x->host_start >= y->host_end)
934 return 1;
935 return 0;
938 #include "splay-tree.h"
940 typedef struct acc_dispatch_t
942 /* This is a linked list of data mapped using the
943 acc_map_data/acc_unmap_data or "acc enter data"/"acc exit data" pragmas.
944 Unlike mapped_data in the goacc_thread struct, unmapping can
945 happen out-of-order with respect to mapping. */
946 /* This is guarded by the lock in the "outer" struct gomp_device_descr. */
947 struct target_mem_desc *data_environ;
949 /* Execute. */
950 __typeof (GOMP_OFFLOAD_openacc_exec) *exec_func;
952 /* Create/destroy TLS data. */
953 __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func;
954 __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data)
955 *destroy_thread_data_func;
957 struct {
958 /* Once created and put into the "active" list, asyncqueues are then never
959 destructed and removed from the "active" list, other than if the TODO
960 device is shut down. */
961 gomp_mutex_t lock;
962 int nasyncqueue;
963 struct goacc_asyncqueue **asyncqueue;
964 struct goacc_asyncqueue_list *active;
966 __typeof (GOMP_OFFLOAD_openacc_async_construct) *construct_func;
967 __typeof (GOMP_OFFLOAD_openacc_async_destruct) *destruct_func;
968 __typeof (GOMP_OFFLOAD_openacc_async_test) *test_func;
969 __typeof (GOMP_OFFLOAD_openacc_async_synchronize) *synchronize_func;
970 __typeof (GOMP_OFFLOAD_openacc_async_serialize) *serialize_func;
971 __typeof (GOMP_OFFLOAD_openacc_async_queue_callback) *queue_callback_func;
973 __typeof (GOMP_OFFLOAD_openacc_async_exec) *exec_func;
974 __typeof (GOMP_OFFLOAD_openacc_async_dev2host) *dev2host_func;
975 __typeof (GOMP_OFFLOAD_openacc_async_host2dev) *host2dev_func;
976 } async;
978 /* NVIDIA target specific routines. */
979 struct {
980 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device)
981 *get_current_device_func;
982 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context)
983 *get_current_context_func;
984 __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream) *get_stream_func;
985 __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream) *set_stream_func;
986 } cuda;
987 } acc_dispatch_t;
989 /* Various state of the accelerator device. */
990 enum gomp_device_state
992 GOMP_DEVICE_UNINITIALIZED,
993 GOMP_DEVICE_INITIALIZED,
994 GOMP_DEVICE_FINALIZED
997 /* This structure describes accelerator device.
998 It contains name of the corresponding libgomp plugin, function handlers for
999 interaction with the device, ID-number of the device, and information about
1000 mapped memory. */
1001 struct gomp_device_descr
1003 /* Immutable data, which is only set during initialization, and which is not
1004 guarded by the lock. */
1006 /* The name of the device. */
1007 const char *name;
1009 /* Capabilities of device (supports OpenACC, OpenMP). */
1010 unsigned int capabilities;
1012 /* This is the ID number of device among devices of the same type. */
1013 int target_id;
1015 /* This is the TYPE of device. */
1016 enum offload_target_type type;
1018 /* Function handlers. */
1019 __typeof (GOMP_OFFLOAD_get_name) *get_name_func;
1020 __typeof (GOMP_OFFLOAD_get_caps) *get_caps_func;
1021 __typeof (GOMP_OFFLOAD_get_type) *get_type_func;
1022 __typeof (GOMP_OFFLOAD_get_num_devices) *get_num_devices_func;
1023 __typeof (GOMP_OFFLOAD_init_device) *init_device_func;
1024 __typeof (GOMP_OFFLOAD_fini_device) *fini_device_func;
1025 __typeof (GOMP_OFFLOAD_version) *version_func;
1026 __typeof (GOMP_OFFLOAD_load_image) *load_image_func;
1027 __typeof (GOMP_OFFLOAD_unload_image) *unload_image_func;
1028 __typeof (GOMP_OFFLOAD_alloc) *alloc_func;
1029 __typeof (GOMP_OFFLOAD_free) *free_func;
1030 __typeof (GOMP_OFFLOAD_dev2host) *dev2host_func;
1031 __typeof (GOMP_OFFLOAD_host2dev) *host2dev_func;
1032 __typeof (GOMP_OFFLOAD_dev2dev) *dev2dev_func;
1033 __typeof (GOMP_OFFLOAD_can_run) *can_run_func;
1034 __typeof (GOMP_OFFLOAD_run) *run_func;
1035 __typeof (GOMP_OFFLOAD_async_run) *async_run_func;
1037 /* Splay tree containing information about mapped memory regions. */
1038 struct splay_tree_s mem_map;
1040 /* Mutex for the mutable data. */
1041 gomp_mutex_t lock;
1043 /* Current state of the device. OpenACC allows to move from INITIALIZED state
1044 back to UNINITIALIZED state. OpenMP allows only to move from INITIALIZED
1045 to FINALIZED state (at program shutdown). */
1046 enum gomp_device_state state;
1048 /* OpenACC-specific data and functions. */
1049 /* This is mutable because of its mutable data_environ and target_data
1050 members. */
1051 acc_dispatch_t openacc;
1054 /* Kind of the pragma, for which gomp_map_vars () is called. */
1055 enum gomp_map_vars_kind
1057 GOMP_MAP_VARS_OPENACC,
1058 GOMP_MAP_VARS_TARGET,
1059 GOMP_MAP_VARS_DATA,
1060 GOMP_MAP_VARS_ENTER_DATA
1063 extern void gomp_acc_insert_pointer (size_t, void **, size_t *, void *, int);
1064 extern void gomp_acc_remove_pointer (void *, size_t, bool, int, int, int);
1065 extern void gomp_acc_declare_allocate (bool, size_t, void **, size_t *,
1066 unsigned short *);
1067 struct gomp_coalesce_buf;
1068 extern void gomp_copy_host2dev (struct gomp_device_descr *,
1069 struct goacc_asyncqueue *, void *, const void *,
1070 size_t, struct gomp_coalesce_buf *);
1071 extern void gomp_copy_dev2host (struct gomp_device_descr *,
1072 struct goacc_asyncqueue *, void *, const void *,
1073 size_t);
1075 extern struct target_mem_desc *gomp_map_vars (struct gomp_device_descr *,
1076 size_t, void **, void **,
1077 size_t *, void *, bool,
1078 enum gomp_map_vars_kind);
1079 extern struct target_mem_desc *gomp_map_vars_async (struct gomp_device_descr *,
1080 struct goacc_asyncqueue *,
1081 size_t, void **, void **,
1082 size_t *, void *, bool,
1083 enum gomp_map_vars_kind);
1084 extern void gomp_unmap_tgt (struct target_mem_desc *);
1085 extern void gomp_unmap_vars (struct target_mem_desc *, bool);
1086 extern void gomp_unmap_vars_async (struct target_mem_desc *, bool,
1087 struct goacc_asyncqueue *);
1088 extern void gomp_init_device (struct gomp_device_descr *);
1089 extern bool gomp_fini_device (struct gomp_device_descr *);
1090 extern void gomp_free_memmap (struct splay_tree_s *);
1091 extern void gomp_unload_device (struct gomp_device_descr *);
1092 extern bool gomp_remove_var (struct gomp_device_descr *, splay_tree_key);
1094 /* work.c */
1096 extern void gomp_init_work_share (struct gomp_work_share *, size_t, unsigned);
1097 extern void gomp_fini_work_share (struct gomp_work_share *);
1098 extern bool gomp_work_share_start (size_t);
1099 extern void gomp_work_share_end (void);
1100 extern bool gomp_work_share_end_cancel (void);
1101 extern void gomp_work_share_end_nowait (void);
1103 static inline void
1104 gomp_work_share_init_done (void)
1106 struct gomp_thread *thr = gomp_thread ();
1107 if (__builtin_expect (thr->ts.last_work_share != NULL, 1))
1108 gomp_ptrlock_set (&thr->ts.last_work_share->next_ws, thr->ts.work_share);
1111 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1112 # pragma GCC visibility pop
1113 #endif
1115 /* Now that we're back to default visibility, include the globals. */
1116 #include "libgomp_g.h"
1118 /* Include omp.h by parts. */
1119 #include "omp-lock.h"
1120 #define _LIBGOMP_OMP_LOCK_DEFINED 1
1121 #include "omp.h.in"
1123 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \
1124 || !defined (HAVE_ATTRIBUTE_ALIAS) \
1125 || !defined (HAVE_AS_SYMVER_DIRECTIVE) \
1126 || !defined (PIC) \
1127 || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT)
1128 # undef LIBGOMP_GNU_SYMBOL_VERSIONING
1129 #endif
1131 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING
1132 extern void gomp_init_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1133 extern void gomp_destroy_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1134 extern void gomp_set_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1135 extern void gomp_unset_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1136 extern int gomp_test_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1137 extern void gomp_init_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1138 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1139 extern void gomp_set_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1140 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1141 extern int gomp_test_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1143 extern void gomp_init_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1144 extern void gomp_destroy_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1145 extern void gomp_set_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1146 extern void gomp_unset_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1147 extern int gomp_test_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1148 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1149 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1150 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1151 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1152 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1154 # define omp_lock_symver(fn) \
1155 __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
1156 __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
1157 #else
1158 # define gomp_init_lock_30 omp_init_lock
1159 # define gomp_destroy_lock_30 omp_destroy_lock
1160 # define gomp_set_lock_30 omp_set_lock
1161 # define gomp_unset_lock_30 omp_unset_lock
1162 # define gomp_test_lock_30 omp_test_lock
1163 # define gomp_init_nest_lock_30 omp_init_nest_lock
1164 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock
1165 # define gomp_set_nest_lock_30 omp_set_nest_lock
1166 # define gomp_unset_nest_lock_30 omp_unset_nest_lock
1167 # define gomp_test_nest_lock_30 omp_test_nest_lock
1168 #endif
1170 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1171 # define attribute_hidden __attribute__ ((visibility ("hidden")))
1172 #else
1173 # define attribute_hidden
1174 #endif
1176 #if __GNUC__ >= 9
1177 # define HAVE_ATTRIBUTE_COPY
1178 #endif
1180 #ifdef HAVE_ATTRIBUTE_COPY
1181 # define attribute_copy(arg) __attribute__ ((copy (arg)))
1182 #else
1183 # define attribute_copy(arg)
1184 #endif
1186 #ifdef HAVE_ATTRIBUTE_ALIAS
1187 # define strong_alias(fn, al) \
1188 extern __typeof (fn) al __attribute__ ((alias (#fn))) attribute_copy (fn);
1190 # define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__)
1191 # define ialias_str1(x) ialias_str2(x)
1192 # define ialias_str2(x) #x
1193 # define ialias(fn) \
1194 extern __typeof (fn) gomp_ialias_##fn \
1195 __attribute__ ((alias (#fn))) attribute_hidden attribute_copy (fn);
1196 # define ialias_redirect(fn) \
1197 extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
1198 # define ialias_call(fn) gomp_ialias_ ## fn
1199 #else
1200 # define ialias(fn)
1201 # define ialias_redirect(fn)
1202 # define ialias_call(fn) fn
1203 #endif
1205 /* Helper function for priority_node_to_task() and
1206 task_to_priority_node().
1208 Return the offset from a task to its priority_node entry. The
1209 priority_node entry is has a type of TYPE. */
1211 static inline size_t
1212 priority_queue_offset (enum priority_queue_type type)
1214 return offsetof (struct gomp_task, pnode[(int) type]);
1217 /* Return the task associated with a priority NODE of type TYPE. */
1219 static inline struct gomp_task *
1220 priority_node_to_task (enum priority_queue_type type,
1221 struct priority_node *node)
1223 return (struct gomp_task *) ((char *) node - priority_queue_offset (type));
1226 /* Return the priority node of type TYPE for a given TASK. */
1228 static inline struct priority_node *
1229 task_to_priority_node (enum priority_queue_type type,
1230 struct gomp_task *task)
1232 return (struct priority_node *) ((char *) task
1233 + priority_queue_offset (type));
1236 #ifdef LIBGOMP_USE_PTHREADS
1237 static inline gomp_thread_handle
1238 gomp_thread_self (void)
1240 return pthread_self ();
1243 static inline gomp_thread_handle
1244 gomp_thread_to_pthread_t (struct gomp_thread *thr)
1246 struct gomp_thread *this_thr = gomp_thread ();
1247 if (thr == this_thr)
1248 return pthread_self ();
1249 #ifdef GOMP_NEEDS_THREAD_HANDLE
1250 return thr->handle;
1251 #else
1252 /* On Linux with initial-exec TLS, the pthread_t of the thread containing
1253 thr can be computed from thr, this_thr and pthread_self (),
1254 as the distance between this_thr and pthread_self () is constant. */
1255 return pthread_self () + ((uintptr_t) thr - (uintptr_t) this_thr);
1256 #endif
1258 #else
1259 static inline gomp_thread_handle
1260 gomp_thread_self (void)
1262 return (gomp_thread_handle) {};
1265 static inline gomp_thread_handle
1266 gomp_thread_to_pthread_t (struct gomp_thread *thr)
1268 (void) thr;
1269 return gomp_thread_self ();
1271 #endif
1273 #endif /* LIBGOMP_H */