libgomp/libgomp.h

   1 /* Copyright (C) 2005-2014 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains data types and function declarations that are not
  27    part of the official OpenACC or OpenMP user interfaces.  There are
  28    declarations in here that are part of the GNU Offloading and Multi
  29    Processing ABI, in that the compiler is required to know about them
  30    and use them.
  31
  32    The convention is that the all caps prefix "GOMP" is used group items
  33    that are part of the external ABI, and the lower case prefix "gomp"
  34    is used group items that are completely private to the library.  */
  35
  36 #ifndef LIBGOMP_H
  37 #define LIBGOMP_H 1
  38
  39 #include "config.h"
  40 #include "gstdint.h"
  41
  42 #include <pthread.h>
  43 #include <stdbool.h>
  44 #include <stdlib.h>
  45 #include <stdarg.h>
  46
  47 #ifdef HAVE_ATTRIBUTE_VISIBILITY
  48 # pragma GCC visibility push(hidden)
  49 #endif
  50
  51 /* If we were a C++ library, we'd get this from <std/atomic>.  */
  52 enum memmodel
  53 {
  54   MEMMODEL_RELAXED = 0,
  55   MEMMODEL_CONSUME = 1,
  56   MEMMODEL_ACQUIRE = 2,
  57   MEMMODEL_RELEASE = 3,
  58   MEMMODEL_ACQ_REL = 4,
  59   MEMMODEL_SEQ_CST = 5
  60 };
  61
  62 #include "sem.h"
  63 #include "mutex.h"
  64 #include "bar.h"
  65 #include "ptrlock.h"
  66
  67
  68 /* This structure contains the data to control one work-sharing construct,
  69    either a LOOP (FOR/DO) or a SECTIONS.  */
  70
  71 enum gomp_schedule_type
  72 {
  73   GFS_RUNTIME,
  74   GFS_STATIC,
  75   GFS_DYNAMIC,
  76   GFS_GUIDED,
  77   GFS_AUTO
  78 };
  79
  80 struct gomp_work_share
  81 {
  82   /* This member records the SCHEDULE clause to be used for this construct.
  83      The user specification of "runtime" will already have been resolved.
  84      If this is a SECTIONS construct, this value will always be DYNAMIC.  */
  85   enum gomp_schedule_type sched;
  86
  87   int mode;
  88
  89   union {
  90     struct {
  91       /* This is the chunk_size argument to the SCHEDULE clause.  */
  92       long chunk_size;
  93
  94       /* This is the iteration end point.  If this is a SECTIONS construct,
  95          this is the number of contained sections.  */
  96       long end;
  97
  98       /* This is the iteration step.  If this is a SECTIONS construct, this
  99          is always 1.  */
 100       long incr;
 101     };
 102
 103     struct {
 104       /* The same as above, but for the unsigned long long loop variants.  */
 105       unsigned long long chunk_size_ull;
 106       unsigned long long end_ull;
 107       unsigned long long incr_ull;
 108     };
 109   };
 110
 111   /* This is a circular queue that details which threads will be allowed
 112      into the ordered region and in which order.  When a thread allocates
 113      iterations on which it is going to work, it also registers itself at
 114      the end of the array.  When a thread reaches the ordered region, it
 115      checks to see if it is the one at the head of the queue.  If not, it
 116      blocks on its RELEASE semaphore.  */
 117   unsigned *ordered_team_ids;
 118
 119   /* This is the number of threads that have registered themselves in
 120      the circular queue ordered_team_ids.  */
 121   unsigned ordered_num_used;
 122
 123   /* This is the team_id of the currently acknowledged owner of the ordered
 124      section, or -1u if the ordered section has not been acknowledged by
 125      any thread.  This is distinguished from the thread that is *allowed*
 126      to take the section next.  */
 127   unsigned ordered_owner;
 128
 129   /* This is the index into the circular queue ordered_team_ids of the
 130      current thread that's allowed into the ordered reason.  */
 131   unsigned ordered_cur;
 132
 133   /* This is a chain of allocated gomp_work_share blocks, valid only
 134      in the first gomp_work_share struct in the block.  */
 135   struct gomp_work_share *next_alloc;
 136
 137   /* The above fields are written once during workshare initialization,
 138      or related to ordered worksharing.  Make sure the following fields
 139      are in a different cache line.  */
 140
 141   /* This lock protects the update of the following members.  */
 142   gomp_mutex_t lock __attribute__((aligned (64)));
 143
 144   /* This is the count of the number of threads that have exited the work
 145      share construct.  If the construct was marked nowait, they have moved on
 146      to other work; otherwise they're blocked on a barrier.  The last member
 147      of the team to exit the work share construct must deallocate it.  */
 148   unsigned threads_completed;
 149
 150   union {
 151     /* This is the next iteration value to be allocated.  In the case of
 152        GFS_STATIC loops, this the iteration start point and never changes.  */
 153     long next;
 154
 155     /* The same, but with unsigned long long type.  */
 156     unsigned long long next_ull;
 157
 158     /* This is the returned data structure for SINGLE COPYPRIVATE.  */
 159     void *copyprivate;
 160   };
 161
 162   union {
 163     /* Link to gomp_work_share struct for next work sharing construct
 164        encountered after this one.  */
 165     gomp_ptrlock_t next_ws;
 166
 167     /* gomp_work_share structs are chained in the free work share cache
 168        through this.  */
 169     struct gomp_work_share *next_free;
 170   };
 171
 172   /* If only few threads are in the team, ordered_team_ids can point
 173      to this array which fills the padding at the end of this struct.  */
 174   unsigned inline_ordered_team_ids[0];
 175 };
 176
 177 /* This structure contains all of the thread-local data associated with
 178    a thread team.  This is the data that must be saved when a thread
 179    encounters a nested PARALLEL construct.  */
 180
 181 struct gomp_team_state
 182 {
 183   /* This is the team of which the thread is currently a member.  */
 184   struct gomp_team *team;
 185
 186   /* This is the work share construct which this thread is currently
 187      processing.  Recall that with NOWAIT, not all threads may be
 188      processing the same construct.  */
 189   struct gomp_work_share *work_share;
 190
 191   /* This is the previous work share construct or NULL if there wasn't any.
 192      When all threads are done with the current work sharing construct,
 193      the previous one can be freed.  The current one can't, as its
 194      next_ws field is used.  */
 195   struct gomp_work_share *last_work_share;
 196
 197   /* This is the ID of this thread within the team.  This value is
 198      guaranteed to be between 0 and N-1, where N is the number of
 199      threads in the team.  */
 200   unsigned team_id;
 201
 202   /* Nesting level.  */
 203   unsigned level;
 204
 205   /* Active nesting level.  Only active parallel regions are counted.  */
 206   unsigned active_level;
 207
 208   /* Place-partition-var, offset and length into gomp_places_list array.  */
 209   unsigned place_partition_off;
 210   unsigned place_partition_len;
 211
 212 #ifdef HAVE_SYNC_BUILTINS
 213   /* Number of single stmts encountered.  */
 214   unsigned long single_count;
 215 #endif
 216
 217   /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the
 218      trip number through the loop.  So first time a particular loop
 219      is encountered this number is 0, the second time through the loop
 220      is 1, etc.  This is unused when the compiler knows in advance that
 221      the loop is statically scheduled.  */
 222   unsigned long static_trip;
 223 };
 224
 225 struct target_mem_desc;
 226 struct gomp_memory_mapping;
 227
 228 /* These are the OpenMP 4.0 Internal Control Variables described in
 229    section 2.3.1.  Those described as having one copy per task are
 230    stored within the structure; those described as having one copy
 231    for the whole program are (naturally) global variables.  */
 232
 233 struct gomp_task_icv
 234 {
 235   unsigned long nthreads_var;
 236   enum gomp_schedule_type run_sched_var;
 237   int run_sched_modifier;
 238   int default_device_var;
 239   unsigned int thread_limit_var;
 240   bool dyn_var;
 241   bool nest_var;
 242   char bind_var;
 243   /* Internal ICV.  */
 244   struct target_mem_desc *target_data;
 245 };
 246
 247 extern struct gomp_task_icv gomp_global_icv;
 248 #ifndef HAVE_SYNC_BUILTINS
 249 extern gomp_mutex_t gomp_managed_threads_lock;
 250 #endif
 251 extern unsigned long gomp_max_active_levels_var;
 252 extern bool gomp_cancel_var;
 253 extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var;
 254 extern unsigned long gomp_available_cpus, gomp_managed_threads;
 255 extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len;
 256 extern char *gomp_bind_var_list;
 257 extern unsigned long gomp_bind_var_list_len;
 258 extern void **gomp_places_list;
 259 extern unsigned long gomp_places_list_len;
 260 extern int gomp_debug_var;
 261 extern int goacc_device_num;
 262 extern char* goacc_device_type;
 263
 264 enum gomp_task_kind
 265 {
 266   GOMP_TASK_IMPLICIT,
 267   GOMP_TASK_IFFALSE,
 268   GOMP_TASK_WAITING,
 269   GOMP_TASK_TIED
 270 };
 271
 272 struct gomp_task;
 273 struct gomp_taskgroup;
 274 struct htab;
 275
 276 struct gomp_task_depend_entry
 277 {
 278   void *addr;
 279   struct gomp_task_depend_entry *next;
 280   struct gomp_task_depend_entry *prev;
 281   struct gomp_task *task;
 282   bool is_in;
 283   bool redundant;
 284   bool redundant_out;
 285 };
 286
 287 struct gomp_dependers_vec
 288 {
 289   size_t n_elem;
 290   size_t allocated;
 291   struct gomp_task *elem[];
 292 };
 293
 294 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies.  */
 295
 296 struct gomp_taskwait
 297 {
 298   bool in_taskwait;
 299   bool in_depend_wait;
 300   size_t n_depend;
 301   struct gomp_task *last_parent_depends_on;
 302   gomp_sem_t taskwait_sem;
 303 };
 304
 305 /* This structure describes a "task" to be run by a thread.  */
 306
 307 struct gomp_task
 308 {
 309   struct gomp_task *parent;
 310   struct gomp_task *children;
 311   struct gomp_task *next_child;
 312   struct gomp_task *prev_child;
 313   struct gomp_task *next_queue;
 314   struct gomp_task *prev_queue;
 315   struct gomp_task *next_taskgroup;
 316   struct gomp_task *prev_taskgroup;
 317   struct gomp_taskgroup *taskgroup;
 318   struct gomp_dependers_vec *dependers;
 319   struct htab *depend_hash;
 320   struct gomp_taskwait *taskwait;
 321   size_t depend_count;
 322   size_t num_dependees;
 323   struct gomp_task_icv icv;
 324   void (*fn) (void *);
 325   void *fn_data;
 326   enum gomp_task_kind kind;
 327   bool in_tied_task;
 328   bool final_task;
 329   bool copy_ctors_done;
 330   bool parent_depends_on;
 331   struct gomp_task_depend_entry depend[];
 332 };
 333
 334 struct gomp_taskgroup
 335 {
 336   struct gomp_taskgroup *prev;
 337   struct gomp_task *children;
 338   bool in_taskgroup_wait;
 339   bool cancelled;
 340   gomp_sem_t taskgroup_sem;
 341   size_t num_children;
 342 };
 343
 344 /* This structure describes a "team" of threads.  These are the threads
 345    that are spawned by a PARALLEL constructs, as well as the work sharing
 346    constructs that the team encounters.  */
 347
 348 struct gomp_team
 349 {
 350   /* This is the number of threads in the current team.  */
 351   unsigned nthreads;
 352
 353   /* This is number of gomp_work_share structs that have been allocated
 354      as a block last time.  */
 355   unsigned work_share_chunk;
 356
 357   /* This is the saved team state that applied to a master thread before
 358      the current thread was created.  */
 359   struct gomp_team_state prev_ts;
 360
 361   /* This semaphore should be used by the master thread instead of its
 362      "native" semaphore in the thread structure.  Required for nested
 363      parallels, as the master is a member of two teams.  */
 364   gomp_sem_t master_release;
 365
 366   /* This points to an array with pointers to the release semaphore
 367      of the threads in the team.  */
 368   gomp_sem_t **ordered_release;
 369
 370   /* List of work shares on which gomp_fini_work_share hasn't been
 371      called yet.  If the team hasn't been cancelled, this should be
 372      equal to each thr->ts.work_share, but otherwise it can be a possibly
 373      long list of workshares.  */
 374   struct gomp_work_share *work_shares_to_free;
 375
 376   /* List of gomp_work_share structs chained through next_free fields.
 377      This is populated and taken off only by the first thread in the
 378      team encountering a new work sharing construct, in a critical
 379      section.  */
 380   struct gomp_work_share *work_share_list_alloc;
 381
 382   /* List of gomp_work_share structs freed by free_work_share.  New
 383      entries are atomically added to the start of the list, and
 384      alloc_work_share can safely only move all but the first entry
 385      to work_share_list alloc, as free_work_share can happen concurrently
 386      with alloc_work_share.  */
 387   struct gomp_work_share *work_share_list_free;
 388
 389 #ifdef HAVE_SYNC_BUILTINS
 390   /* Number of simple single regions encountered by threads in this
 391      team.  */
 392   unsigned long single_count;
 393 #else
 394   /* Mutex protecting addition of workshares to work_share_list_free.  */
 395   gomp_mutex_t work_share_list_free_lock;
 396 #endif
 397
 398   /* This barrier is used for most synchronization of the team.  */
 399   gomp_barrier_t barrier;
 400
 401   /* Initial work shares, to avoid allocating any gomp_work_share
 402      structs in the common case.  */
 403   struct gomp_work_share work_shares[8];
 404
 405   gomp_mutex_t task_lock;
 406   struct gomp_task *task_queue;
 407   /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team.  */
 408   unsigned int task_count;
 409   /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled.  */
 410   unsigned int task_queued_count;
 411   /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
 412      directly in gomp_barrier_handle_tasks; tasks spawned
 413      from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
 414      that is called from a task run from gomp_barrier_handle_tasks.
 415      task_running_count should be always <= team->nthreads,
 416      and if current task isn't in_tied_task, then it will be
 417      even < team->nthreads.  */
 418   unsigned int task_running_count;
 419   int work_share_cancelled;
 420   int team_cancelled;
 421
 422   /* This array contains structures for implicit tasks.  */
 423   struct gomp_task implicit_task[];
 424 };
 425
 426 /* This structure contains all data that is private to libgomp and is
 427    allocated per thread.  */
 428
 429 struct gomp_thread
 430 {
 431   /* This is the function that the thread should run upon launch.  */
 432   void (*fn) (void *data);
 433   void *data;
 434
 435   /* This is the current team state for this thread.  The ts.team member
 436      is NULL only if the thread is idle.  */
 437   struct gomp_team_state ts;
 438
 439   /* This is the task that the thread is currently executing.  */
 440   struct gomp_task *task;
 441
 442   /* This semaphore is used for ordered loops.  */
 443   gomp_sem_t release;
 444
 445   /* Place this thread is bound to plus one, or zero if not bound
 446      to any place.  */
 447   unsigned int place;
 448
 449   /* User pthread thread pool */
 450   struct gomp_thread_pool *thread_pool;
 451 };
 452
 453
 454 struct gomp_thread_pool
 455 {
 456   /* This array manages threads spawned from the top level, which will
 457      return to the idle loop once the current PARALLEL construct ends.  */
 458   struct gomp_thread **threads;
 459   unsigned threads_size;
 460   unsigned threads_used;
 461   struct gomp_team *last_team;
 462   /* Number of threads running in this contention group.  */
 463   unsigned long threads_busy;
 464
 465   /* This barrier holds and releases threads waiting in threads.  */
 466   gomp_barrier_t threads_dock;
 467 };
 468
 469 enum gomp_cancel_kind
 470 {
 471   GOMP_CANCEL_PARALLEL = 1,
 472   GOMP_CANCEL_LOOP = 2,
 473   GOMP_CANCEL_FOR = GOMP_CANCEL_LOOP,
 474   GOMP_CANCEL_DO = GOMP_CANCEL_LOOP,
 475   GOMP_CANCEL_SECTIONS = 4,
 476   GOMP_CANCEL_TASKGROUP = 8
 477 };
 478
 479 /* ... and here is that TLS data.  */
 480
 481 #if defined HAVE_TLS || defined USE_EMUTLS
 482 extern __thread struct gomp_thread gomp_tls_data;
 483 static inline struct gomp_thread *gomp_thread (void)
 484 {
 485   return &gomp_tls_data;
 486 }
 487 #else
 488 extern pthread_key_t gomp_tls_key;
 489 static inline struct gomp_thread *gomp_thread (void)
 490 {
 491   return pthread_getspecific (gomp_tls_key);
 492 }
 493 #endif
 494
 495 extern struct gomp_task_icv *gomp_new_icv (void);
 496
 497 /* Here's how to access the current copy of the ICVs.  */
 498
 499 static inline struct gomp_task_icv *gomp_icv (bool write)
 500 {
 501   struct gomp_task *task = gomp_thread ()->task;
 502   if (task)
 503     return &task->icv;
 504   else if (write)
 505     return gomp_new_icv ();
 506   else
 507     return &gomp_global_icv;
 508 }
 509
 510 /* The attributes to be used during thread creation.  */
 511 extern pthread_attr_t gomp_thread_attr;
 512
 513 /* Function prototypes.  */
 514
 515 /* affinity.c */
 516
 517 extern void gomp_init_affinity (void);
 518 extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int);
 519 extern void **gomp_affinity_alloc (unsigned long, bool);
 520 extern void gomp_affinity_init_place (void *);
 521 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
 522                                     long, bool);
 523 extern bool gomp_affinity_remove_cpu (void *, unsigned long);
 524 extern bool gomp_affinity_copy_place (void *, void *, long);
 525 extern bool gomp_affinity_same_place (void *, void *);
 526 extern bool gomp_affinity_finalize_place_list (bool);
 527 extern bool gomp_affinity_init_level (int, unsigned long, bool);
 528 extern void gomp_affinity_print_place (void *);
 529
 530 /* alloc.c */
 531
 532 extern void *gomp_malloc (size_t) __attribute__((malloc));
 533 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc));
 534 extern void *gomp_realloc (void *, size_t);
 535
 536 /* Avoid conflicting prototypes of alloca() in system headers by using
 537    GCC's builtin alloca().  */
 538 #define gomp_alloca(x)  __builtin_alloca(x)
 539
 540 /* error.c */
 541
 542 extern void gomp_vdebug (int, const char *, va_list);
 543 extern void gomp_debug (int, const char *, ...)
 544         __attribute__((format (printf, 2, 3)));
 545 #define gomp_vdebug(KIND, FMT, VALIST) \
 546   do { \
 547     if (__builtin_expect (gomp_debug_var, 0)) \
 548       (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
 549   } while (0)
 550 #define gomp_debug(KIND, ...) \
 551   do { \
 552     if (__builtin_expect (gomp_debug_var, 0)) \
 553       (gomp_debug) ((KIND), __VA_ARGS__); \
 554   } while (0)
 555 extern void gomp_verror (const char *, va_list);
 556 extern void gomp_error (const char *, ...)
 557         __attribute__((format (printf, 1, 2)));
 558 extern void gomp_vfatal (const char *, va_list)
 559         __attribute__((noreturn));
 560 extern void gomp_fatal (const char *, ...)
 561         __attribute__((noreturn, format (printf, 1, 2)));
 562
 563 /* iter.c */
 564
 565 extern int gomp_iter_static_next (long *, long *);
 566 extern bool gomp_iter_dynamic_next_locked (long *, long *);
 567 extern bool gomp_iter_guided_next_locked (long *, long *);
 568
 569 #ifdef HAVE_SYNC_BUILTINS
 570 extern bool gomp_iter_dynamic_next (long *, long *);
 571 extern bool gomp_iter_guided_next (long *, long *);
 572 #endif
 573
 574 /* iter_ull.c */
 575
 576 extern int gomp_iter_ull_static_next (unsigned long long *,
 577                                       unsigned long long *);
 578 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *,
 579                                                unsigned long long *);
 580 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *,
 581                                               unsigned long long *);
 582
 583 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
 584 extern bool gomp_iter_ull_dynamic_next (unsigned long long *,
 585                                         unsigned long long *);
 586 extern bool gomp_iter_ull_guided_next (unsigned long long *,
 587                                        unsigned long long *);
 588 #endif
 589
 590 /* ordered.c */
 591
 592 extern void gomp_ordered_first (void);
 593 extern void gomp_ordered_last (void);
 594 extern void gomp_ordered_next (void);
 595 extern void gomp_ordered_static_init (void);
 596 extern void gomp_ordered_static_next (void);
 597 extern void gomp_ordered_sync (void);
 598
 599 /* parallel.c */
 600
 601 extern unsigned gomp_resolve_num_threads (unsigned, unsigned);
 602
 603 /* proc.c (in config/) */
 604
 605 extern void gomp_init_num_threads (void);
 606 extern unsigned gomp_dynamic_max_threads (void);
 607
 608 /* task.c */
 609
 610 extern void gomp_init_task (struct gomp_task *, struct gomp_task *,
 611                             struct gomp_task_icv *);
 612 extern void gomp_end_task (void);
 613 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t);
 614
 615 static void inline
 616 gomp_finish_task (struct gomp_task *task)
 617 {
 618   if (__builtin_expect (task->depend_hash != NULL, 0))
 619     free (task->depend_hash);
 620 }
 621
 622 /* team.c */
 623
 624 extern struct gomp_team *gomp_new_team (unsigned);
 625 extern void gomp_team_start (void (*) (void *), void *, unsigned,
 626                              unsigned, struct gomp_team *);
 627 extern void gomp_team_end (void);
 628 extern void gomp_free_thread (void *);
 629
 630 /* target.c */
 631
 632 extern void gomp_init_targets_once (void);
 633 extern int gomp_get_num_devices (void);
 634
 635 #include "libgomp_target.h"
 636 #include "splay-tree.h"
 637
 638 struct target_mem_desc {
 639   /* Reference count.  */
 640   uintptr_t refcount;
 641   /* All the splay nodes allocated together.  */
 642   splay_tree_node array;
 643   /* Start of the target region.  */
 644   uintptr_t tgt_start;
 645   /* End of the targer region.  */
 646   uintptr_t tgt_end;
 647   /* Handle to free.  */
 648   void *to_free;
 649   /* Previous target_mem_desc.  */
 650   struct target_mem_desc *prev;
 651   /* Number of items in following list.  */
 652   size_t list_count;
 653
 654   /* Corresponding target device descriptor.  */
 655   struct gomp_device_descr *device_descr;
 656
 657   /* Memory mapping info for the thread that created this descriptor.  */
 658   struct gomp_memory_mapping *mem_map;
 659
 660   /* List of splay keys to remove (or decrease refcount)
 661      at the end of region.  */
 662   splay_tree_key list[];
 663 };
 664
 665 #define TARGET_CAP_SHARED_MEM   1
 666 #define TARGET_CAP_NATIVE_EXEC  2
 667 #define TARGET_CAP_OPENMP_400   4
 668 #define TARGET_CAP_OPENACC_200  8
 669
 670 /* Information about mapped memory regions (per device/context).  */
 671
 672 struct gomp_memory_mapping
 673 {
 674   /* Splay tree containing information about mapped memory regions.  */
 675   struct splay_tree_s splay_tree;
 676
 677   /* Mutex for operating with the splay tree and other shared structures.  */
 678   gomp_mutex_t lock;
 679
 680   /* True when tables have been added to this memory map.  */
 681   bool is_initialized;
 682 };
 683
 684 typedef struct acc_dispatch_t
 685 {
 686   /* This is a linked list of data mapped using the
 687      acc_map_data/acc_unmap_data or "acc enter data"/"acc exit data" pragmas
 688      (TODO).  Unlike mapped_data in the goacc_thread struct, unmapping can
 689      happen out-of-order with respect to mapping.  */
 690   struct target_mem_desc *data_environ;
 691
 692   /* Open or close a device instance.  */
 693   void *(*open_device_func) (int n);
 694   int (*close_device_func) (void *h);
 695
 696   /* Set or get the device number.  */
 697   int (*get_device_num_func) (void);
 698   void (*set_device_num_func) (int);
 699
 700   /* Execute.  */
 701   void (*exec_func) (void (*) (void *), size_t, void **, void **, size_t *,
 702                      unsigned short *, int, int, int, int, void *);
 703
 704   /* Async cleanup callback registration.  */
 705   void (*register_async_cleanup_func) (void *);
 706
 707   /* Asynchronous routines.  */
 708   int (*async_test_func) (int);
 709   int (*async_test_all_func) (void);
 710   void (*async_wait_func) (int);
 711   void (*async_wait_async_func) (int, int);
 712   void (*async_wait_all_func) (void);
 713   void (*async_wait_all_async_func) (int);
 714   void (*async_set_async_func) (int);
 715
 716   /* Create/destroy TLS data.  */
 717   void *(*create_thread_data_func) (void *);
 718   void (*destroy_thread_data_func) (void *);
 719
 720   /* NVIDIA target specific routines.  */
 721   struct {
 722     void *(*get_current_device_func) (void);
 723     void *(*get_current_context_func) (void);
 724     void *(*get_stream_func) (int);
 725     int (*set_stream_func) (int, void *);
 726   } cuda;
 727 } acc_dispatch_t;
 728
 729 /* This structure describes accelerator device.
 730    It contains name of the corresponding libgomp plugin, function handlers for
 731    interaction with the device, ID-number of the device, and information about
 732    mapped memory.  */
 733 struct gomp_device_descr
 734 {
 735   /* The name of the device.  */
 736   const char *name;
 737
 738   /* Capabilities of device (supports OpenACC, OpenMP).  */
 739   unsigned int capabilities;
 740
 741   /* This is the ID number of device.  It could be specified in DEVICE-clause of
 742      TARGET construct.  */
 743   int id;
 744
 745   /* This is the ID number of device among devices of the same type.  */
 746   int target_id;
 747
 748   /* This is the TYPE of device.  */
 749   enum offload_target_type type;
 750
 751   /* Set to true when device is initialized.  */
 752   bool is_initialized;
 753
 754   /* True when offload regions have been registered with this device.  */
 755   bool offload_regions_registered;
 756
 757   /* Function handlers.  */
 758   const char *(*get_name_func) (void);
 759   unsigned int (*get_caps_func) (void);
 760   int (*get_type_func) (void);
 761   int (*get_num_devices_func) (void);
 762   void (*register_image_func) (void *, void *);
 763   void (*init_device_func) (int);
 764   void (*fini_device_func) (int);
 765   int (*get_table_func) (int, struct mapping_table **);
 766   void *(*alloc_func) (int, size_t);
 767   void (*free_func) (int, void *);
 768   void *(*dev2host_func) (int, void *, const void *, size_t);
 769   void *(*host2dev_func) (int, void *, const void *, size_t);
 770   void (*run_func) (int, void *, void *);
 771
 772   /* OpenACC-specific functions.  */
 773   acc_dispatch_t openacc;
 774
 775   /* Memory-mapping info for this device instance.  */
 776   struct gomp_memory_mapping mem_map;
 777
 778   /* Extra information required for a device instance by a given target.  */
 779   void *target_data;
 780 };
 781
 782 extern void gomp_acc_insert_pointer (size_t, void **, size_t *, void *);
 783 extern void gomp_acc_remove_pointer (void *, bool, int, int);
 784
 785 extern struct target_mem_desc *gomp_map_vars (struct gomp_device_descr *,
 786                                               size_t, void **, void **,
 787                                               size_t *, void *, bool, bool);
 788
 789 extern void gomp_copy_from_async (struct target_mem_desc *);
 790
 791 extern void gomp_unmap_vars (struct target_mem_desc *, bool);
 792
 793 extern void gomp_init_device (struct gomp_device_descr *);
 794
 795 extern void gomp_init_tables (const struct gomp_device_descr *,
 796                               struct gomp_memory_mapping *);
 797
 798 extern void gomp_fini_device (struct gomp_device_descr *);
 799
 800 extern void gomp_free_memmap (struct gomp_device_descr *);
 801
 802 /* work.c */
 803
 804 extern void gomp_init_work_share (struct gomp_work_share *, bool, unsigned);
 805 extern void gomp_fini_work_share (struct gomp_work_share *);
 806 extern bool gomp_work_share_start (bool);
 807 extern void gomp_work_share_end (void);
 808 extern bool gomp_work_share_end_cancel (void);
 809 extern void gomp_work_share_end_nowait (void);
 810
 811 static inline void
 812 gomp_work_share_init_done (void)
 813 {
 814   struct gomp_thread *thr = gomp_thread ();
 815   if (__builtin_expect (thr->ts.last_work_share != NULL, 1))
 816     gomp_ptrlock_set (&thr->ts.last_work_share->next_ws, thr->ts.work_share);
 817 }
 818
 819 #ifdef HAVE_ATTRIBUTE_VISIBILITY
 820 # pragma GCC visibility pop
 821 #endif
 822
 823 /* Now that we're back to default visibility, include the globals.  */
 824 #include "libgomp_g.h"
 825
 826 /* Include omp.h by parts.  */
 827 #include "omp-lock.h"
 828 #define _LIBGOMP_OMP_LOCK_DEFINED 1
 829 #include "omp.h.in"
 830
 831 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \
 832     || !defined (HAVE_ATTRIBUTE_ALIAS) \
 833     || !defined (HAVE_AS_SYMVER_DIRECTIVE) \
 834     || !defined (PIC) \
 835     || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT)
 836 # undef LIBGOMP_GNU_SYMBOL_VERSIONING
 837 #endif
 838
 839 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING
 840 extern void gomp_init_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 841 extern void gomp_destroy_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 842 extern void gomp_set_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 843 extern void gomp_unset_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 844 extern int gomp_test_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 845 extern void gomp_init_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 846 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 847 extern void gomp_set_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 848 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 849 extern int gomp_test_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 850
 851 extern void gomp_init_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 852 extern void gomp_destroy_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 853 extern void gomp_set_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 854 extern void gomp_unset_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 855 extern int gomp_test_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 856 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 857 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 858 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 859 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 860 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 861
 862 # define strong_alias(fn, al) \
 863   extern __typeof (fn) al __attribute__ ((alias (#fn)));
 864 # define omp_lock_symver(fn) \
 865   __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
 866   __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
 867 #else
 868 # define gomp_init_lock_30 omp_init_lock
 869 # define gomp_destroy_lock_30 omp_destroy_lock
 870 # define gomp_set_lock_30 omp_set_lock
 871 # define gomp_unset_lock_30 omp_unset_lock
 872 # define gomp_test_lock_30 omp_test_lock
 873 # define gomp_init_nest_lock_30 omp_init_nest_lock
 874 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock
 875 # define gomp_set_nest_lock_30 omp_set_nest_lock
 876 # define gomp_unset_nest_lock_30 omp_unset_nest_lock
 877 # define gomp_test_nest_lock_30 omp_test_nest_lock
 878 #endif
 879
 880 #ifdef HAVE_ATTRIBUTE_VISIBILITY
 881 # define attribute_hidden __attribute__ ((visibility ("hidden")))
 882 #else
 883 # define attribute_hidden
 884 #endif
 885
 886 #ifdef HAVE_ATTRIBUTE_ALIAS
 887 # define ialias_ulp     ialias_str1(__USER_LABEL_PREFIX__)
 888 # define ialias_str1(x) ialias_str2(x)
 889 # define ialias_str2(x) #x
 890 # define ialias(fn) \
 891   extern __typeof (fn) gomp_ialias_##fn \
 892     __attribute__ ((alias (#fn))) attribute_hidden;
 893 # define ialias_redirect(fn) \
 894   extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
 895 # define ialias_call(fn) gomp_ialias_ ## fn
 896 #else
 897 # define ialias(fn)
 898 # define ialias_redirect(fn)
 899 # define ialias_call(fn) fn
 900 #endif
 901
 902 #endif /* LIBGOMP_H */