libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2016 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "oacc-int.h"
  31 #include "openacc.h"
  32 #include <assert.h>
  33 #include <stdlib.h>
  34 #include <strings.h>
  35 #include <stdbool.h>
  36 #include <string.h>
  37
  38 /* This lock is used to protect access to cached_base_dev, dispatchers and
  39    the (abstract) initialisation state of attached offloading devices.  */
  40
  41 static gomp_mutex_t acc_device_lock;
  42
  43 /* A cached version of the dispatcher for the global "current" accelerator type,
  44    e.g. used as the default when creating new host threads.  This is the
  45    device-type equivalent of goacc_device_num (which specifies which device to
  46    use out of potentially several of the same type).  If there are several
  47    devices of a given type, this points at the first one.  */
  48
  49 static struct gomp_device_descr *cached_base_dev = NULL;
  50
  51 #if defined HAVE_TLS || defined USE_EMUTLS
  52 __thread struct goacc_thread *goacc_tls_data;
  53 #else
  54 pthread_key_t goacc_tls_key;
  55 #endif
  56 static pthread_key_t goacc_cleanup_key;
  57
  58 static struct goacc_thread *goacc_threads;
  59 static gomp_mutex_t goacc_thread_lock;
  60
  61 /* An array of dispatchers for device types, indexed by the type.  This array
  62    only references "base" devices, and other instances of the same type are
  63    found by simply indexing from each such device (which are stored linearly,
  64    grouped by device in target.c:devices).  */
  65 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  66
  67 attribute_hidden void
  68 goacc_register (struct gomp_device_descr *disp)
  69 {
  70   /* Only register the 0th device here.  */
  71   if (disp->target_id != 0)
  72     return;
  73
  74   gomp_mutex_lock (&acc_device_lock);
  75
  76   assert (acc_device_type (disp->type) != acc_device_none
  77           && acc_device_type (disp->type) != acc_device_default
  78           && acc_device_type (disp->type) != acc_device_not_host);
  79   assert (!dispatchers[disp->type]);
  80   dispatchers[disp->type] = disp;
  81
  82   gomp_mutex_unlock (&acc_device_lock);
  83 }
  84
  85 /* OpenACC names some things a little differently.  */
  86
  87 static const char *
  88 get_openacc_name (const char *name)
  89 {
  90   if (strcmp (name, "nvptx") == 0)
  91     return "nvidia";
  92   else
  93     return name;
  94 }
  95
  96 static const char *
  97 name_of_acc_device_t (enum acc_device_t type)
  98 {
  99   switch (type)
 100     {
 101     case acc_device_none: return "none";
 102     case acc_device_default: return "default";
 103     case acc_device_host: return "host";
 104     case acc_device_not_host: return "not_host";
 105     case acc_device_nvidia: return "nvidia";
 106     default: gomp_fatal ("unknown device type %u", (unsigned) type);
 107     }
 108 }
 109
 110 /* ACC_DEVICE_LOCK must be held before calling this function.  If FAIL_IS_ERROR
 111    is true, this function raises an error if there are no devices of type D,
 112    otherwise it returns NULL in that case.  */
 113
 114 static struct gomp_device_descr *
 115 resolve_device (acc_device_t d, bool fail_is_error)
 116 {
 117   acc_device_t d_arg = d;
 118
 119   switch (d)
 120     {
 121     case acc_device_default:
 122       {
 123         if (goacc_device_type)
 124           {
 125             /* Lookup the named device.  */
 126             while (++d != _ACC_device_hwm)
 127               if (dispatchers[d]
 128                   && !strcasecmp (goacc_device_type,
 129                                   get_openacc_name (dispatchers[d]->name))
 130                   && dispatchers[d]->get_num_devices_func () > 0)
 131                 goto found;
 132
 133             if (fail_is_error)
 134               {
 135                 gomp_mutex_unlock (&acc_device_lock);
 136                 gomp_fatal ("device type %s not supported", goacc_device_type);
 137               }
 138             else
 139               return NULL;
 140           }
 141
 142         /* No default device specified, so start scanning for any non-host
 143            device that is available.  */
 144         d = acc_device_not_host;
 145       }
 146       /* FALLTHROUGH */
 147
 148     case acc_device_not_host:
 149       /* Find the first available device after acc_device_not_host.  */
 150       while (++d != _ACC_device_hwm)
 151         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 152           goto found;
 153       if (d_arg == acc_device_default)
 154         {
 155           d = acc_device_host;
 156           goto found;
 157         }
 158       if (fail_is_error)
 159         {
 160           gomp_mutex_unlock (&acc_device_lock);
 161           gomp_fatal ("no device found");
 162         }
 163       else
 164         return NULL;
 165       break;
 166
 167     case acc_device_host:
 168       break;
 169
 170     default:
 171       if (d > _ACC_device_hwm)
 172         {
 173           if (fail_is_error)
 174             goto unsupported_device;
 175           else
 176             return NULL;
 177         }
 178       break;
 179     }
 180  found:
 181
 182   assert (d != acc_device_none
 183           && d != acc_device_default
 184           && d != acc_device_not_host);
 185
 186   if (dispatchers[d] == NULL && fail_is_error)
 187     {
 188     unsupported_device:
 189       gomp_mutex_unlock (&acc_device_lock);
 190       gomp_fatal ("device type %s not supported", name_of_acc_device_t (d));
 191     }
 192
 193   return dispatchers[d];
 194 }
 195
 196 /* Emit a suitable error if no device of a particular type is available, or
 197    the given device number is out-of-range.  */
 198 static void
 199 acc_dev_num_out_of_range (acc_device_t d, int ord, int ndevs)
 200 {
 201   if (ndevs == 0)
 202     gomp_fatal ("no devices of type %s available", name_of_acc_device_t (d));
 203   else
 204     gomp_fatal ("device %u out of range", ord);
 205 }
 206
 207 /* This is called when plugins have been initialized, and serves to call
 208    (indirectly) the target's device_init hook.  Calling multiple times without
 209    an intervening acc_shutdown_1 call is an error.  ACC_DEVICE_LOCK must be
 210    held before calling this function.  */
 211
 212 static struct gomp_device_descr *
 213 acc_init_1 (acc_device_t d)
 214 {
 215   struct gomp_device_descr *base_dev, *acc_dev;
 216   int ndevs;
 217
 218   base_dev = resolve_device (d, true);
 219
 220   ndevs = base_dev->get_num_devices_func ();
 221
 222   if (ndevs <= 0 || goacc_device_num >= ndevs)
 223     acc_dev_num_out_of_range (d, goacc_device_num, ndevs);
 224
 225   acc_dev = &base_dev[goacc_device_num];
 226
 227   gomp_mutex_lock (&acc_dev->lock);
 228   if (acc_dev->state == GOMP_DEVICE_INITIALIZED)
 229     {
 230       gomp_mutex_unlock (&acc_dev->lock);
 231       gomp_fatal ("device already active");
 232     }
 233
 234   gomp_init_device (acc_dev);
 235   gomp_mutex_unlock (&acc_dev->lock);
 236
 237   return base_dev;
 238 }
 239
 240 /* ACC_DEVICE_LOCK must be held before calling this function.  */
 241
 242 static void
 243 acc_shutdown_1 (acc_device_t d)
 244 {
 245   struct gomp_device_descr *base_dev;
 246   struct goacc_thread *walk;
 247   int ndevs, i;
 248   bool devices_active = false;
 249
 250   /* Get the base device for this device type.  */
 251   base_dev = resolve_device (d, true);
 252
 253   ndevs = base_dev->get_num_devices_func ();
 254
 255   /* Unload all the devices of this type that have been opened.  */
 256   for (i = 0; i < ndevs; i++)
 257     {
 258       struct gomp_device_descr *acc_dev = &base_dev[i];
 259
 260       gomp_mutex_lock (&acc_dev->lock);
 261       gomp_unload_device (acc_dev);
 262       gomp_mutex_unlock (&acc_dev->lock);
 263     }
 264
 265   gomp_mutex_lock (&goacc_thread_lock);
 266
 267   /* Free target-specific TLS data and close all devices.  */
 268   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 269     {
 270       if (walk->target_tls)
 271         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 272
 273       walk->target_tls = NULL;
 274
 275       /* This would mean the user is shutting down OpenACC in the middle of an
 276          "acc data" pragma.  Likely not intentional.  */
 277       if (walk->mapped_data)
 278         {
 279           gomp_mutex_unlock (&goacc_thread_lock);
 280           gomp_fatal ("shutdown in 'acc data' region");
 281         }
 282
 283       /* Similarly, if this happens then user code has done something weird.  */
 284       if (walk->saved_bound_dev)
 285         {
 286           gomp_mutex_unlock (&goacc_thread_lock);
 287           gomp_fatal ("shutdown during host fallback");
 288         }
 289
 290       if (walk->dev)
 291         {
 292           gomp_mutex_lock (&walk->dev->lock);
 293           gomp_free_memmap (&walk->dev->mem_map);
 294           gomp_mutex_unlock (&walk->dev->lock);
 295
 296           walk->dev = NULL;
 297           walk->base_dev = NULL;
 298         }
 299     }
 300
 301   gomp_mutex_unlock (&goacc_thread_lock);
 302
 303
 304   /* Close all the devices of this type that have been opened.  */
 305   for (i = 0; i < ndevs; i++)
 306     {
 307       struct gomp_device_descr *acc_dev = &base_dev[i];
 308       gomp_mutex_lock (&acc_dev->lock);
 309       if (acc_dev->state == GOMP_DEVICE_INITIALIZED)
 310         {
 311           devices_active = true;
 312           acc_dev->fini_device_func (acc_dev->target_id);
 313           acc_dev->state = GOMP_DEVICE_UNINITIALIZED;
 314         }
 315       gomp_mutex_unlock (&acc_dev->lock);
 316     }
 317
 318   if (!devices_active)
 319     gomp_fatal ("no device initialized");
 320 }
 321
 322 static struct goacc_thread *
 323 goacc_new_thread (void)
 324 {
 325   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 326
 327 #if defined HAVE_TLS || defined USE_EMUTLS
 328   goacc_tls_data = thr;
 329 #else
 330   pthread_setspecific (goacc_tls_key, thr);
 331 #endif
 332
 333   pthread_setspecific (goacc_cleanup_key, thr);
 334
 335   gomp_mutex_lock (&goacc_thread_lock);
 336   thr->next = goacc_threads;
 337   goacc_threads = thr;
 338   gomp_mutex_unlock (&goacc_thread_lock);
 339
 340   return thr;
 341 }
 342
 343 static void
 344 goacc_destroy_thread (void *data)
 345 {
 346   struct goacc_thread *thr = data, *walk, *prev;
 347
 348   gomp_mutex_lock (&goacc_thread_lock);
 349
 350   if (thr)
 351     {
 352       struct gomp_device_descr *acc_dev = thr->dev;
 353
 354       if (acc_dev && thr->target_tls)
 355         {
 356           acc_dev->openacc.destroy_thread_data_func (thr->target_tls);
 357           thr->target_tls = NULL;
 358         }
 359
 360       assert (!thr->mapped_data);
 361
 362       /* Remove from thread list.  */
 363       for (prev = NULL, walk = goacc_threads; walk;
 364            prev = walk, walk = walk->next)
 365         if (walk == thr)
 366           {
 367             if (prev == NULL)
 368               goacc_threads = walk->next;
 369             else
 370               prev->next = walk->next;
 371
 372             free (thr);
 373
 374             break;
 375           }
 376
 377       assert (walk);
 378     }
 379
 380   gomp_mutex_unlock (&goacc_thread_lock);
 381 }
 382
 383 /* Use the ORD'th device instance for the current host thread (or -1 for the
 384    current global default).  The device (and the runtime) must be initialised
 385    before calling this function.  */
 386
 387 void
 388 goacc_attach_host_thread_to_device (int ord)
 389 {
 390   struct goacc_thread *thr = goacc_thread ();
 391   struct gomp_device_descr *acc_dev = NULL, *base_dev = NULL;
 392   int num_devices;
 393
 394   if (thr && thr->dev && (thr->dev->target_id == ord || ord < 0))
 395     return;
 396
 397   if (ord < 0)
 398     ord = goacc_device_num;
 399
 400   /* Decide which type of device to use.  If the current thread has a device
 401      type already (e.g. set by acc_set_device_type), use that, else use the
 402      global default.  */
 403   if (thr && thr->base_dev)
 404     base_dev = thr->base_dev;
 405   else
 406     {
 407       assert (cached_base_dev);
 408       base_dev = cached_base_dev;
 409     }
 410
 411   num_devices = base_dev->get_num_devices_func ();
 412   if (num_devices <= 0 || ord >= num_devices)
 413     acc_dev_num_out_of_range (acc_device_type (base_dev->type), ord,
 414                               num_devices);
 415
 416   if (!thr)
 417     thr = goacc_new_thread ();
 418
 419   thr->base_dev = base_dev;
 420   thr->dev = acc_dev = &base_dev[ord];
 421   thr->saved_bound_dev = NULL;
 422   thr->mapped_data = NULL;
 423
 424   thr->target_tls
 425     = acc_dev->openacc.create_thread_data_func (ord);
 426
 427   acc_dev->openacc.async_set_async_func (acc_async_sync);
 428 }
 429
 430 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 431    init/shutdown is per-process or per-thread.  We choose per-process.  */
 432
 433 void
 434 acc_init (acc_device_t d)
 435 {
 436   if (!cached_base_dev)
 437     gomp_init_targets_once ();
 438
 439   gomp_mutex_lock (&acc_device_lock);
 440
 441   cached_base_dev = acc_init_1 (d);
 442
 443   gomp_mutex_unlock (&acc_device_lock);
 444
 445   goacc_attach_host_thread_to_device (-1);
 446 }
 447
 448 ialias (acc_init)
 449
 450 void
 451 acc_shutdown (acc_device_t d)
 452 {
 453   gomp_init_targets_once ();
 454
 455   gomp_mutex_lock (&acc_device_lock);
 456
 457   acc_shutdown_1 (d);
 458
 459   gomp_mutex_unlock (&acc_device_lock);
 460 }
 461
 462 ialias (acc_shutdown)
 463
 464 int
 465 acc_get_num_devices (acc_device_t d)
 466 {
 467   int n = 0;
 468   struct gomp_device_descr *acc_dev;
 469
 470   if (d == acc_device_none)
 471     return 0;
 472
 473   gomp_init_targets_once ();
 474
 475   gomp_mutex_lock (&acc_device_lock);
 476   acc_dev = resolve_device (d, false);
 477   gomp_mutex_unlock (&acc_device_lock);
 478
 479   if (!acc_dev)
 480     return 0;
 481
 482   n = acc_dev->get_num_devices_func ();
 483   if (n < 0)
 484     n = 0;
 485
 486   return n;
 487 }
 488
 489 ialias (acc_get_num_devices)
 490
 491 /* Set the device type for the current thread only (using the current global
 492    default device number), initialising that device if necessary.  Also set the
 493    default device type for new threads to D.  */
 494
 495 void
 496 acc_set_device_type (acc_device_t d)
 497 {
 498   struct gomp_device_descr *base_dev, *acc_dev;
 499   struct goacc_thread *thr = goacc_thread ();
 500
 501   gomp_mutex_lock (&acc_device_lock);
 502
 503   if (!cached_base_dev)
 504     gomp_init_targets_once ();
 505
 506   cached_base_dev = base_dev = resolve_device (d, true);
 507   acc_dev = &base_dev[goacc_device_num];
 508
 509   gomp_mutex_lock (&acc_dev->lock);
 510   if (acc_dev->state == GOMP_DEVICE_UNINITIALIZED)
 511     gomp_init_device (acc_dev);
 512   gomp_mutex_unlock (&acc_dev->lock);
 513
 514   gomp_mutex_unlock (&acc_device_lock);
 515
 516   /* We're changing device type: invalidate the current thread's dev and
 517      base_dev pointers.  */
 518   if (thr && thr->base_dev != base_dev)
 519     {
 520       thr->base_dev = thr->dev = NULL;
 521       if (thr->mapped_data)
 522         gomp_fatal ("acc_set_device_type in 'acc data' region");
 523     }
 524
 525   goacc_attach_host_thread_to_device (-1);
 526 }
 527
 528 ialias (acc_set_device_type)
 529
 530 acc_device_t
 531 acc_get_device_type (void)
 532 {
 533   acc_device_t res = acc_device_none;
 534   struct gomp_device_descr *dev;
 535   struct goacc_thread *thr = goacc_thread ();
 536
 537   if (thr && thr->base_dev)
 538     res = acc_device_type (thr->base_dev->type);
 539   else
 540     {
 541       gomp_init_targets_once ();
 542
 543       gomp_mutex_lock (&acc_device_lock);
 544       dev = resolve_device (acc_device_default, true);
 545       gomp_mutex_unlock (&acc_device_lock);
 546       res = acc_device_type (dev->type);
 547     }
 548
 549   assert (res != acc_device_default
 550           && res != acc_device_not_host);
 551
 552   return res;
 553 }
 554
 555 ialias (acc_get_device_type)
 556
 557 int
 558 acc_get_device_num (acc_device_t d)
 559 {
 560   const struct gomp_device_descr *dev;
 561   struct goacc_thread *thr = goacc_thread ();
 562
 563   if (d >= _ACC_device_hwm)
 564     gomp_fatal ("unknown device type %u", (unsigned) d);
 565
 566   if (!cached_base_dev)
 567     gomp_init_targets_once ();
 568
 569   gomp_mutex_lock (&acc_device_lock);
 570   dev = resolve_device (d, true);
 571   gomp_mutex_unlock (&acc_device_lock);
 572
 573   if (thr && thr->base_dev == dev && thr->dev)
 574     return thr->dev->target_id;
 575
 576   return goacc_device_num;
 577 }
 578
 579 ialias (acc_get_device_num)
 580
 581 void
 582 acc_set_device_num (int ord, acc_device_t d)
 583 {
 584   struct gomp_device_descr *base_dev, *acc_dev;
 585   int num_devices;
 586
 587   if (!cached_base_dev)
 588     gomp_init_targets_once ();
 589
 590   if (ord < 0)
 591     ord = goacc_device_num;
 592
 593   if ((int) d == 0)
 594     /* Set whatever device is being used by the current host thread to use
 595        device instance ORD.  It's unclear if this is supposed to affect other
 596        host threads too (OpenACC 2.0 (3.2.4) acc_set_device_num).  */
 597     goacc_attach_host_thread_to_device (ord);
 598   else
 599     {
 600       gomp_mutex_lock (&acc_device_lock);
 601
 602       cached_base_dev = base_dev = resolve_device (d, true);
 603
 604       num_devices = base_dev->get_num_devices_func ();
 605
 606       if (num_devices <= 0 || ord >= num_devices)
 607         acc_dev_num_out_of_range (d, ord, num_devices);
 608
 609       acc_dev = &base_dev[ord];
 610
 611       gomp_mutex_lock (&acc_dev->lock);
 612       if (acc_dev->state == GOMP_DEVICE_UNINITIALIZED)
 613         gomp_init_device (acc_dev);
 614       gomp_mutex_unlock (&acc_dev->lock);
 615
 616       gomp_mutex_unlock (&acc_device_lock);
 617
 618       goacc_attach_host_thread_to_device (ord);
 619     }
 620
 621   goacc_device_num = ord;
 622 }
 623
 624 ialias (acc_set_device_num)
 625
 626 /* For -O and higher, the compiler always attempts to expand acc_on_device, but
 627    if the user disables the builtin, or calls it via a pointer, we'll need this
 628    version.
 629
 630    Compile this with optimization, so that the compiler expands
 631    this, rather than generating infinitely recursive code.  */
 632
 633 int __attribute__ ((__optimize__ ("O2")))
 634 acc_on_device (acc_device_t dev)
 635 {
 636   return __builtin_acc_on_device (dev);
 637 }
 638
 639 ialias (acc_on_device)
 640
 641 attribute_hidden void
 642 goacc_runtime_initialize (void)
 643 {
 644   gomp_mutex_init (&acc_device_lock);
 645
 646 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 647   pthread_key_create (&goacc_tls_key, NULL);
 648 #endif
 649
 650   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 651
 652   cached_base_dev = NULL;
 653
 654   goacc_threads = NULL;
 655   gomp_mutex_init (&goacc_thread_lock);
 656
 657   /* Initialize and register the 'host' device type.  */
 658   goacc_host_init ();
 659 }
 660
 661 /* Compiler helper functions */
 662
 663 attribute_hidden void
 664 goacc_save_and_set_bind (acc_device_t d)
 665 {
 666   struct goacc_thread *thr = goacc_thread ();
 667
 668   assert (!thr->saved_bound_dev);
 669
 670   thr->saved_bound_dev = thr->dev;
 671   thr->dev = dispatchers[d];
 672 }
 673
 674 attribute_hidden void
 675 goacc_restore_bind (void)
 676 {
 677   struct goacc_thread *thr = goacc_thread ();
 678
 679   thr->dev = thr->saved_bound_dev;
 680   thr->saved_bound_dev = NULL;
 681 }
 682
 683 /* This is called from any OpenACC support function that may need to implicitly
 684    initialize the libgomp runtime, either globally or from a new host thread.
 685    On exit "goacc_thread" will return a valid & populated thread block.  */
 686
 687 attribute_hidden void
 688 goacc_lazy_initialize (void)
 689 {
 690   struct goacc_thread *thr = goacc_thread ();
 691
 692   if (thr && thr->dev)
 693     return;
 694
 695   if (!cached_base_dev)
 696     acc_init (acc_device_default);
 697   else
 698     goacc_attach_host_thread_to_device (-1);
 699 }