libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2015 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "oacc-int.h"
  31 #include "openacc.h"
  32 #include "plugin/plugin-host.h"
  33 #include <assert.h>
  34 #include <stdlib.h>
  35 #include <strings.h>
  36 #include <stdbool.h>
  37 #include <string.h>
  38
  39 /* This lock is used to protect access to cached_base_dev, dispatchers and
  40    the (abstract) initialisation state of attached offloading devices.  */
  41
  42 static gomp_mutex_t acc_device_lock;
  43
  44 /* A cached version of the dispatcher for the global "current" accelerator type,
  45    e.g. used as the default when creating new host threads.  This is the
  46    device-type equivalent of goacc_device_num (which specifies which device to
  47    use out of potentially several of the same type).  If there are several
  48    devices of a given type, this points at the first one.  */
  49
  50 static struct gomp_device_descr *cached_base_dev = NULL;
  51
  52 #if defined HAVE_TLS || defined USE_EMUTLS
  53 __thread struct goacc_thread *goacc_tls_data;
  54 #else
  55 pthread_key_t goacc_tls_key;
  56 #endif
  57 static pthread_key_t goacc_cleanup_key;
  58
  59 static struct goacc_thread *goacc_threads;
  60 static gomp_mutex_t goacc_thread_lock;
  61
  62 /* An array of dispatchers for device types, indexed by the type.  This array
  63    only references "base" devices, and other instances of the same type are
  64    found by simply indexing from each such device (which are stored linearly,
  65    grouped by device in target.c:devices).  */
  66 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  67
  68 attribute_hidden void
  69 goacc_register (struct gomp_device_descr *disp)
  70 {
  71   /* Only register the 0th device here.  */
  72   if (disp->target_id != 0)
  73     return;
  74
  75   gomp_mutex_lock (&acc_device_lock);
  76
  77   assert (acc_device_type (disp->type) != acc_device_none
  78           && acc_device_type (disp->type) != acc_device_default
  79           && acc_device_type (disp->type) != acc_device_not_host);
  80   assert (!dispatchers[disp->type]);
  81   dispatchers[disp->type] = disp;
  82
  83   gomp_mutex_unlock (&acc_device_lock);
  84 }
  85
  86 /* OpenACC names some things a little differently.  */
  87
  88 static const char *
  89 get_openacc_name (const char *name)
  90 {
  91   if (strcmp (name, "nvptx") == 0)
  92     return "nvidia";
  93   else
  94     return name;
  95 }
  96
  97 static const char *
  98 name_of_acc_device_t (enum acc_device_t type)
  99 {
 100   switch (type)
 101     {
 102     case acc_device_none: return "none";
 103     case acc_device_default: return "default";
 104     case acc_device_host: return "host";
 105     case acc_device_host_nonshm: return "host_nonshm";
 106     case acc_device_not_host: return "not_host";
 107     case acc_device_nvidia: return "nvidia";
 108     default: gomp_fatal ("unknown device type %u", (unsigned) type);
 109     }
 110 }
 111
 112 /* ACC_DEVICE_LOCK must be held before calling this function.  If FAIL_IS_ERROR
 113    is true, this function raises an error if there are no devices of type D,
 114    otherwise it returns NULL in that case.  */
 115
 116 static struct gomp_device_descr *
 117 resolve_device (acc_device_t d, bool fail_is_error)
 118 {
 119   acc_device_t d_arg = d;
 120
 121   switch (d)
 122     {
 123     case acc_device_default:
 124       {
 125         if (goacc_device_type)
 126           {
 127             /* Lookup the named device.  */
 128             while (++d != _ACC_device_hwm)
 129               if (dispatchers[d]
 130                   && !strcasecmp (goacc_device_type,
 131                                   get_openacc_name (dispatchers[d]->name))
 132                   && dispatchers[d]->get_num_devices_func () > 0)
 133                 goto found;
 134
 135             if (fail_is_error)
 136               {
 137                 gomp_mutex_unlock (&acc_device_lock);
 138                 gomp_fatal ("device type %s not supported", goacc_device_type);
 139               }
 140             else
 141               return NULL;
 142           }
 143
 144         /* No default device specified, so start scanning for any non-host
 145            device that is available.  */
 146         d = acc_device_not_host;
 147       }
 148       /* FALLTHROUGH */
 149
 150     case acc_device_not_host:
 151       /* Find the first available device after acc_device_not_host.  */
 152       while (++d != _ACC_device_hwm)
 153         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 154           goto found;
 155       if (d_arg == acc_device_default)
 156         {
 157           d = acc_device_host;
 158           goto found;
 159         }
 160       if (fail_is_error)
 161         {
 162           gomp_mutex_unlock (&acc_device_lock);
 163           gomp_fatal ("no device found");
 164         }
 165       else
 166         return NULL;
 167       break;
 168
 169     case acc_device_host:
 170       break;
 171
 172     default:
 173       if (d > _ACC_device_hwm)
 174         {
 175           if (fail_is_error)
 176             goto unsupported_device;
 177           else
 178             return NULL;
 179         }
 180       break;
 181     }
 182  found:
 183
 184   assert (d != acc_device_none
 185           && d != acc_device_default
 186           && d != acc_device_not_host);
 187
 188   if (dispatchers[d] == NULL && fail_is_error)
 189     {
 190     unsupported_device:
 191       gomp_mutex_unlock (&acc_device_lock);
 192       gomp_fatal ("device type %s not supported", name_of_acc_device_t (d));
 193     }
 194
 195   return dispatchers[d];
 196 }
 197
 198 /* Emit a suitable error if no device of a particular type is available, or
 199    the given device number is out-of-range.  */
 200 static void
 201 acc_dev_num_out_of_range (acc_device_t d, int ord, int ndevs)
 202 {
 203   if (ndevs == 0)
 204     gomp_fatal ("no devices of type %s available", name_of_acc_device_t (d));
 205   else
 206     gomp_fatal ("device %u out of range", ord);
 207 }
 208
 209 /* This is called when plugins have been initialized, and serves to call
 210    (indirectly) the target's device_init hook.  Calling multiple times without
 211    an intervening acc_shutdown_1 call is an error.  ACC_DEVICE_LOCK must be
 212    held before calling this function.  */
 213
 214 static struct gomp_device_descr *
 215 acc_init_1 (acc_device_t d)
 216 {
 217   struct gomp_device_descr *base_dev, *acc_dev;
 218   int ndevs;
 219
 220   base_dev = resolve_device (d, true);
 221
 222   ndevs = base_dev->get_num_devices_func ();
 223
 224   if (ndevs <= 0 || goacc_device_num >= ndevs)
 225     acc_dev_num_out_of_range (d, goacc_device_num, ndevs);
 226
 227   acc_dev = &base_dev[goacc_device_num];
 228
 229   gomp_mutex_lock (&acc_dev->lock);
 230   if (acc_dev->is_initialized)
 231     {
 232       gomp_mutex_unlock (&acc_dev->lock);
 233       gomp_fatal ("device already active");
 234     }
 235
 236   gomp_init_device (acc_dev);
 237   gomp_mutex_unlock (&acc_dev->lock);
 238
 239   return base_dev;
 240 }
 241
 242 /* ACC_DEVICE_LOCK must be held before calling this function.  */
 243
 244 static void
 245 acc_shutdown_1 (acc_device_t d)
 246 {
 247   struct gomp_device_descr *base_dev;
 248   struct goacc_thread *walk;
 249   int ndevs, i;
 250   bool devices_active = false;
 251
 252   /* Get the base device for this device type.  */
 253   base_dev = resolve_device (d, true);
 254
 255   ndevs = base_dev->get_num_devices_func ();
 256
 257   /* Unload all the devices of this type that have been opened.  */
 258   for (i = 0; i < ndevs; i++)
 259     {
 260       struct gomp_device_descr *acc_dev = &base_dev[i];
 261
 262       gomp_mutex_lock (&acc_dev->lock);
 263       gomp_unload_device (acc_dev);
 264       gomp_mutex_unlock (&acc_dev->lock);
 265     }
 266
 267   gomp_mutex_lock (&goacc_thread_lock);
 268
 269   /* Free target-specific TLS data and close all devices.  */
 270   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 271     {
 272       if (walk->target_tls)
 273         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 274
 275       walk->target_tls = NULL;
 276
 277       /* This would mean the user is shutting down OpenACC in the middle of an
 278          "acc data" pragma.  Likely not intentional.  */
 279       if (walk->mapped_data)
 280         {
 281           gomp_mutex_unlock (&goacc_thread_lock);
 282           gomp_fatal ("shutdown in 'acc data' region");
 283         }
 284
 285       /* Similarly, if this happens then user code has done something weird.  */
 286       if (walk->saved_bound_dev)
 287         {
 288           gomp_mutex_unlock (&goacc_thread_lock);
 289           gomp_fatal ("shutdown during host fallback");
 290         }
 291
 292       if (walk->dev)
 293         {
 294           gomp_mutex_lock (&walk->dev->lock);
 295           gomp_free_memmap (&walk->dev->mem_map);
 296           gomp_mutex_unlock (&walk->dev->lock);
 297
 298           walk->dev = NULL;
 299           walk->base_dev = NULL;
 300         }
 301     }
 302
 303   gomp_mutex_unlock (&goacc_thread_lock);
 304
 305
 306   /* Close all the devices of this type that have been opened.  */
 307   for (i = 0; i < ndevs; i++)
 308     {
 309       struct gomp_device_descr *acc_dev = &base_dev[i];
 310       gomp_mutex_lock (&acc_dev->lock);
 311       if (acc_dev->is_initialized)
 312         {
 313           devices_active = true;
 314           gomp_fini_device (acc_dev);
 315         }
 316       gomp_mutex_unlock (&acc_dev->lock);
 317     }
 318
 319   if (!devices_active)
 320     gomp_fatal ("no device initialized");
 321 }
 322
 323 static struct goacc_thread *
 324 goacc_new_thread (void)
 325 {
 326   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 327
 328 #if defined HAVE_TLS || defined USE_EMUTLS
 329   goacc_tls_data = thr;
 330 #else
 331   pthread_setspecific (goacc_tls_key, thr);
 332 #endif
 333
 334   pthread_setspecific (goacc_cleanup_key, thr);
 335
 336   gomp_mutex_lock (&goacc_thread_lock);
 337   thr->next = goacc_threads;
 338   goacc_threads = thr;
 339   gomp_mutex_unlock (&goacc_thread_lock);
 340
 341   return thr;
 342 }
 343
 344 static void
 345 goacc_destroy_thread (void *data)
 346 {
 347   struct goacc_thread *thr = data, *walk, *prev;
 348
 349   gomp_mutex_lock (&goacc_thread_lock);
 350
 351   if (thr)
 352     {
 353       struct gomp_device_descr *acc_dev = thr->dev;
 354
 355       if (acc_dev && thr->target_tls)
 356         {
 357           acc_dev->openacc.destroy_thread_data_func (thr->target_tls);
 358           thr->target_tls = NULL;
 359         }
 360
 361       assert (!thr->mapped_data);
 362
 363       /* Remove from thread list.  */
 364       for (prev = NULL, walk = goacc_threads; walk;
 365            prev = walk, walk = walk->next)
 366         if (walk == thr)
 367           {
 368             if (prev == NULL)
 369               goacc_threads = walk->next;
 370             else
 371               prev->next = walk->next;
 372
 373             free (thr);
 374
 375             break;
 376           }
 377
 378       assert (walk);
 379     }
 380
 381   gomp_mutex_unlock (&goacc_thread_lock);
 382 }
 383
 384 /* Use the ORD'th device instance for the current host thread (or -1 for the
 385    current global default).  The device (and the runtime) must be initialised
 386    before calling this function.  */
 387
 388 void
 389 goacc_attach_host_thread_to_device (int ord)
 390 {
 391   struct goacc_thread *thr = goacc_thread ();
 392   struct gomp_device_descr *acc_dev = NULL, *base_dev = NULL;
 393   int num_devices;
 394
 395   if (thr && thr->dev && (thr->dev->target_id == ord || ord < 0))
 396     return;
 397
 398   if (ord < 0)
 399     ord = goacc_device_num;
 400
 401   /* Decide which type of device to use.  If the current thread has a device
 402      type already (e.g. set by acc_set_device_type), use that, else use the
 403      global default.  */
 404   if (thr && thr->base_dev)
 405     base_dev = thr->base_dev;
 406   else
 407     {
 408       assert (cached_base_dev);
 409       base_dev = cached_base_dev;
 410     }
 411
 412   num_devices = base_dev->get_num_devices_func ();
 413   if (num_devices <= 0 || ord >= num_devices)
 414     acc_dev_num_out_of_range (acc_device_type (base_dev->type), ord,
 415                               num_devices);
 416
 417   if (!thr)
 418     thr = goacc_new_thread ();
 419
 420   thr->base_dev = base_dev;
 421   thr->dev = acc_dev = &base_dev[ord];
 422   thr->saved_bound_dev = NULL;
 423   thr->mapped_data = NULL;
 424
 425   thr->target_tls
 426     = acc_dev->openacc.create_thread_data_func (ord);
 427
 428   acc_dev->openacc.async_set_async_func (acc_async_sync);
 429 }
 430
 431 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 432    init/shutdown is per-process or per-thread.  We choose per-process.  */
 433
 434 void
 435 acc_init (acc_device_t d)
 436 {
 437   if (!cached_base_dev)
 438     gomp_init_targets_once ();
 439
 440   gomp_mutex_lock (&acc_device_lock);
 441
 442   cached_base_dev = acc_init_1 (d);
 443
 444   gomp_mutex_unlock (&acc_device_lock);
 445
 446   goacc_attach_host_thread_to_device (-1);
 447 }
 448
 449 ialias (acc_init)
 450
 451 void
 452 acc_shutdown (acc_device_t d)
 453 {
 454   gomp_mutex_lock (&acc_device_lock);
 455
 456   acc_shutdown_1 (d);
 457
 458   gomp_mutex_unlock (&acc_device_lock);
 459 }
 460
 461 ialias (acc_shutdown)
 462
 463 int
 464 acc_get_num_devices (acc_device_t d)
 465 {
 466   int n = 0;
 467   struct gomp_device_descr *acc_dev;
 468
 469   if (d == acc_device_none)
 470     return 0;
 471
 472   gomp_init_targets_once ();
 473
 474   gomp_mutex_lock (&acc_device_lock);
 475   acc_dev = resolve_device (d, false);
 476   gomp_mutex_unlock (&acc_device_lock);
 477
 478   if (!acc_dev)
 479     return 0;
 480
 481   n = acc_dev->get_num_devices_func ();
 482   if (n < 0)
 483     n = 0;
 484
 485   return n;
 486 }
 487
 488 ialias (acc_get_num_devices)
 489
 490 /* Set the device type for the current thread only (using the current global
 491    default device number), initialising that device if necessary.  Also set the
 492    default device type for new threads to D.  */
 493
 494 void
 495 acc_set_device_type (acc_device_t d)
 496 {
 497   struct gomp_device_descr *base_dev, *acc_dev;
 498   struct goacc_thread *thr = goacc_thread ();
 499
 500   gomp_mutex_lock (&acc_device_lock);
 501
 502   if (!cached_base_dev)
 503     gomp_init_targets_once ();
 504
 505   cached_base_dev = base_dev = resolve_device (d, true);
 506   acc_dev = &base_dev[goacc_device_num];
 507
 508   gomp_mutex_lock (&acc_dev->lock);
 509   if (!acc_dev->is_initialized)
 510     gomp_init_device (acc_dev);
 511   gomp_mutex_unlock (&acc_dev->lock);
 512
 513   gomp_mutex_unlock (&acc_device_lock);
 514
 515   /* We're changing device type: invalidate the current thread's dev and
 516      base_dev pointers.  */
 517   if (thr && thr->base_dev != base_dev)
 518     {
 519       thr->base_dev = thr->dev = NULL;
 520       if (thr->mapped_data)
 521         gomp_fatal ("acc_set_device_type in 'acc data' region");
 522     }
 523
 524   goacc_attach_host_thread_to_device (-1);
 525 }
 526
 527 ialias (acc_set_device_type)
 528
 529 acc_device_t
 530 acc_get_device_type (void)
 531 {
 532   acc_device_t res = acc_device_none;
 533   struct gomp_device_descr *dev;
 534   struct goacc_thread *thr = goacc_thread ();
 535
 536   if (thr && thr->base_dev)
 537     res = acc_device_type (thr->base_dev->type);
 538   else
 539     {
 540       gomp_init_targets_once ();
 541
 542       gomp_mutex_lock (&acc_device_lock);
 543       dev = resolve_device (acc_device_default, true);
 544       gomp_mutex_unlock (&acc_device_lock);
 545       res = acc_device_type (dev->type);
 546     }
 547
 548   assert (res != acc_device_default
 549           && res != acc_device_not_host);
 550
 551   return res;
 552 }
 553
 554 ialias (acc_get_device_type)
 555
 556 int
 557 acc_get_device_num (acc_device_t d)
 558 {
 559   const struct gomp_device_descr *dev;
 560   struct goacc_thread *thr = goacc_thread ();
 561
 562   if (d >= _ACC_device_hwm)
 563     gomp_fatal ("unknown device type %u", (unsigned) d);
 564
 565   if (!cached_base_dev)
 566     gomp_init_targets_once ();
 567
 568   gomp_mutex_lock (&acc_device_lock);
 569   dev = resolve_device (d, true);
 570   gomp_mutex_unlock (&acc_device_lock);
 571
 572   if (thr && thr->base_dev == dev && thr->dev)
 573     return thr->dev->target_id;
 574
 575   return goacc_device_num;
 576 }
 577
 578 ialias (acc_get_device_num)
 579
 580 void
 581 acc_set_device_num (int ord, acc_device_t d)
 582 {
 583   struct gomp_device_descr *base_dev, *acc_dev;
 584   int num_devices;
 585
 586   if (!cached_base_dev)
 587     gomp_init_targets_once ();
 588
 589   if (ord < 0)
 590     ord = goacc_device_num;
 591
 592   if ((int) d == 0)
 593     /* Set whatever device is being used by the current host thread to use
 594        device instance ORD.  It's unclear if this is supposed to affect other
 595        host threads too (OpenACC 2.0 (3.2.4) acc_set_device_num).  */
 596     goacc_attach_host_thread_to_device (ord);
 597   else
 598     {
 599       gomp_mutex_lock (&acc_device_lock);
 600
 601       cached_base_dev = base_dev = resolve_device (d, true);
 602
 603       num_devices = base_dev->get_num_devices_func ();
 604
 605       if (num_devices <= 0 || ord >= num_devices)
 606         acc_dev_num_out_of_range (d, ord, num_devices);
 607
 608       acc_dev = &base_dev[ord];
 609
 610       gomp_mutex_lock (&acc_dev->lock);
 611       if (!acc_dev->is_initialized)
 612         gomp_init_device (acc_dev);
 613       gomp_mutex_unlock (&acc_dev->lock);
 614
 615       gomp_mutex_unlock (&acc_device_lock);
 616
 617       goacc_attach_host_thread_to_device (ord);
 618     }
 619
 620   goacc_device_num = ord;
 621 }
 622
 623 ialias (acc_set_device_num)
 624
 625 int
 626 acc_on_device (acc_device_t dev)
 627 {
 628   struct goacc_thread *thr = goacc_thread ();
 629
 630   /* We only want to appear to be the "host_nonshm" plugin from "offloaded"
 631      code -- i.e. within a parallel region.  Test a flag set by the
 632      openacc_parallel hook of the host_nonshm plugin to determine that.  */
 633   if (acc_get_device_type () == acc_device_host_nonshm
 634       && thr && thr->target_tls
 635       && ((struct nonshm_thread *)thr->target_tls)->nonshm_exec)
 636     return dev == acc_device_host_nonshm || dev == acc_device_not_host;
 637
 638   /* For OpenACC, libgomp is only built for the host, so this is sufficient.  */
 639   return dev == acc_device_host || dev == acc_device_none;
 640 }
 641
 642 ialias (acc_on_device)
 643
 644 attribute_hidden void
 645 goacc_runtime_initialize (void)
 646 {
 647   gomp_mutex_init (&acc_device_lock);
 648
 649 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 650   pthread_key_create (&goacc_tls_key, NULL);
 651 #endif
 652
 653   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 654
 655   cached_base_dev = NULL;
 656
 657   goacc_threads = NULL;
 658   gomp_mutex_init (&goacc_thread_lock);
 659 }
 660
 661 /* Compiler helper functions */
 662
 663 attribute_hidden void
 664 goacc_save_and_set_bind (acc_device_t d)
 665 {
 666   struct goacc_thread *thr = goacc_thread ();
 667
 668   assert (!thr->saved_bound_dev);
 669
 670   thr->saved_bound_dev = thr->dev;
 671   thr->dev = dispatchers[d];
 672 }
 673
 674 attribute_hidden void
 675 goacc_restore_bind (void)
 676 {
 677   struct goacc_thread *thr = goacc_thread ();
 678
 679   thr->dev = thr->saved_bound_dev;
 680   thr->saved_bound_dev = NULL;
 681 }
 682
 683 /* This is called from any OpenACC support function that may need to implicitly
 684    initialize the libgomp runtime, either globally or from a new host thread.
 685    On exit "goacc_thread" will return a valid & populated thread block.  */
 686
 687 attribute_hidden void
 688 goacc_lazy_initialize (void)
 689 {
 690   struct goacc_thread *thr = goacc_thread ();
 691
 692   if (thr && thr->dev)
 693     return;
 694
 695   if (!cached_base_dev)
 696     acc_init (acc_device_default);
 697   else
 698     goacc_attach_host_thread_to_device (-1);
 699 }