libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2015 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "oacc-int.h"
  31 #include "openacc.h"
  32 #include "plugin/plugin-host.h"
  33 #include <assert.h>
  34 #include <stdlib.h>
  35 #include <strings.h>
  36 #include <stdbool.h>
  37 #include <string.h>
  38
  39 /* This lock is used to protect access to cached_base_dev, dispatchers and
  40    the (abstract) initialisation state of attached offloading devices.  */
  41
  42 static gomp_mutex_t acc_device_lock;
  43
  44 /* A cached version of the dispatcher for the global "current" accelerator type,
  45    e.g. used as the default when creating new host threads.  This is the
  46    device-type equivalent of goacc_device_num (which specifies which device to
  47    use out of potentially several of the same type).  If there are several
  48    devices of a given type, this points at the first one.  */
  49
  50 static struct gomp_device_descr *cached_base_dev = NULL;
  51
  52 #if defined HAVE_TLS || defined USE_EMUTLS
  53 __thread struct goacc_thread *goacc_tls_data;
  54 #else
  55 pthread_key_t goacc_tls_key;
  56 #endif
  57 static pthread_key_t goacc_cleanup_key;
  58
  59 static struct goacc_thread *goacc_threads;
  60 static gomp_mutex_t goacc_thread_lock;
  61
  62 /* An array of dispatchers for device types, indexed by the type.  This array
  63    only references "base" devices, and other instances of the same type are
  64    found by simply indexing from each such device (which are stored linearly,
  65    grouped by device in target.c:devices).  */
  66 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  67
  68 attribute_hidden void
  69 goacc_register (struct gomp_device_descr *disp)
  70 {
  71   /* Only register the 0th device here.  */
  72   if (disp->target_id != 0)
  73     return;
  74
  75   gomp_mutex_lock (&acc_device_lock);
  76
  77   assert (acc_device_type (disp->type) != acc_device_none
  78           && acc_device_type (disp->type) != acc_device_default
  79           && acc_device_type (disp->type) != acc_device_not_host);
  80   assert (!dispatchers[disp->type]);
  81   dispatchers[disp->type] = disp;
  82
  83   gomp_mutex_unlock (&acc_device_lock);
  84 }
  85
  86 /* OpenACC names some things a little differently.  */
  87
  88 static const char *
  89 get_openacc_name (const char *name)
  90 {
  91   if (strcmp (name, "nvptx") == 0)
  92     return "nvidia";
  93   else
  94     return name;
  95 }
  96
  97 static const char *
  98 name_of_acc_device_t (enum acc_device_t type)
  99 {
 100   switch (type)
 101     {
 102     case acc_device_none: return "none";
 103     case acc_device_default: return "default";
 104     case acc_device_host: return "host";
 105     case acc_device_host_nonshm: return "host_nonshm";
 106     case acc_device_not_host: return "not_host";
 107     case acc_device_nvidia: return "nvidia";
 108     default: gomp_fatal ("unknown device type %u", (unsigned) type);
 109     }
 110 }
 111
 112 /* ACC_DEVICE_LOCK must be held before calling this function.  If FAIL_IS_ERROR
 113    is true, this function raises an error if there are no devices of type D,
 114    otherwise it returns NULL in that case.  */
 115
 116 static struct gomp_device_descr *
 117 resolve_device (acc_device_t d, bool fail_is_error)
 118 {
 119   acc_device_t d_arg = d;
 120
 121   switch (d)
 122     {
 123     case acc_device_default:
 124       {
 125         if (goacc_device_type)
 126           {
 127             /* Lookup the named device.  */
 128             while (++d != _ACC_device_hwm)
 129               if (dispatchers[d]
 130                   && !strcasecmp (goacc_device_type,
 131                                   get_openacc_name (dispatchers[d]->name))
 132                   && dispatchers[d]->get_num_devices_func () > 0)
 133                 goto found;
 134
 135             if (fail_is_error)
 136               {
 137                 gomp_mutex_unlock (&acc_device_lock);
 138                 gomp_fatal ("device type %s not supported", goacc_device_type);
 139               }
 140             else
 141               return NULL;
 142           }
 143
 144         /* No default device specified, so start scanning for any non-host
 145            device that is available.  */
 146         d = acc_device_not_host;
 147       }
 148       /* FALLTHROUGH */
 149
 150     case acc_device_not_host:
 151       /* Find the first available device after acc_device_not_host.  */
 152       while (++d != _ACC_device_hwm)
 153         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 154           goto found;
 155       if (d_arg == acc_device_default)
 156         {
 157           d = acc_device_host;
 158           goto found;
 159         }
 160       if (fail_is_error)
 161         {
 162           gomp_mutex_unlock (&acc_device_lock);
 163           gomp_fatal ("no device found");
 164         }
 165       else
 166         return NULL;
 167       break;
 168
 169     case acc_device_host:
 170       break;
 171
 172     default:
 173       if (d > _ACC_device_hwm)
 174         {
 175           if (fail_is_error)
 176             goto unsupported_device;
 177           else
 178             return NULL;
 179         }
 180       break;
 181     }
 182  found:
 183
 184   assert (d != acc_device_none
 185           && d != acc_device_default
 186           && d != acc_device_not_host);
 187
 188   if (dispatchers[d] == NULL && fail_is_error)
 189     {
 190     unsupported_device:
 191       gomp_mutex_unlock (&acc_device_lock);
 192       gomp_fatal ("device type %s not supported", name_of_acc_device_t (d));
 193     }
 194
 195   return dispatchers[d];
 196 }
 197
 198 /* Emit a suitable error if no device of a particular type is available, or
 199    the given device number is out-of-range.  */
 200 static void
 201 acc_dev_num_out_of_range (acc_device_t d, int ord, int ndevs)
 202 {
 203   if (ndevs == 0)
 204     gomp_fatal ("no devices of type %s available", name_of_acc_device_t (d));
 205   else
 206     gomp_fatal ("device %u out of range", ord);
 207 }
 208
 209 /* This is called when plugins have been initialized, and serves to call
 210    (indirectly) the target's device_init hook.  Calling multiple times without
 211    an intervening acc_shutdown_1 call is an error.  ACC_DEVICE_LOCK must be
 212    held before calling this function.  */
 213
 214 static struct gomp_device_descr *
 215 acc_init_1 (acc_device_t d)
 216 {
 217   struct gomp_device_descr *base_dev, *acc_dev;
 218   int ndevs;
 219
 220   base_dev = resolve_device (d, true);
 221
 222   ndevs = base_dev->get_num_devices_func ();
 223
 224   if (ndevs <= 0 || goacc_device_num >= ndevs)
 225     acc_dev_num_out_of_range (d, goacc_device_num, ndevs);
 226
 227   acc_dev = &base_dev[goacc_device_num];
 228
 229   gomp_mutex_lock (&acc_dev->lock);
 230   if (acc_dev->is_initialized)
 231     {
 232       gomp_mutex_unlock (&acc_dev->lock);
 233       gomp_fatal ("device already active");
 234     }
 235
 236   gomp_init_device (acc_dev);
 237   gomp_mutex_unlock (&acc_dev->lock);
 238
 239   return base_dev;
 240 }
 241
 242 /* ACC_DEVICE_LOCK must be held before calling this function.  */
 243
 244 static void
 245 acc_shutdown_1 (acc_device_t d)
 246 {
 247   struct gomp_device_descr *base_dev;
 248   struct goacc_thread *walk;
 249   int ndevs, i;
 250   bool devices_active = false;
 251
 252   /* Get the base device for this device type.  */
 253   base_dev = resolve_device (d, true);
 254
 255   gomp_mutex_lock (&goacc_thread_lock);
 256
 257   /* Free target-specific TLS data and close all devices.  */
 258   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 259     {
 260       if (walk->target_tls)
 261         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 262
 263       walk->target_tls = NULL;
 264
 265       /* This would mean the user is shutting down OpenACC in the middle of an
 266          "acc data" pragma.  Likely not intentional.  */
 267       if (walk->mapped_data)
 268         {
 269           gomp_mutex_unlock (&goacc_thread_lock);
 270           gomp_fatal ("shutdown in 'acc data' region");
 271         }
 272
 273       /* Similarly, if this happens then user code has done something weird.  */
 274       if (walk->saved_bound_dev)
 275         {
 276           gomp_mutex_unlock (&goacc_thread_lock);
 277           gomp_fatal ("shutdown during host fallback");
 278         }
 279
 280       if (walk->dev)
 281         {
 282           gomp_mutex_lock (&walk->dev->lock);
 283           gomp_free_memmap (&walk->dev->mem_map);
 284           gomp_mutex_unlock (&walk->dev->lock);
 285
 286           walk->dev = NULL;
 287           walk->base_dev = NULL;
 288         }
 289     }
 290
 291   gomp_mutex_unlock (&goacc_thread_lock);
 292
 293   ndevs = base_dev->get_num_devices_func ();
 294
 295   /* Close all the devices of this type that have been opened.  */
 296   for (i = 0; i < ndevs; i++)
 297     {
 298       struct gomp_device_descr *acc_dev = &base_dev[i];
 299       gomp_mutex_lock (&acc_dev->lock);
 300       if (acc_dev->is_initialized)
 301         {
 302           devices_active = true;
 303           gomp_fini_device (acc_dev);
 304         }
 305       gomp_mutex_unlock (&acc_dev->lock);
 306     }
 307
 308   if (!devices_active)
 309     gomp_fatal ("no device initialized");
 310 }
 311
 312 static struct goacc_thread *
 313 goacc_new_thread (void)
 314 {
 315   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 316
 317 #if defined HAVE_TLS || defined USE_EMUTLS
 318   goacc_tls_data = thr;
 319 #else
 320   pthread_setspecific (goacc_tls_key, thr);
 321 #endif
 322
 323   pthread_setspecific (goacc_cleanup_key, thr);
 324
 325   gomp_mutex_lock (&goacc_thread_lock);
 326   thr->next = goacc_threads;
 327   goacc_threads = thr;
 328   gomp_mutex_unlock (&goacc_thread_lock);
 329
 330   return thr;
 331 }
 332
 333 static void
 334 goacc_destroy_thread (void *data)
 335 {
 336   struct goacc_thread *thr = data, *walk, *prev;
 337
 338   gomp_mutex_lock (&goacc_thread_lock);
 339
 340   if (thr)
 341     {
 342       struct gomp_device_descr *acc_dev = thr->dev;
 343
 344       if (acc_dev && thr->target_tls)
 345         {
 346           acc_dev->openacc.destroy_thread_data_func (thr->target_tls);
 347           thr->target_tls = NULL;
 348         }
 349
 350       assert (!thr->mapped_data);
 351
 352       /* Remove from thread list.  */
 353       for (prev = NULL, walk = goacc_threads; walk;
 354            prev = walk, walk = walk->next)
 355         if (walk == thr)
 356           {
 357             if (prev == NULL)
 358               goacc_threads = walk->next;
 359             else
 360               prev->next = walk->next;
 361
 362             free (thr);
 363
 364             break;
 365           }
 366
 367       assert (walk);
 368     }
 369
 370   gomp_mutex_unlock (&goacc_thread_lock);
 371 }
 372
 373 /* Use the ORD'th device instance for the current host thread (or -1 for the
 374    current global default).  The device (and the runtime) must be initialised
 375    before calling this function.  */
 376
 377 void
 378 goacc_attach_host_thread_to_device (int ord)
 379 {
 380   struct goacc_thread *thr = goacc_thread ();
 381   struct gomp_device_descr *acc_dev = NULL, *base_dev = NULL;
 382   int num_devices;
 383
 384   if (thr && thr->dev && (thr->dev->target_id == ord || ord < 0))
 385     return;
 386
 387   if (ord < 0)
 388     ord = goacc_device_num;
 389
 390   /* Decide which type of device to use.  If the current thread has a device
 391      type already (e.g. set by acc_set_device_type), use that, else use the
 392      global default.  */
 393   if (thr && thr->base_dev)
 394     base_dev = thr->base_dev;
 395   else
 396     {
 397       assert (cached_base_dev);
 398       base_dev = cached_base_dev;
 399     }
 400
 401   num_devices = base_dev->get_num_devices_func ();
 402   if (num_devices <= 0 || ord >= num_devices)
 403     acc_dev_num_out_of_range (acc_device_type (base_dev->type), ord,
 404                               num_devices);
 405
 406   if (!thr)
 407     thr = goacc_new_thread ();
 408
 409   thr->base_dev = base_dev;
 410   thr->dev = acc_dev = &base_dev[ord];
 411   thr->saved_bound_dev = NULL;
 412   thr->mapped_data = NULL;
 413
 414   thr->target_tls
 415     = acc_dev->openacc.create_thread_data_func (ord);
 416
 417   acc_dev->openacc.async_set_async_func (acc_async_sync);
 418 }
 419
 420 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 421    init/shutdown is per-process or per-thread.  We choose per-process.  */
 422
 423 void
 424 acc_init (acc_device_t d)
 425 {
 426   if (!cached_base_dev)
 427     gomp_init_targets_once ();
 428
 429   gomp_mutex_lock (&acc_device_lock);
 430
 431   cached_base_dev = acc_init_1 (d);
 432
 433   gomp_mutex_unlock (&acc_device_lock);
 434
 435   goacc_attach_host_thread_to_device (-1);
 436 }
 437
 438 ialias (acc_init)
 439
 440 void
 441 acc_shutdown (acc_device_t d)
 442 {
 443   gomp_mutex_lock (&acc_device_lock);
 444
 445   acc_shutdown_1 (d);
 446
 447   gomp_mutex_unlock (&acc_device_lock);
 448 }
 449
 450 ialias (acc_shutdown)
 451
 452 int
 453 acc_get_num_devices (acc_device_t d)
 454 {
 455   int n = 0;
 456   struct gomp_device_descr *acc_dev;
 457
 458   if (d == acc_device_none)
 459     return 0;
 460
 461   gomp_init_targets_once ();
 462
 463   gomp_mutex_lock (&acc_device_lock);
 464   acc_dev = resolve_device (d, false);
 465   gomp_mutex_unlock (&acc_device_lock);
 466
 467   if (!acc_dev)
 468     return 0;
 469
 470   n = acc_dev->get_num_devices_func ();
 471   if (n < 0)
 472     n = 0;
 473
 474   return n;
 475 }
 476
 477 ialias (acc_get_num_devices)
 478
 479 /* Set the device type for the current thread only (using the current global
 480    default device number), initialising that device if necessary.  Also set the
 481    default device type for new threads to D.  */
 482
 483 void
 484 acc_set_device_type (acc_device_t d)
 485 {
 486   struct gomp_device_descr *base_dev, *acc_dev;
 487   struct goacc_thread *thr = goacc_thread ();
 488
 489   gomp_mutex_lock (&acc_device_lock);
 490
 491   if (!cached_base_dev)
 492     gomp_init_targets_once ();
 493
 494   cached_base_dev = base_dev = resolve_device (d, true);
 495   acc_dev = &base_dev[goacc_device_num];
 496
 497   gomp_mutex_lock (&acc_dev->lock);
 498   if (!acc_dev->is_initialized)
 499     gomp_init_device (acc_dev);
 500   gomp_mutex_unlock (&acc_dev->lock);
 501
 502   gomp_mutex_unlock (&acc_device_lock);
 503
 504   /* We're changing device type: invalidate the current thread's dev and
 505      base_dev pointers.  */
 506   if (thr && thr->base_dev != base_dev)
 507     {
 508       thr->base_dev = thr->dev = NULL;
 509       if (thr->mapped_data)
 510         gomp_fatal ("acc_set_device_type in 'acc data' region");
 511     }
 512
 513   goacc_attach_host_thread_to_device (-1);
 514 }
 515
 516 ialias (acc_set_device_type)
 517
 518 acc_device_t
 519 acc_get_device_type (void)
 520 {
 521   acc_device_t res = acc_device_none;
 522   struct gomp_device_descr *dev;
 523   struct goacc_thread *thr = goacc_thread ();
 524
 525   if (thr && thr->base_dev)
 526     res = acc_device_type (thr->base_dev->type);
 527   else
 528     {
 529       gomp_init_targets_once ();
 530
 531       gomp_mutex_lock (&acc_device_lock);
 532       dev = resolve_device (acc_device_default, true);
 533       gomp_mutex_unlock (&acc_device_lock);
 534       res = acc_device_type (dev->type);
 535     }
 536
 537   assert (res != acc_device_default
 538           && res != acc_device_not_host);
 539
 540   return res;
 541 }
 542
 543 ialias (acc_get_device_type)
 544
 545 int
 546 acc_get_device_num (acc_device_t d)
 547 {
 548   const struct gomp_device_descr *dev;
 549   struct goacc_thread *thr = goacc_thread ();
 550
 551   if (d >= _ACC_device_hwm)
 552     gomp_fatal ("unknown device type %u", (unsigned) d);
 553
 554   if (!cached_base_dev)
 555     gomp_init_targets_once ();
 556
 557   gomp_mutex_lock (&acc_device_lock);
 558   dev = resolve_device (d, true);
 559   gomp_mutex_unlock (&acc_device_lock);
 560
 561   if (thr && thr->base_dev == dev && thr->dev)
 562     return thr->dev->target_id;
 563
 564   return goacc_device_num;
 565 }
 566
 567 ialias (acc_get_device_num)
 568
 569 void
 570 acc_set_device_num (int ord, acc_device_t d)
 571 {
 572   struct gomp_device_descr *base_dev, *acc_dev;
 573   int num_devices;
 574
 575   if (!cached_base_dev)
 576     gomp_init_targets_once ();
 577
 578   if (ord < 0)
 579     ord = goacc_device_num;
 580
 581   if ((int) d == 0)
 582     /* Set whatever device is being used by the current host thread to use
 583        device instance ORD.  It's unclear if this is supposed to affect other
 584        host threads too (OpenACC 2.0 (3.2.4) acc_set_device_num).  */
 585     goacc_attach_host_thread_to_device (ord);
 586   else
 587     {
 588       gomp_mutex_lock (&acc_device_lock);
 589
 590       cached_base_dev = base_dev = resolve_device (d, true);
 591
 592       num_devices = base_dev->get_num_devices_func ();
 593
 594       if (num_devices <= 0 || ord >= num_devices)
 595         acc_dev_num_out_of_range (d, ord, num_devices);
 596
 597       acc_dev = &base_dev[ord];
 598
 599       gomp_mutex_lock (&acc_dev->lock);
 600       if (!acc_dev->is_initialized)
 601         gomp_init_device (acc_dev);
 602       gomp_mutex_unlock (&acc_dev->lock);
 603
 604       gomp_mutex_unlock (&acc_device_lock);
 605
 606       goacc_attach_host_thread_to_device (ord);
 607     }
 608
 609   goacc_device_num = ord;
 610 }
 611
 612 ialias (acc_set_device_num)
 613
 614 int
 615 acc_on_device (acc_device_t dev)
 616 {
 617   struct goacc_thread *thr = goacc_thread ();
 618
 619   /* We only want to appear to be the "host_nonshm" plugin from "offloaded"
 620      code -- i.e. within a parallel region.  Test a flag set by the
 621      openacc_parallel hook of the host_nonshm plugin to determine that.  */
 622   if (acc_get_device_type () == acc_device_host_nonshm
 623       && thr && thr->target_tls
 624       && ((struct nonshm_thread *)thr->target_tls)->nonshm_exec)
 625     return dev == acc_device_host_nonshm || dev == acc_device_not_host;
 626
 627   /* For OpenACC, libgomp is only built for the host, so this is sufficient.  */
 628   return dev == acc_device_host || dev == acc_device_none;
 629 }
 630
 631 ialias (acc_on_device)
 632
 633 attribute_hidden void
 634 goacc_runtime_initialize (void)
 635 {
 636   gomp_mutex_init (&acc_device_lock);
 637
 638 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 639   pthread_key_create (&goacc_tls_key, NULL);
 640 #endif
 641
 642   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 643
 644   cached_base_dev = NULL;
 645
 646   goacc_threads = NULL;
 647   gomp_mutex_init (&goacc_thread_lock);
 648 }
 649
 650 /* Compiler helper functions */
 651
 652 attribute_hidden void
 653 goacc_save_and_set_bind (acc_device_t d)
 654 {
 655   struct goacc_thread *thr = goacc_thread ();
 656
 657   assert (!thr->saved_bound_dev);
 658
 659   thr->saved_bound_dev = thr->dev;
 660   thr->dev = dispatchers[d];
 661 }
 662
 663 attribute_hidden void
 664 goacc_restore_bind (void)
 665 {
 666   struct goacc_thread *thr = goacc_thread ();
 667
 668   thr->dev = thr->saved_bound_dev;
 669   thr->saved_bound_dev = NULL;
 670 }
 671
 672 /* This is called from any OpenACC support function that may need to implicitly
 673    initialize the libgomp runtime, either globally or from a new host thread.
 674    On exit "goacc_thread" will return a valid & populated thread block.  */
 675
 676 attribute_hidden void
 677 goacc_lazy_initialize (void)
 678 {
 679   struct goacc_thread *thr = goacc_thread ();
 680
 681   if (thr && thr->dev)
 682     return;
 683
 684   if (!cached_base_dev)
 685     acc_init (acc_device_default);
 686   else
 687     goacc_attach_host_thread_to_device (-1);
 688 }