libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2015 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "oacc-int.h"
  31 #include "openacc.h"
  32 #include <assert.h>
  33 #include <stdlib.h>
  34 #include <strings.h>
  35 #include <stdbool.h>
  36 #include <string.h>
  37
  38 /* This lock is used to protect access to cached_base_dev, dispatchers and
  39    the (abstract) initialisation state of attached offloading devices.  */
  40
  41 static gomp_mutex_t acc_device_lock;
  42
  43 /* A cached version of the dispatcher for the global "current" accelerator type,
  44    e.g. used as the default when creating new host threads.  This is the
  45    device-type equivalent of goacc_device_num (which specifies which device to
  46    use out of potentially several of the same type).  If there are several
  47    devices of a given type, this points at the first one.  */
  48
  49 static struct gomp_device_descr *cached_base_dev = NULL;
  50
  51 #if defined HAVE_TLS || defined USE_EMUTLS
  52 __thread struct goacc_thread *goacc_tls_data;
  53 #else
  54 pthread_key_t goacc_tls_key;
  55 #endif
  56 static pthread_key_t goacc_cleanup_key;
  57
  58 static struct goacc_thread *goacc_threads;
  59 static gomp_mutex_t goacc_thread_lock;
  60
  61 /* An array of dispatchers for device types, indexed by the type.  This array
  62    only references "base" devices, and other instances of the same type are
  63    found by simply indexing from each such device (which are stored linearly,
  64    grouped by device in target.c:devices).  */
  65 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  66
  67 attribute_hidden void
  68 goacc_register (struct gomp_device_descr *disp)
  69 {
  70   /* Only register the 0th device here.  */
  71   if (disp->target_id != 0)
  72     return;
  73
  74   gomp_mutex_lock (&acc_device_lock);
  75
  76   assert (acc_device_type (disp->type) != acc_device_none
  77           && acc_device_type (disp->type) != acc_device_default
  78           && acc_device_type (disp->type) != acc_device_not_host);
  79   assert (!dispatchers[disp->type]);
  80   dispatchers[disp->type] = disp;
  81
  82   gomp_mutex_unlock (&acc_device_lock);
  83 }
  84
  85 /* OpenACC names some things a little differently.  */
  86
  87 static const char *
  88 get_openacc_name (const char *name)
  89 {
  90   if (strcmp (name, "nvptx") == 0)
  91     return "nvidia";
  92   else
  93     return name;
  94 }
  95
  96 static const char *
  97 name_of_acc_device_t (enum acc_device_t type)
  98 {
  99   switch (type)
 100     {
 101     case acc_device_none: return "none";
 102     case acc_device_default: return "default";
 103     case acc_device_host: return "host";
 104     case acc_device_host_nonshm: return "host_nonshm";
 105     case acc_device_not_host: return "not_host";
 106     case acc_device_nvidia: return "nvidia";
 107     default: gomp_fatal ("unknown device type %u", (unsigned) type);
 108     }
 109 }
 110
 111 /* ACC_DEVICE_LOCK should be held before calling this function.  */
 112
 113 static struct gomp_device_descr *
 114 resolve_device (acc_device_t d)
 115 {
 116   acc_device_t d_arg = d;
 117
 118   switch (d)
 119     {
 120     case acc_device_default:
 121       {
 122         if (goacc_device_type)
 123           {
 124             /* Lookup the named device.  */
 125             while (++d != _ACC_device_hwm)
 126               if (dispatchers[d]
 127                   && !strcasecmp (goacc_device_type,
 128                                   get_openacc_name (dispatchers[d]->name))
 129                   && dispatchers[d]->get_num_devices_func () > 0)
 130                 goto found;
 131
 132             gomp_fatal ("device type %s not supported", goacc_device_type);
 133           }
 134
 135         /* No default device specified, so start scanning for any non-host
 136            device that is available.  */
 137         d = acc_device_not_host;
 138       }
 139       /* FALLTHROUGH */
 140
 141     case acc_device_not_host:
 142       /* Find the first available device after acc_device_not_host.  */
 143       while (++d != _ACC_device_hwm)
 144         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 145           goto found;
 146       if (d_arg == acc_device_default)
 147         {
 148           d = acc_device_host;
 149           goto found;
 150         }
 151       gomp_fatal ("no device found");
 152       break;
 153
 154     case acc_device_host:
 155       break;
 156
 157     default:
 158       if (d > _ACC_device_hwm)
 159         gomp_fatal ("device %u out of range", (unsigned)d);
 160       break;
 161     }
 162  found:
 163
 164   assert (d != acc_device_none
 165           && d != acc_device_default
 166           && d != acc_device_not_host);
 167
 168   return dispatchers[d];
 169 }
 170
 171 /* This is called when plugins have been initialized, and serves to call
 172    (indirectly) the target's device_init hook.  Calling multiple times without
 173    an intervening acc_shutdown_1 call is an error.  ACC_DEVICE_LOCK should be
 174    held before calling this function.  */
 175
 176 static struct gomp_device_descr *
 177 acc_init_1 (acc_device_t d)
 178 {
 179   struct gomp_device_descr *base_dev, *acc_dev;
 180   int ndevs;
 181
 182   base_dev = resolve_device (d);
 183
 184   ndevs = base_dev->get_num_devices_func ();
 185
 186   if (!base_dev || ndevs <= 0 || goacc_device_num >= ndevs)
 187     gomp_fatal ("device %s not supported", name_of_acc_device_t (d));
 188
 189   acc_dev = &base_dev[goacc_device_num];
 190
 191   gomp_mutex_lock (&acc_dev->lock);
 192   if (acc_dev->is_initialized)
 193     {
 194       gomp_mutex_unlock (&acc_dev->lock);
 195       gomp_fatal ("device already active");
 196     }
 197
 198   gomp_init_device (acc_dev);
 199   gomp_mutex_unlock (&acc_dev->lock);
 200
 201   return base_dev;
 202 }
 203
 204 /* ACC_DEVICE_LOCK should be held before calling this function.  */
 205
 206 static void
 207 acc_shutdown_1 (acc_device_t d)
 208 {
 209   struct gomp_device_descr *base_dev;
 210   struct goacc_thread *walk;
 211   int ndevs, i;
 212   bool devices_active = false;
 213
 214   /* Get the base device for this device type.  */
 215   base_dev = resolve_device (d);
 216
 217   if (!base_dev)
 218     gomp_fatal ("device %s not supported", name_of_acc_device_t (d));
 219
 220   gomp_mutex_lock (&goacc_thread_lock);
 221
 222   /* Free target-specific TLS data and close all devices.  */
 223   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 224     {
 225       if (walk->target_tls)
 226         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 227
 228       walk->target_tls = NULL;
 229
 230       /* This would mean the user is shutting down OpenACC in the middle of an
 231          "acc data" pragma.  Likely not intentional.  */
 232       if (walk->mapped_data)
 233         {
 234           gomp_mutex_unlock (&goacc_thread_lock);
 235           gomp_fatal ("shutdown in 'acc data' region");
 236         }
 237
 238       /* Similarly, if this happens then user code has done something weird.  */
 239       if (walk->saved_bound_dev)
 240         {
 241           gomp_mutex_unlock (&goacc_thread_lock);
 242           gomp_fatal ("shutdown during host fallback");
 243         }
 244
 245       if (walk->dev)
 246         {
 247           gomp_mutex_lock (&walk->dev->lock);
 248           gomp_free_memmap (&walk->dev->mem_map);
 249           gomp_mutex_unlock (&walk->dev->lock);
 250
 251           walk->dev = NULL;
 252           walk->base_dev = NULL;
 253         }
 254     }
 255
 256   gomp_mutex_unlock (&goacc_thread_lock);
 257
 258   ndevs = base_dev->get_num_devices_func ();
 259
 260   /* Close all the devices of this type that have been opened.  */
 261   for (i = 0; i < ndevs; i++)
 262     {
 263       struct gomp_device_descr *acc_dev = &base_dev[i];
 264       gomp_mutex_lock (&acc_dev->lock);
 265       if (acc_dev->is_initialized)
 266         {
 267           devices_active = true;
 268           gomp_fini_device (acc_dev);
 269         }
 270       gomp_mutex_unlock (&acc_dev->lock);
 271     }
 272
 273   if (!devices_active)
 274     gomp_fatal ("no device initialized");
 275 }
 276
 277 static struct goacc_thread *
 278 goacc_new_thread (void)
 279 {
 280   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 281
 282 #if defined HAVE_TLS || defined USE_EMUTLS
 283   goacc_tls_data = thr;
 284 #else
 285   pthread_setspecific (goacc_tls_key, thr);
 286 #endif
 287
 288   pthread_setspecific (goacc_cleanup_key, thr);
 289
 290   gomp_mutex_lock (&goacc_thread_lock);
 291   thr->next = goacc_threads;
 292   goacc_threads = thr;
 293   gomp_mutex_unlock (&goacc_thread_lock);
 294
 295   return thr;
 296 }
 297
 298 static void
 299 goacc_destroy_thread (void *data)
 300 {
 301   struct goacc_thread *thr = data, *walk, *prev;
 302
 303   gomp_mutex_lock (&goacc_thread_lock);
 304
 305   if (thr)
 306     {
 307       struct gomp_device_descr *acc_dev = thr->dev;
 308
 309       if (acc_dev && thr->target_tls)
 310         {
 311           acc_dev->openacc.destroy_thread_data_func (thr->target_tls);
 312           thr->target_tls = NULL;
 313         }
 314
 315       assert (!thr->mapped_data);
 316
 317       /* Remove from thread list.  */
 318       for (prev = NULL, walk = goacc_threads; walk;
 319            prev = walk, walk = walk->next)
 320         if (walk == thr)
 321           {
 322             if (prev == NULL)
 323               goacc_threads = walk->next;
 324             else
 325               prev->next = walk->next;
 326
 327             free (thr);
 328
 329             break;
 330           }
 331
 332       assert (walk);
 333     }
 334
 335   gomp_mutex_unlock (&goacc_thread_lock);
 336 }
 337
 338 /* Use the ORD'th device instance for the current host thread (or -1 for the
 339    current global default).  The device (and the runtime) must be initialised
 340    before calling this function.  */
 341
 342 void
 343 goacc_attach_host_thread_to_device (int ord)
 344 {
 345   struct goacc_thread *thr = goacc_thread ();
 346   struct gomp_device_descr *acc_dev = NULL, *base_dev = NULL;
 347   int num_devices;
 348
 349   if (thr && thr->dev && (thr->dev->target_id == ord || ord < 0))
 350     return;
 351
 352   if (ord < 0)
 353     ord = goacc_device_num;
 354
 355   /* Decide which type of device to use.  If the current thread has a device
 356      type already (e.g. set by acc_set_device_type), use that, else use the
 357      global default.  */
 358   if (thr && thr->base_dev)
 359     base_dev = thr->base_dev;
 360   else
 361     {
 362       assert (cached_base_dev);
 363       base_dev = cached_base_dev;
 364     }
 365
 366   num_devices = base_dev->get_num_devices_func ();
 367   if (num_devices <= 0 || ord >= num_devices)
 368     gomp_fatal ("device %u out of range", ord);
 369
 370   if (!thr)
 371     thr = goacc_new_thread ();
 372
 373   thr->base_dev = base_dev;
 374   thr->dev = acc_dev = &base_dev[ord];
 375   thr->saved_bound_dev = NULL;
 376   thr->mapped_data = NULL;
 377
 378   thr->target_tls
 379     = acc_dev->openacc.create_thread_data_func (ord);
 380
 381   acc_dev->openacc.async_set_async_func (acc_async_sync);
 382 }
 383
 384 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 385    init/shutdown is per-process or per-thread.  We choose per-process.  */
 386
 387 void
 388 acc_init (acc_device_t d)
 389 {
 390   if (!cached_base_dev)
 391     gomp_init_targets_once ();
 392
 393   gomp_mutex_lock (&acc_device_lock);
 394
 395   cached_base_dev = acc_init_1 (d);
 396
 397   gomp_mutex_unlock (&acc_device_lock);
 398
 399   goacc_attach_host_thread_to_device (-1);
 400 }
 401
 402 ialias (acc_init)
 403
 404 void
 405 acc_shutdown (acc_device_t d)
 406 {
 407   gomp_mutex_lock (&acc_device_lock);
 408
 409   acc_shutdown_1 (d);
 410
 411   gomp_mutex_unlock (&acc_device_lock);
 412 }
 413
 414 ialias (acc_shutdown)
 415
 416 int
 417 acc_get_num_devices (acc_device_t d)
 418 {
 419   int n = 0;
 420   struct gomp_device_descr *acc_dev;
 421
 422   if (d == acc_device_none)
 423     return 0;
 424
 425   gomp_init_targets_once ();
 426
 427   gomp_mutex_lock (&acc_device_lock);
 428   acc_dev = resolve_device (d);
 429   gomp_mutex_unlock (&acc_device_lock);
 430
 431   if (!acc_dev)
 432     return 0;
 433
 434   n = acc_dev->get_num_devices_func ();
 435   if (n < 0)
 436     n = 0;
 437
 438   return n;
 439 }
 440
 441 ialias (acc_get_num_devices)
 442
 443 /* Set the device type for the current thread only (using the current global
 444    default device number), initialising that device if necessary.  Also set the
 445    default device type for new threads to D.  */
 446
 447 void
 448 acc_set_device_type (acc_device_t d)
 449 {
 450   struct gomp_device_descr *base_dev, *acc_dev;
 451   struct goacc_thread *thr = goacc_thread ();
 452
 453   gomp_mutex_lock (&acc_device_lock);
 454
 455   if (!cached_base_dev)
 456     gomp_init_targets_once ();
 457
 458   cached_base_dev = base_dev = resolve_device (d);
 459   acc_dev = &base_dev[goacc_device_num];
 460
 461   gomp_mutex_lock (&acc_dev->lock);
 462   if (!acc_dev->is_initialized)
 463     gomp_init_device (acc_dev);
 464   gomp_mutex_unlock (&acc_dev->lock);
 465
 466   gomp_mutex_unlock (&acc_device_lock);
 467
 468   /* We're changing device type: invalidate the current thread's dev and
 469      base_dev pointers.  */
 470   if (thr && thr->base_dev != base_dev)
 471     {
 472       thr->base_dev = thr->dev = NULL;
 473       if (thr->mapped_data)
 474         gomp_fatal ("acc_set_device_type in 'acc data' region");
 475     }
 476
 477   goacc_attach_host_thread_to_device (-1);
 478 }
 479
 480 ialias (acc_set_device_type)
 481
 482 acc_device_t
 483 acc_get_device_type (void)
 484 {
 485   acc_device_t res = acc_device_none;
 486   struct gomp_device_descr *dev;
 487   struct goacc_thread *thr = goacc_thread ();
 488
 489   if (thr && thr->base_dev)
 490     res = acc_device_type (thr->base_dev->type);
 491   else
 492     {
 493       gomp_init_targets_once ();
 494
 495       gomp_mutex_lock (&acc_device_lock);
 496       dev = resolve_device (acc_device_default);
 497       gomp_mutex_unlock (&acc_device_lock);
 498       res = acc_device_type (dev->type);
 499     }
 500
 501   assert (res != acc_device_default
 502           && res != acc_device_not_host);
 503
 504   return res;
 505 }
 506
 507 ialias (acc_get_device_type)
 508
 509 int
 510 acc_get_device_num (acc_device_t d)
 511 {
 512   const struct gomp_device_descr *dev;
 513   struct goacc_thread *thr = goacc_thread ();
 514
 515   if (d >= _ACC_device_hwm)
 516     gomp_fatal ("device %u out of range", (unsigned)d);
 517
 518   if (!cached_base_dev)
 519     gomp_init_targets_once ();
 520
 521   gomp_mutex_lock (&acc_device_lock);
 522   dev = resolve_device (d);
 523   gomp_mutex_unlock (&acc_device_lock);
 524   if (!dev)
 525     gomp_fatal ("device %s not supported", name_of_acc_device_t (d));
 526
 527   if (thr && thr->base_dev == dev && thr->dev)
 528     return thr->dev->target_id;
 529
 530   return goacc_device_num;
 531 }
 532
 533 ialias (acc_get_device_num)
 534
 535 void
 536 acc_set_device_num (int ord, acc_device_t d)
 537 {
 538   struct gomp_device_descr *base_dev, *acc_dev;
 539   int num_devices;
 540
 541   if (!cached_base_dev)
 542     gomp_init_targets_once ();
 543
 544   if (ord < 0)
 545     ord = goacc_device_num;
 546
 547   if ((int) d == 0)
 548     /* Set whatever device is being used by the current host thread to use
 549        device instance ORD.  It's unclear if this is supposed to affect other
 550        host threads too (OpenACC 2.0 (3.2.4) acc_set_device_num).  */
 551     goacc_attach_host_thread_to_device (ord);
 552   else
 553     {
 554       gomp_mutex_lock (&acc_device_lock);
 555
 556       cached_base_dev = base_dev = resolve_device (d);
 557
 558       num_devices = base_dev->get_num_devices_func ();
 559
 560       if (ord >= num_devices)
 561         gomp_fatal ("device %u out of range", ord);
 562
 563       acc_dev = &base_dev[ord];
 564
 565       gomp_mutex_lock (&acc_dev->lock);
 566       if (!acc_dev->is_initialized)
 567         gomp_init_device (acc_dev);
 568       gomp_mutex_unlock (&acc_dev->lock);
 569
 570       gomp_mutex_unlock (&acc_device_lock);
 571
 572       goacc_attach_host_thread_to_device (ord);
 573     }
 574
 575   goacc_device_num = ord;
 576 }
 577
 578 ialias (acc_set_device_num)
 579
 580 int
 581 acc_on_device (acc_device_t dev)
 582 {
 583   if (acc_get_device_type () == acc_device_host_nonshm)
 584     return dev == acc_device_host_nonshm || dev == acc_device_not_host;
 585
 586   /* Just rely on the compiler builtin.  */
 587   return __builtin_acc_on_device (dev);
 588 }
 589
 590 ialias (acc_on_device)
 591
 592 attribute_hidden void
 593 goacc_runtime_initialize (void)
 594 {
 595   gomp_mutex_init (&acc_device_lock);
 596
 597 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 598   pthread_key_create (&goacc_tls_key, NULL);
 599 #endif
 600
 601   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 602
 603   cached_base_dev = NULL;
 604
 605   goacc_threads = NULL;
 606   gomp_mutex_init (&goacc_thread_lock);
 607 }
 608
 609 /* Compiler helper functions */
 610
 611 attribute_hidden void
 612 goacc_save_and_set_bind (acc_device_t d)
 613 {
 614   struct goacc_thread *thr = goacc_thread ();
 615
 616   assert (!thr->saved_bound_dev);
 617
 618   thr->saved_bound_dev = thr->dev;
 619   thr->dev = dispatchers[d];
 620 }
 621
 622 attribute_hidden void
 623 goacc_restore_bind (void)
 624 {
 625   struct goacc_thread *thr = goacc_thread ();
 626
 627   thr->dev = thr->saved_bound_dev;
 628   thr->saved_bound_dev = NULL;
 629 }
 630
 631 /* This is called from any OpenACC support function that may need to implicitly
 632    initialize the libgomp runtime, either globally or from a new host thread.
 633    On exit "goacc_thread" will return a valid & populated thread block.  */
 634
 635 attribute_hidden void
 636 goacc_lazy_initialize (void)
 637 {
 638   struct goacc_thread *thr = goacc_thread ();
 639
 640   if (thr && thr->dev)
 641     return;
 642
 643   if (!cached_base_dev)
 644     acc_init (acc_device_default);
 645   else
 646     goacc_attach_host_thread_to_device (-1);
 647 }