libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2015 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "oacc-int.h"
  31 #include "openacc.h"
  32 #include <assert.h>
  33 #include <stdlib.h>
  34 #include <strings.h>
  35 #include <stdbool.h>
  36 #include <string.h>
  37
  38 static gomp_mutex_t acc_device_lock;
  39
  40 /* A cached version of the dispatcher for the global "current" accelerator type,
  41    e.g. used as the default when creating new host threads.  This is the
  42    device-type equivalent of goacc_device_num (which specifies which device to
  43    use out of potentially several of the same type).  If there are several
  44    devices of a given type, this points at the first one.  */
  45
  46 static struct gomp_device_descr *cached_base_dev = NULL;
  47
  48 #if defined HAVE_TLS || defined USE_EMUTLS
  49 __thread struct goacc_thread *goacc_tls_data;
  50 #else
  51 pthread_key_t goacc_tls_key;
  52 #endif
  53 static pthread_key_t goacc_cleanup_key;
  54
  55 static struct goacc_thread *goacc_threads;
  56 static gomp_mutex_t goacc_thread_lock;
  57
  58 /* An array of dispatchers for device types, indexed by the type.  This array
  59    only references "base" devices, and other instances of the same type are
  60    found by simply indexing from each such device (which are stored linearly,
  61    grouped by device in target.c:devices).  */
  62 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  63
  64 attribute_hidden void
  65 goacc_register (struct gomp_device_descr *disp)
  66 {
  67   /* Only register the 0th device here.  */
  68   if (disp->target_id != 0)
  69     return;
  70
  71   gomp_mutex_lock (&acc_device_lock);
  72
  73   assert (acc_device_type (disp->type) != acc_device_none
  74           && acc_device_type (disp->type) != acc_device_default
  75           && acc_device_type (disp->type) != acc_device_not_host);
  76   assert (!dispatchers[disp->type]);
  77   dispatchers[disp->type] = disp;
  78
  79   gomp_mutex_unlock (&acc_device_lock);
  80 }
  81
  82 /* OpenACC names some things a little differently.  */
  83
  84 static const char *
  85 get_openacc_name (const char *name)
  86 {
  87   if (strcmp (name, "nvptx") == 0)
  88     return "nvidia";
  89   else
  90     return name;
  91 }
  92
  93 static const char *
  94 name_of_acc_device_t (enum acc_device_t type)
  95 {
  96   switch (type)
  97     {
  98     case acc_device_none: return "none";
  99     case acc_device_default: return "default";
 100     case acc_device_host: return "host";
 101     case acc_device_host_nonshm: return "host_nonshm";
 102     case acc_device_not_host: return "not_host";
 103     case acc_device_nvidia: return "nvidia";
 104     default: gomp_fatal ("unknown device type %u", (unsigned) type);
 105     }
 106 }
 107
 108 static struct gomp_device_descr *
 109 resolve_device (acc_device_t d)
 110 {
 111   acc_device_t d_arg = d;
 112
 113   switch (d)
 114     {
 115     case acc_device_default:
 116       {
 117         if (goacc_device_type)
 118           {
 119             /* Lookup the named device.  */
 120             while (++d != _ACC_device_hwm)
 121               if (dispatchers[d]
 122                   && !strcasecmp (goacc_device_type,
 123                                   get_openacc_name (dispatchers[d]->name))
 124                   && dispatchers[d]->get_num_devices_func () > 0)
 125                 goto found;
 126
 127             gomp_fatal ("device type %s not supported", goacc_device_type);
 128           }
 129
 130         /* No default device specified, so start scanning for any non-host
 131            device that is available.  */
 132         d = acc_device_not_host;
 133       }
 134       /* FALLTHROUGH */
 135
 136     case acc_device_not_host:
 137       /* Find the first available device after acc_device_not_host.  */
 138       while (++d != _ACC_device_hwm)
 139         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 140           goto found;
 141       if (d_arg == acc_device_default)
 142         {
 143           d = acc_device_host;
 144           goto found;
 145         }
 146       gomp_fatal ("no device found");
 147       break;
 148
 149     case acc_device_host:
 150       break;
 151
 152     default:
 153       if (d > _ACC_device_hwm)
 154         gomp_fatal ("device %u out of range", (unsigned)d);
 155       break;
 156     }
 157  found:
 158
 159   assert (d != acc_device_none
 160           && d != acc_device_default
 161           && d != acc_device_not_host);
 162
 163   return dispatchers[d];
 164 }
 165
 166 /* This is called when plugins have been initialized, and serves to call
 167    (indirectly) the target's device_init hook.  Calling multiple times without
 168    an intervening acc_shutdown_1 call is an error.  */
 169
 170 static struct gomp_device_descr *
 171 acc_init_1 (acc_device_t d)
 172 {
 173   struct gomp_device_descr *base_dev, *acc_dev;
 174   int ndevs;
 175
 176   base_dev = resolve_device (d);
 177
 178   ndevs = base_dev->get_num_devices_func ();
 179
 180   if (!base_dev || ndevs <= 0 || goacc_device_num >= ndevs)
 181     gomp_fatal ("device %s not supported", name_of_acc_device_t (d));
 182
 183   acc_dev = &base_dev[goacc_device_num];
 184
 185   if (acc_dev->is_initialized)
 186     gomp_fatal ("device already active");
 187
 188   gomp_init_device (acc_dev);
 189
 190   return base_dev;
 191 }
 192
 193 static void
 194 acc_shutdown_1 (acc_device_t d)
 195 {
 196   struct gomp_device_descr *base_dev;
 197   struct goacc_thread *walk;
 198   int ndevs, i;
 199   bool devices_active = false;
 200
 201   /* Get the base device for this device type.  */
 202   base_dev = resolve_device (d);
 203
 204   if (!base_dev)
 205     gomp_fatal ("device %s not supported", name_of_acc_device_t (d));
 206
 207   gomp_mutex_lock (&goacc_thread_lock);
 208
 209   /* Free target-specific TLS data and close all devices.  */
 210   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 211     {
 212       if (walk->target_tls)
 213         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 214
 215       walk->target_tls = NULL;
 216
 217       /* This would mean the user is shutting down OpenACC in the middle of an
 218          "acc data" pragma.  Likely not intentional.  */
 219       if (walk->mapped_data)
 220         gomp_fatal ("shutdown in 'acc data' region");
 221
 222       /* Similarly, if this happens then user code has done something weird.  */
 223       if (walk->saved_bound_dev)
 224         gomp_fatal ("shutdown during host fallback");
 225
 226       if (walk->dev)
 227         {
 228           gomp_mutex_lock (&walk->dev->lock);
 229           gomp_free_memmap (&walk->dev->mem_map);
 230           gomp_mutex_unlock (&walk->dev->lock);
 231
 232           walk->dev = NULL;
 233           walk->base_dev = NULL;
 234         }
 235     }
 236
 237   gomp_mutex_unlock (&goacc_thread_lock);
 238
 239   ndevs = base_dev->get_num_devices_func ();
 240
 241   /* Close all the devices of this type that have been opened.  */
 242   for (i = 0; i < ndevs; i++)
 243     {
 244       struct gomp_device_descr *acc_dev = &base_dev[i];
 245       if (acc_dev->is_initialized)
 246         {
 247           devices_active = true;
 248           gomp_fini_device (acc_dev);
 249         }
 250     }
 251
 252   if (!devices_active)
 253     gomp_fatal ("no device initialized");
 254 }
 255
 256 static struct goacc_thread *
 257 goacc_new_thread (void)
 258 {
 259   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 260
 261 #if defined HAVE_TLS || defined USE_EMUTLS
 262   goacc_tls_data = thr;
 263 #else
 264   pthread_setspecific (goacc_tls_key, thr);
 265 #endif
 266
 267   pthread_setspecific (goacc_cleanup_key, thr);
 268
 269   gomp_mutex_lock (&goacc_thread_lock);
 270   thr->next = goacc_threads;
 271   goacc_threads = thr;
 272   gomp_mutex_unlock (&goacc_thread_lock);
 273
 274   return thr;
 275 }
 276
 277 static void
 278 goacc_destroy_thread (void *data)
 279 {
 280   struct goacc_thread *thr = data, *walk, *prev;
 281
 282   gomp_mutex_lock (&goacc_thread_lock);
 283
 284   if (thr)
 285     {
 286       struct gomp_device_descr *acc_dev = thr->dev;
 287
 288       if (acc_dev && thr->target_tls)
 289         {
 290           acc_dev->openacc.destroy_thread_data_func (thr->target_tls);
 291           thr->target_tls = NULL;
 292         }
 293
 294       assert (!thr->mapped_data);
 295
 296       /* Remove from thread list.  */
 297       for (prev = NULL, walk = goacc_threads; walk;
 298            prev = walk, walk = walk->next)
 299         if (walk == thr)
 300           {
 301             if (prev == NULL)
 302               goacc_threads = walk->next;
 303             else
 304               prev->next = walk->next;
 305
 306             free (thr);
 307
 308             break;
 309           }
 310
 311       assert (walk);
 312     }
 313
 314   gomp_mutex_unlock (&goacc_thread_lock);
 315 }
 316
 317 /* Use the ORD'th device instance for the current host thread (or -1 for the
 318    current global default).  The device (and the runtime) must be initialised
 319    before calling this function.  */
 320
 321 void
 322 goacc_attach_host_thread_to_device (int ord)
 323 {
 324   struct goacc_thread *thr = goacc_thread ();
 325   struct gomp_device_descr *acc_dev = NULL, *base_dev = NULL;
 326   int num_devices;
 327
 328   if (thr && thr->dev && (thr->dev->target_id == ord || ord < 0))
 329     return;
 330
 331   if (ord < 0)
 332     ord = goacc_device_num;
 333
 334   /* Decide which type of device to use.  If the current thread has a device
 335      type already (e.g. set by acc_set_device_type), use that, else use the
 336      global default.  */
 337   if (thr && thr->base_dev)
 338     base_dev = thr->base_dev;
 339   else
 340     {
 341       assert (cached_base_dev);
 342       base_dev = cached_base_dev;
 343     }
 344
 345   num_devices = base_dev->get_num_devices_func ();
 346   if (num_devices <= 0 || ord >= num_devices)
 347     gomp_fatal ("device %u out of range", ord);
 348
 349   if (!thr)
 350     thr = goacc_new_thread ();
 351
 352   thr->base_dev = base_dev;
 353   thr->dev = acc_dev = &base_dev[ord];
 354   thr->saved_bound_dev = NULL;
 355   thr->mapped_data = NULL;
 356
 357   thr->target_tls
 358     = acc_dev->openacc.create_thread_data_func (ord);
 359
 360   acc_dev->openacc.async_set_async_func (acc_async_sync);
 361 }
 362
 363 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 364    init/shutdown is per-process or per-thread.  We choose per-process.  */
 365
 366 void
 367 acc_init (acc_device_t d)
 368 {
 369   if (!cached_base_dev)
 370     gomp_init_targets_once ();
 371
 372   gomp_mutex_lock (&acc_device_lock);
 373
 374   cached_base_dev = acc_init_1 (d);
 375
 376   gomp_mutex_unlock (&acc_device_lock);
 377
 378   goacc_attach_host_thread_to_device (-1);
 379 }
 380
 381 ialias (acc_init)
 382
 383 void
 384 acc_shutdown (acc_device_t d)
 385 {
 386   gomp_mutex_lock (&acc_device_lock);
 387
 388   acc_shutdown_1 (d);
 389
 390   gomp_mutex_unlock (&acc_device_lock);
 391 }
 392
 393 ialias (acc_shutdown)
 394
 395 int
 396 acc_get_num_devices (acc_device_t d)
 397 {
 398   int n = 0;
 399   struct gomp_device_descr *acc_dev;
 400
 401   if (d == acc_device_none)
 402     return 0;
 403
 404   gomp_init_targets_once ();
 405
 406   acc_dev = resolve_device (d);
 407   if (!acc_dev)
 408     return 0;
 409
 410   n = acc_dev->get_num_devices_func ();
 411   if (n < 0)
 412     n = 0;
 413
 414   return n;
 415 }
 416
 417 ialias (acc_get_num_devices)
 418
 419 /* Set the device type for the current thread only (using the current global
 420    default device number), initialising that device if necessary.  Also set the
 421    default device type for new threads to D.  */
 422
 423 void
 424 acc_set_device_type (acc_device_t d)
 425 {
 426   struct gomp_device_descr *base_dev, *acc_dev;
 427   struct goacc_thread *thr = goacc_thread ();
 428
 429   gomp_mutex_lock (&acc_device_lock);
 430
 431   if (!cached_base_dev)
 432     gomp_init_targets_once ();
 433
 434   cached_base_dev = base_dev = resolve_device (d);
 435   acc_dev = &base_dev[goacc_device_num];
 436
 437   if (!acc_dev->is_initialized)
 438     gomp_init_device (acc_dev);
 439
 440   gomp_mutex_unlock (&acc_device_lock);
 441
 442   /* We're changing device type: invalidate the current thread's dev and
 443      base_dev pointers.  */
 444   if (thr && thr->base_dev != base_dev)
 445     {
 446       thr->base_dev = thr->dev = NULL;
 447       if (thr->mapped_data)
 448         gomp_fatal ("acc_set_device_type in 'acc data' region");
 449     }
 450
 451   goacc_attach_host_thread_to_device (-1);
 452 }
 453
 454 ialias (acc_set_device_type)
 455
 456 acc_device_t
 457 acc_get_device_type (void)
 458 {
 459   acc_device_t res = acc_device_none;
 460   struct gomp_device_descr *dev;
 461   struct goacc_thread *thr = goacc_thread ();
 462
 463   if (thr && thr->base_dev)
 464     res = acc_device_type (thr->base_dev->type);
 465   else
 466     {
 467       gomp_init_targets_once ();
 468
 469       dev = resolve_device (acc_device_default);
 470       res = acc_device_type (dev->type);
 471     }
 472
 473   assert (res != acc_device_default
 474           && res != acc_device_not_host);
 475
 476   return res;
 477 }
 478
 479 ialias (acc_get_device_type)
 480
 481 int
 482 acc_get_device_num (acc_device_t d)
 483 {
 484   const struct gomp_device_descr *dev;
 485   struct goacc_thread *thr = goacc_thread ();
 486
 487   if (d >= _ACC_device_hwm)
 488     gomp_fatal ("device %u out of range", (unsigned)d);
 489
 490   if (!cached_base_dev)
 491     gomp_init_targets_once ();
 492
 493   dev = resolve_device (d);
 494   if (!dev)
 495     gomp_fatal ("device %s not supported", name_of_acc_device_t (d));
 496
 497   if (thr && thr->base_dev == dev && thr->dev)
 498     return thr->dev->target_id;
 499
 500   return goacc_device_num;
 501 }
 502
 503 ialias (acc_get_device_num)
 504
 505 void
 506 acc_set_device_num (int ord, acc_device_t d)
 507 {
 508   struct gomp_device_descr *base_dev, *acc_dev;
 509   int num_devices;
 510
 511   if (!cached_base_dev)
 512     gomp_init_targets_once ();
 513
 514   if (ord < 0)
 515     ord = goacc_device_num;
 516
 517   if ((int) d == 0)
 518     /* Set whatever device is being used by the current host thread to use
 519        device instance ORD.  It's unclear if this is supposed to affect other
 520        host threads too (OpenACC 2.0 (3.2.4) acc_set_device_num).  */
 521     goacc_attach_host_thread_to_device (ord);
 522   else
 523     {
 524       gomp_mutex_lock (&acc_device_lock);
 525
 526       cached_base_dev = base_dev = resolve_device (d);
 527
 528       num_devices = base_dev->get_num_devices_func ();
 529
 530       if (ord >= num_devices)
 531         gomp_fatal ("device %u out of range", ord);
 532
 533       acc_dev = &base_dev[ord];
 534
 535       if (!acc_dev->is_initialized)
 536         gomp_init_device (acc_dev);
 537
 538       gomp_mutex_unlock (&acc_device_lock);
 539
 540       goacc_attach_host_thread_to_device (ord);
 541     }
 542
 543   goacc_device_num = ord;
 544 }
 545
 546 ialias (acc_set_device_num)
 547
 548 int
 549 acc_on_device (acc_device_t dev)
 550 {
 551   if (acc_get_device_type () == acc_device_host_nonshm)
 552     return dev == acc_device_host_nonshm || dev == acc_device_not_host;
 553
 554   /* Just rely on the compiler builtin.  */
 555   return __builtin_acc_on_device (dev);
 556 }
 557
 558 ialias (acc_on_device)
 559
 560 attribute_hidden void
 561 goacc_runtime_initialize (void)
 562 {
 563   gomp_mutex_init (&acc_device_lock);
 564
 565 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 566   pthread_key_create (&goacc_tls_key, NULL);
 567 #endif
 568
 569   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 570
 571   cached_base_dev = NULL;
 572
 573   goacc_threads = NULL;
 574   gomp_mutex_init (&goacc_thread_lock);
 575 }
 576
 577 /* Compiler helper functions */
 578
 579 attribute_hidden void
 580 goacc_save_and_set_bind (acc_device_t d)
 581 {
 582   struct goacc_thread *thr = goacc_thread ();
 583
 584   assert (!thr->saved_bound_dev);
 585
 586   thr->saved_bound_dev = thr->dev;
 587   thr->dev = dispatchers[d];
 588 }
 589
 590 attribute_hidden void
 591 goacc_restore_bind (void)
 592 {
 593   struct goacc_thread *thr = goacc_thread ();
 594
 595   thr->dev = thr->saved_bound_dev;
 596   thr->saved_bound_dev = NULL;
 597 }
 598
 599 /* This is called from any OpenACC support function that may need to implicitly
 600    initialize the libgomp runtime, either globally or from a new host thread.
 601    On exit "goacc_thread" will return a valid & populated thread block.  */
 602
 603 attribute_hidden void
 604 goacc_lazy_initialize (void)
 605 {
 606   struct goacc_thread *thr = goacc_thread ();
 607
 608   if (thr && thr->dev)
 609     return;
 610
 611   if (!cached_base_dev)
 612     acc_init (acc_device_default);
 613   else
 614     goacc_attach_host_thread_to_device (-1);
 615 }