libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2014 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "libgomp_target.h"
  31 #include "oacc-int.h"
  32 #include "openacc.h"
  33 #include <assert.h>
  34 #include <stdlib.h>
  35 #include <strings.h>
  36 #include <stdbool.h>
  37 #include <stdio.h>
  38 #include <string.h>
  39
  40 static gomp_mutex_t acc_device_lock;
  41
  42 /* The dispatch table for the current accelerator device.  This is global, so
  43    you can only have one type of device open at any given time in a program.
  44    This is the "base" device in that several devices that use the same
  45    dispatch table may be active concurrently: this one (the "zeroth") is used
  46    for overall initialisation/shutdown, and other instances -- not necessarily
  47    including this one -- may be opened and closed once the base device has
  48    been initialized.  */
  49 struct gomp_device_descr const *base_dev;
  50
  51 #if defined HAVE_TLS || defined USE_EMUTLS
  52 __thread struct goacc_thread *goacc_tls_data;
  53 #else
  54 pthread_key_t goacc_tls_key;
  55 #endif
  56 static pthread_key_t goacc_cleanup_key;
  57
  58 /* Current dispatcher, and how it was initialized */
  59 static acc_device_t init_key = _ACC_device_hwm;
  60
  61 static struct goacc_thread *goacc_threads;
  62 static gomp_mutex_t goacc_thread_lock;
  63
  64 /* An array of dispatchers for device types, indexed by the type.  This array
  65    only references "base" devices, and other instances of the same type are
  66    found by simply indexing from each such device (which are stored linearly,
  67    grouped by device in target.c:devices).  */
  68 static struct gomp_device_descr const *dispatchers[_ACC_device_hwm] = { 0 };
  69
  70 attribute_hidden void
  71 goacc_register (struct gomp_device_descr const *disp)
  72 {
  73   /* Only register the 0th device here.  */
  74   if (disp->target_id != 0)
  75     return;
  76
  77   gomp_mutex_lock (&acc_device_lock);
  78
  79   assert (acc_device_type (disp->type) != acc_device_none
  80           && acc_device_type (disp->type) != acc_device_default
  81           && acc_device_type (disp->type) != acc_device_not_host);
  82   assert (!dispatchers[disp->type]);
  83   dispatchers[disp->type] = disp;
  84
  85   gomp_mutex_unlock (&acc_device_lock);
  86 }
  87
  88 /* OpenACC names some things a little differently.  */
  89
  90 static const char *
  91 get_openacc_name (const char *name)
  92 {
  93   if (strcmp (name, "nvptx") == 0)
  94     return "nvidia";
  95   else
  96     return name;
  97 }
  98
  99 static struct gomp_device_descr const *
 100 resolve_device (acc_device_t d)
 101 {
 102   acc_device_t d_arg = d;
 103
 104   switch (d)
 105     {
 106     case acc_device_default:
 107       {
 108         if (goacc_device_type)
 109           {
 110             /* Lookup the named device.  */
 111             while (++d != _ACC_device_hwm)
 112               if (dispatchers[d]
 113                   && !strcasecmp (goacc_device_type,
 114                                   get_openacc_name (dispatchers[d]->name))
 115                   && dispatchers[d]->get_num_devices_func () > 0)
 116                 goto found;
 117
 118             gomp_fatal ("device type %s not supported", goacc_device_type);
 119           }
 120
 121         /* No default device specified, so start scanning for any non-host
 122            device that is available.  */
 123         d = acc_device_not_host;
 124       }
 125       /* FALLTHROUGH */
 126
 127     case acc_device_not_host:
 128       /* Find the first available device after acc_device_not_host.  */
 129       while (++d != _ACC_device_hwm)
 130         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 131           goto found;
 132       if (d_arg == acc_device_default)
 133         {
 134           d = acc_device_host;
 135           goto found;
 136         }
 137       gomp_fatal ("no device found");
 138       break;
 139
 140     case acc_device_host:
 141       break;
 142
 143     default:
 144       if (d > _ACC_device_hwm)
 145         gomp_fatal ("device %u out of range", (unsigned)d);
 146       break;
 147     }
 148  found:
 149
 150   assert (d != acc_device_none
 151           && d != acc_device_default
 152           && d != acc_device_not_host);
 153
 154   return dispatchers[d];
 155 }
 156
 157 /* This is called when plugins have been initialized, and serves to call
 158    (indirectly) the target's device_init hook.  Calling multiple times without
 159    an intervening acc_shutdown_1 call is an error.  */
 160
 161 static struct gomp_device_descr const *
 162 acc_init_1 (acc_device_t d)
 163 {
 164   struct gomp_device_descr const *acc_dev;
 165
 166   acc_dev = resolve_device (d);
 167
 168   if (!acc_dev || acc_dev->get_num_devices_func () <= 0)
 169     gomp_fatal ("device %u not supported", (unsigned)d);
 170
 171   if (acc_dev->is_initialized)
 172     gomp_fatal ("device already active");
 173
 174   /* We need to remember what we were intialized as, to check shutdown etc.  */
 175   init_key = d;
 176
 177   gomp_init_device ((struct gomp_device_descr *) acc_dev);
 178
 179   return acc_dev;
 180 }
 181
 182 static struct goacc_thread *
 183 goacc_new_thread (void)
 184 {
 185   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 186
 187 #if defined HAVE_TLS || defined USE_EMUTLS
 188   goacc_tls_data = thr;
 189 #else
 190   pthread_setspecific (goacc_tls_key, thr);
 191 #endif
 192
 193   pthread_setspecific (goacc_cleanup_key, thr);
 194
 195   gomp_mutex_lock (&goacc_thread_lock);
 196   thr->next = goacc_threads;
 197   goacc_threads = thr;
 198   gomp_mutex_unlock (&goacc_thread_lock);
 199
 200   return thr;
 201 }
 202
 203 static void
 204 goacc_destroy_thread (void *data)
 205 {
 206   struct goacc_thread *thr = data, *walk, *prev;
 207
 208   gomp_mutex_lock (&goacc_thread_lock);
 209
 210   if (thr)
 211     {
 212       if (base_dev && thr->target_tls)
 213         {
 214           base_dev->openacc.destroy_thread_data_func (thr->target_tls);
 215           thr->target_tls = NULL;
 216         }
 217
 218       assert (!thr->mapped_data);
 219
 220       /* Remove from thread list.  */
 221       for (prev = NULL, walk = goacc_threads; walk;
 222            prev = walk, walk = walk->next)
 223         if (walk == thr)
 224           {
 225             if (prev == NULL)
 226               goacc_threads = walk->next;
 227             else
 228               prev->next = walk->next;
 229
 230             free (thr);
 231
 232             break;
 233           }
 234
 235       assert (walk);
 236     }
 237
 238   gomp_mutex_unlock (&goacc_thread_lock);
 239 }
 240
 241 /* Open the ORD'th device of the currently-active type (base_dev must be
 242    initialised before calling).  If ORD is < 0, open the default-numbered
 243    device (set by the ACC_DEVICE_NUM environment variable or a call to
 244    acc_set_device_num), or leave any currently-opened device as is.  "Opening"
 245    consists of calling the device's open_device_func hook, and setting up
 246    thread-local data (maybe allocating, then initializing with information
 247    pertaining to the newly-opened or previously-opened device).  */
 248
 249 static void
 250 lazy_open (int ord)
 251 {
 252   struct goacc_thread *thr = goacc_thread ();
 253   struct gomp_device_descr *acc_dev;
 254
 255   if (thr && thr->dev)
 256     {
 257       assert (ord < 0 || ord == thr->dev->target_id);
 258       return;
 259     }
 260
 261   assert (base_dev);
 262
 263   if (ord < 0)
 264     ord = goacc_device_num;
 265
 266   /* The OpenACC 2.0 spec leaves the runtime's behaviour when an out-of-range
 267      device is requested as implementation-defined (4.2 ACC_DEVICE_NUM).
 268      We choose to raise an error in such a case.  */
 269   if (ord >= base_dev->get_num_devices_func ())
 270     gomp_fatal ("device %u does not exist", ord);
 271
 272   if (!thr)
 273     thr = goacc_new_thread ();
 274
 275   acc_dev = thr->dev = (struct gomp_device_descr *) &base_dev[ord];
 276
 277   assert (acc_dev->target_id == ord);
 278
 279   thr->saved_bound_dev = NULL;
 280   thr->mapped_data = NULL;
 281
 282   if (!acc_dev->target_data)
 283     acc_dev->target_data = acc_dev->openacc.open_device_func (ord);
 284
 285   thr->target_tls
 286     = acc_dev->openacc.create_thread_data_func (acc_dev->target_data);
 287
 288   acc_dev->openacc.async_set_async_func (acc_async_sync);
 289
 290   if (!acc_dev->mem_map.is_initialized)
 291     gomp_init_tables (acc_dev, &acc_dev->mem_map);
 292 }
 293
 294 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 295    init/shutdown is per-process or per-thread.  We choose per-process.  */
 296
 297 void
 298 acc_init (acc_device_t d)
 299 {
 300   if (!base_dev)
 301     gomp_init_targets_once ();
 302
 303   gomp_mutex_lock (&acc_device_lock);
 304
 305   base_dev = acc_init_1 (d);
 306
 307   lazy_open (-1);
 308
 309   gomp_mutex_unlock (&acc_device_lock);
 310 }
 311
 312 ialias (acc_init)
 313
 314 static void
 315 acc_shutdown_1 (acc_device_t d)
 316 {
 317   struct goacc_thread *walk;
 318
 319   /* We don't check whether d matches the actual device found, because
 320      OpenACC 2.0 (3.2.12) says the parameters to the init and this
 321      call must match (for the shutdown call anyway, it's silent on
 322      others).  */
 323
 324   if (!base_dev)
 325     gomp_fatal ("no device initialized");
 326   if (d != init_key)
 327     gomp_fatal ("device %u(%u) is initialized",
 328                 (unsigned) init_key, (unsigned) base_dev->type);
 329
 330   gomp_mutex_lock (&goacc_thread_lock);
 331
 332   /* Free target-specific TLS data and close all devices.  */
 333   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 334     {
 335       if (walk->target_tls)
 336         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 337
 338       walk->target_tls = NULL;
 339
 340       /* This would mean the user is shutting down OpenACC in the middle of an
 341          "acc data" pragma.  Likely not intentional.  */
 342       if (walk->mapped_data)
 343         gomp_fatal ("shutdown in 'acc data' region");
 344
 345       if (walk->dev)
 346         {
 347           if (walk->dev->openacc.close_device_func (walk->dev->target_data) < 0)
 348             gomp_fatal ("failed to close device");
 349
 350           walk->dev->target_data = NULL;
 351
 352           gomp_free_memmap (walk->dev);
 353
 354           walk->dev = NULL;
 355         }
 356     }
 357
 358   gomp_mutex_unlock (&goacc_thread_lock);
 359
 360   gomp_fini_device ((struct gomp_device_descr *) base_dev);
 361
 362   base_dev = NULL;
 363 }
 364
 365 void
 366 acc_shutdown (acc_device_t d)
 367 {
 368   gomp_mutex_lock (&acc_device_lock);
 369
 370   acc_shutdown_1 (d);
 371
 372   gomp_mutex_unlock (&acc_device_lock);
 373 }
 374
 375 ialias (acc_shutdown)
 376
 377 /* This function is called after plugins have been initialized.  It deals with
 378    the "base" device, and is used to prepare the runtime for dealing with a
 379    number of such devices (as implemented by some particular plugin).  If the
 380    argument device type D matches a previous call to the function, return the
 381    current base device, else shut the old device down and re-initialize with
 382    the new device type.  */
 383
 384 static struct gomp_device_descr const *
 385 lazy_init (acc_device_t d)
 386 {
 387   if (base_dev)
 388     {
 389       /* Re-initializing the same device, do nothing.  */
 390       if (d == init_key)
 391         return base_dev;
 392
 393       acc_shutdown_1 (init_key);
 394     }
 395
 396   assert (!base_dev);
 397
 398   return acc_init_1 (d);
 399 }
 400
 401 /* Ensure that plugins are loaded, initialize and open the (default-numbered)
 402    device.  */
 403
 404 static void
 405 lazy_init_and_open (acc_device_t d)
 406 {
 407   if (!base_dev)
 408     gomp_init_targets_once ();
 409
 410   gomp_mutex_lock (&acc_device_lock);
 411
 412   base_dev = lazy_init (d);
 413
 414   lazy_open (-1);
 415
 416   gomp_mutex_unlock (&acc_device_lock);
 417 }
 418
 419 int
 420 acc_get_num_devices (acc_device_t d)
 421 {
 422   int n = 0;
 423   struct gomp_device_descr const *acc_dev;
 424
 425   if (d == acc_device_none)
 426     return 0;
 427
 428   if (!base_dev)
 429     gomp_init_targets_once ();
 430
 431   acc_dev = resolve_device (d);
 432   if (!acc_dev)
 433     return 0;
 434
 435   n = acc_dev->get_num_devices_func ();
 436   if (n < 0)
 437     n = 0;
 438
 439   return n;
 440 }
 441
 442 ialias (acc_get_num_devices)
 443
 444 void
 445 acc_set_device_type (acc_device_t d)
 446 {
 447   lazy_init_and_open (d);
 448 }
 449
 450 ialias (acc_set_device_type)
 451
 452 acc_device_t
 453 acc_get_device_type (void)
 454 {
 455   acc_device_t res = acc_device_none;
 456   const struct gomp_device_descr *dev;
 457
 458   if (base_dev)
 459     res = acc_device_type (base_dev->type);
 460   else
 461     {
 462       gomp_init_targets_once ();
 463
 464       dev = resolve_device (acc_device_default);
 465       res = acc_device_type (dev->type);
 466     }
 467
 468   assert (res != acc_device_default
 469           && res != acc_device_not_host);
 470
 471   return res;
 472 }
 473
 474 ialias (acc_get_device_type)
 475
 476 int
 477 acc_get_device_num (acc_device_t d)
 478 {
 479   const struct gomp_device_descr *dev;
 480   int num;
 481
 482   if (d >= _ACC_device_hwm)
 483     gomp_fatal ("device %u out of range", (unsigned)d);
 484
 485   if (!base_dev)
 486     gomp_init_targets_once ();
 487
 488   dev = resolve_device (d);
 489   if (!dev)
 490     gomp_fatal ("no devices of type %u", d);
 491
 492   /* We might not have called lazy_open for this host thread yet, in which case
 493      the get_device_num_func hook will return -1.  */
 494   num = dev->openacc.get_device_num_func ();
 495   if (num < 0)
 496     num = goacc_device_num;
 497
 498   return num;
 499 }
 500
 501 ialias (acc_get_device_num)
 502
 503 void
 504 acc_set_device_num (int n, acc_device_t d)
 505 {
 506   const struct gomp_device_descr *dev;
 507   int num_devices;
 508
 509   if (!base_dev)
 510     gomp_init_targets_once ();
 511
 512   if ((int) d == 0)
 513     {
 514       int i;
 515
 516       /* A device setting of zero sets all device types on the system to use
 517          the Nth instance of that device type.  Only attempt it for initialized
 518          devices though.  */
 519       for (i = acc_device_not_host + 1; i < _ACC_device_hwm; i++)
 520         {
 521           dev = resolve_device (d);
 522           if (dev && dev->is_initialized)
 523             dev->openacc.set_device_num_func (n);
 524         }
 525
 526       /* ...and for future calls to acc_init/acc_set_device_type, etc.  */
 527       goacc_device_num = n;
 528     }
 529   else
 530     {
 531       struct goacc_thread *thr = goacc_thread ();
 532
 533       gomp_mutex_lock (&acc_device_lock);
 534
 535       base_dev = lazy_init (d);
 536
 537       num_devices = base_dev->get_num_devices_func ();
 538
 539       if (n >= num_devices)
 540         gomp_fatal ("device %u out of range", n);
 541
 542       /* If we're changing the device number, de-associate this thread with
 543          the device (but don't close the device, since it may be in use by
 544          other threads).  */
 545       if (thr && thr->dev && n != thr->dev->target_id)
 546         thr->dev = NULL;
 547
 548       lazy_open (n);
 549
 550       gomp_mutex_unlock (&acc_device_lock);
 551     }
 552 }
 553
 554 ialias (acc_set_device_num)
 555
 556 int
 557 acc_on_device (acc_device_t dev)
 558 {
 559   struct goacc_thread *thr = goacc_thread ();
 560
 561   if (thr && thr->dev
 562       && acc_device_type (thr->dev->type) == acc_device_host_nonshm)
 563     return dev == acc_device_host_nonshm || dev == acc_device_not_host;
 564
 565   /* Just rely on the compiler builtin.  */
 566   return __builtin_acc_on_device (dev);
 567 }
 568 ialias (acc_on_device)
 569
 570 attribute_hidden void
 571 goacc_runtime_initialize (void)
 572 {
 573   gomp_mutex_init (&acc_device_lock);
 574
 575 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 576   pthread_key_create (&goacc_tls_key, NULL);
 577 #endif
 578
 579   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 580
 581   base_dev = NULL;
 582
 583   goacc_threads = NULL;
 584   gomp_mutex_init (&goacc_thread_lock);
 585 }
 586
 587 /* Compiler helper functions */
 588
 589 attribute_hidden void
 590 goacc_save_and_set_bind (acc_device_t d)
 591 {
 592   struct goacc_thread *thr = goacc_thread ();
 593
 594   assert (!thr->saved_bound_dev);
 595
 596   thr->saved_bound_dev = thr->dev;
 597   thr->dev = (struct gomp_device_descr *) dispatchers[d];
 598 }
 599
 600 attribute_hidden void
 601 goacc_restore_bind (void)
 602 {
 603   struct goacc_thread *thr = goacc_thread ();
 604
 605   thr->dev = thr->saved_bound_dev;
 606   thr->saved_bound_dev = NULL;
 607 }
 608
 609 /* This is called from any OpenACC support function that may need to implicitly
 610    initialize the libgomp runtime.  On exit all such initialization will have
 611    been done, and both the global ACC_dev and the per-host-thread ACC_memmap
 612    pointers will be valid.  */
 613
 614 attribute_hidden void
 615 goacc_lazy_initialize (void)
 616 {
 617   struct goacc_thread *thr = goacc_thread ();
 618
 619   if (thr && thr->dev)
 620     return;
 621
 622   if (!base_dev)
 623     lazy_init_and_open (acc_device_default);
 624   else
 625     {
 626       gomp_mutex_lock (&acc_device_lock);
 627       lazy_open (-1);
 628       gomp_mutex_unlock (&acc_device_lock);
 629     }
 630 }