libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2014 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "libgomp_target.h"
  31 #include "oacc-int.h"
  32 #include "openacc.h"
  33 #include <assert.h>
  34 #include <stdlib.h>
  35 #include <strings.h>
  36 #include <stdbool.h>
  37 #include <stdio.h>
  38 #include <string.h>
  39
  40 static gomp_mutex_t acc_device_lock;
  41
  42 /* The dispatch table for the current accelerator device.  This is global, so
  43    you can only have one type of device open at any given time in a program.
  44    This is the "base" device in that several devices that use the same
  45    dispatch table may be active concurrently: this one (the "zeroth") is used
  46    for overall initialisation/shutdown, and other instances -- not necessarily
  47    including this one -- may be opened and closed once the base device has
  48    been initialized.  */
  49 struct gomp_device_descr *base_dev;
  50
  51 #if defined HAVE_TLS || defined USE_EMUTLS
  52 __thread struct goacc_thread *goacc_tls_data;
  53 #else
  54 pthread_key_t goacc_tls_key;
  55 #endif
  56 static pthread_key_t goacc_cleanup_key;
  57
  58 /* Current dispatcher, and how it was initialized */
  59 static acc_device_t init_key = _ACC_device_hwm;
  60
  61 static struct goacc_thread *goacc_threads;
  62 static gomp_mutex_t goacc_thread_lock;
  63
  64 /* An array of dispatchers for device types, indexed by the type.  This array
  65    only references "base" devices, and other instances of the same type are
  66    found by simply indexing from each such device (which are stored linearly,
  67    grouped by device in target.c:devices).  */
  68 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  69
  70 attribute_hidden void
  71 goacc_register (struct gomp_device_descr *disp)
  72 {
  73   /* Only register the 0th device here.  */
  74   if (disp->target_id != 0)
  75     return;
  76
  77   gomp_mutex_lock (&acc_device_lock);
  78
  79   assert (acc_device_type (disp->type) != acc_device_none
  80           && acc_device_type (disp->type) != acc_device_default
  81           && acc_device_type (disp->type) != acc_device_not_host);
  82   assert (!dispatchers[disp->type]);
  83   dispatchers[disp->type] = disp;
  84
  85   gomp_mutex_unlock (&acc_device_lock);
  86 }
  87
  88 /* OpenACC names some things a little differently.  */
  89
  90 static const char *
  91 get_openacc_name (const char *name)
  92 {
  93   if (strcmp (name, "nvptx") == 0)
  94     return "nvidia";
  95   else
  96     return name;
  97 }
  98
  99 static struct gomp_device_descr *
 100 resolve_device (acc_device_t d)
 101 {
 102   acc_device_t d_arg = d;
 103
 104   switch (d)
 105     {
 106     case acc_device_default:
 107       {
 108         if (goacc_device_type)
 109           {
 110             /* Lookup the named device.  */
 111             while (++d != _ACC_device_hwm)
 112               if (dispatchers[d]
 113                   && !strcasecmp (goacc_device_type,
 114                                   get_openacc_name (dispatchers[d]->name))
 115                   && dispatchers[d]->get_num_devices_func () > 0)
 116                 goto found;
 117
 118             gomp_fatal ("device type %s not supported", goacc_device_type);
 119           }
 120
 121         /* No default device specified, so start scanning for any non-host
 122            device that is available.  */
 123         d = acc_device_not_host;
 124       }
 125       /* FALLTHROUGH */
 126
 127     case acc_device_not_host:
 128       /* Find the first available device after acc_device_not_host.  */
 129       while (++d != _ACC_device_hwm)
 130         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 131           goto found;
 132       if (d_arg == acc_device_default)
 133         {
 134           d = acc_device_host;
 135           goto found;
 136         }
 137       gomp_fatal ("no device found");
 138       break;
 139
 140     case acc_device_host:
 141       break;
 142
 143     default:
 144       if (d > _ACC_device_hwm)
 145         gomp_fatal ("device %u out of range", (unsigned)d);
 146       break;
 147     }
 148  found:
 149
 150   assert (d != acc_device_none
 151           && d != acc_device_default
 152           && d != acc_device_not_host);
 153
 154   return dispatchers[d];
 155 }
 156
 157 /* This is called when plugins have been initialized, and serves to call
 158    (indirectly) the target's device_init hook.  Calling multiple times without
 159    an intervening acc_shutdown_1 call is an error.  */
 160
 161 static struct gomp_device_descr *
 162 acc_init_1 (acc_device_t d)
 163 {
 164   struct gomp_device_descr *acc_dev;
 165
 166   acc_dev = resolve_device (d);
 167
 168   if (!acc_dev || acc_dev->get_num_devices_func () <= 0)
 169     gomp_fatal ("device %u not supported", (unsigned)d);
 170
 171   if (acc_dev->is_initialized)
 172     gomp_fatal ("device already active");
 173
 174   /* We need to remember what we were intialized as, to check shutdown etc.  */
 175   init_key = d;
 176
 177   gomp_init_device (acc_dev);
 178
 179   return acc_dev;
 180 }
 181
 182 static struct goacc_thread *
 183 goacc_new_thread (void)
 184 {
 185   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 186
 187 #if defined HAVE_TLS || defined USE_EMUTLS
 188   goacc_tls_data = thr;
 189 #else
 190   pthread_setspecific (goacc_tls_key, thr);
 191 #endif
 192
 193   pthread_setspecific (goacc_cleanup_key, thr);
 194
 195   gomp_mutex_lock (&goacc_thread_lock);
 196   thr->next = goacc_threads;
 197   goacc_threads = thr;
 198   gomp_mutex_unlock (&goacc_thread_lock);
 199
 200   return thr;
 201 }
 202
 203 static void
 204 goacc_destroy_thread (void *data)
 205 {
 206   struct goacc_thread *thr = data, *walk, *prev;
 207
 208   gomp_mutex_lock (&goacc_thread_lock);
 209
 210   if (thr)
 211     {
 212       if (base_dev && thr->target_tls)
 213         {
 214           base_dev->openacc.destroy_thread_data_func (thr->target_tls);
 215           thr->target_tls = NULL;
 216         }
 217
 218       assert (!thr->mapped_data);
 219
 220       /* Remove from thread list.  */
 221       for (prev = NULL, walk = goacc_threads; walk;
 222            prev = walk, walk = walk->next)
 223         if (walk == thr)
 224           {
 225             if (prev == NULL)
 226               goacc_threads = walk->next;
 227             else
 228               prev->next = walk->next;
 229
 230             free (thr);
 231
 232             break;
 233           }
 234
 235       assert (walk);
 236     }
 237
 238   gomp_mutex_unlock (&goacc_thread_lock);
 239 }
 240
 241 /* Open the ORD'th device of the currently-active type (base_dev must be
 242    initialised before calling).  If ORD is < 0, open the default-numbered
 243    device (set by the ACC_DEVICE_NUM environment variable or a call to
 244    acc_set_device_num), or leave any currently-opened device as is.  "Opening"
 245    consists of calling the device's open_device_func hook, and setting up
 246    thread-local data (maybe allocating, then initializing with information
 247    pertaining to the newly-opened or previously-opened device).  */
 248
 249 static void
 250 lazy_open (int ord)
 251 {
 252   struct goacc_thread *thr = goacc_thread ();
 253   struct gomp_device_descr *acc_dev;
 254
 255   if (thr && thr->dev)
 256     {
 257       assert (ord < 0 || ord == thr->dev->target_id);
 258       return;
 259     }
 260
 261   assert (base_dev);
 262
 263   if (ord < 0)
 264     ord = goacc_device_num;
 265
 266   /* The OpenACC 2.0 spec leaves the runtime's behaviour when an out-of-range
 267      device is requested as implementation-defined (4.2 ACC_DEVICE_NUM).
 268      We choose to raise an error in such a case.  */
 269   if (ord >= base_dev->get_num_devices_func ())
 270     gomp_fatal ("device %u does not exist", ord);
 271
 272   if (!thr)
 273     thr = goacc_new_thread ();
 274
 275   acc_dev = thr->dev = &base_dev[ord];
 276
 277   assert (acc_dev->target_id == ord);
 278
 279   thr->saved_bound_dev = NULL;
 280   thr->mapped_data = NULL;
 281
 282   if (!acc_dev->openacc.target_data)
 283     acc_dev->openacc.target_data = acc_dev->openacc.open_device_func (ord);
 284
 285   thr->target_tls
 286     = acc_dev->openacc.create_thread_data_func (acc_dev->openacc.target_data);
 287
 288   acc_dev->openacc.async_set_async_func (acc_async_sync);
 289
 290   if (!acc_dev->mem_map.is_initialized)
 291     gomp_init_tables (acc_dev, &acc_dev->mem_map);
 292 }
 293
 294 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 295    init/shutdown is per-process or per-thread.  We choose per-process.  */
 296
 297 void
 298 acc_init (acc_device_t d)
 299 {
 300   if (!base_dev)
 301     gomp_init_targets_once ();
 302
 303   gomp_mutex_lock (&acc_device_lock);
 304
 305   base_dev = acc_init_1 (d);
 306
 307   lazy_open (-1);
 308
 309   gomp_mutex_unlock (&acc_device_lock);
 310 }
 311
 312 ialias (acc_init)
 313
 314 static void
 315 acc_shutdown_1 (acc_device_t d)
 316 {
 317   struct goacc_thread *walk;
 318
 319   /* We don't check whether d matches the actual device found, because
 320      OpenACC 2.0 (3.2.12) says the parameters to the init and this
 321      call must match (for the shutdown call anyway, it's silent on
 322      others).  */
 323
 324   if (!base_dev)
 325     gomp_fatal ("no device initialized");
 326   if (d != init_key)
 327     gomp_fatal ("device %u(%u) is initialized",
 328                 (unsigned) init_key, (unsigned) base_dev->type);
 329
 330   gomp_mutex_lock (&goacc_thread_lock);
 331
 332   /* Free target-specific TLS data and close all devices.  */
 333   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 334     {
 335       if (walk->target_tls)
 336         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 337
 338       walk->target_tls = NULL;
 339
 340       /* This would mean the user is shutting down OpenACC in the middle of an
 341          "acc data" pragma.  Likely not intentional.  */
 342       if (walk->mapped_data)
 343         gomp_fatal ("shutdown in 'acc data' region");
 344
 345       if (walk->dev)
 346         {
 347           void *target_data = walk->dev->openacc.target_data;
 348           if (walk->dev->openacc.close_device_func (target_data) < 0)
 349             gomp_fatal ("failed to close device");
 350
 351           walk->dev->openacc.target_data = target_data = NULL;
 352
 353           gomp_free_memmap (&walk->dev->mem_map);
 354
 355           walk->dev = NULL;
 356         }
 357     }
 358
 359   gomp_mutex_unlock (&goacc_thread_lock);
 360
 361   gomp_fini_device (base_dev);
 362
 363   base_dev = NULL;
 364 }
 365
 366 void
 367 acc_shutdown (acc_device_t d)
 368 {
 369   gomp_mutex_lock (&acc_device_lock);
 370
 371   acc_shutdown_1 (d);
 372
 373   gomp_mutex_unlock (&acc_device_lock);
 374 }
 375
 376 ialias (acc_shutdown)
 377
 378 /* This function is called after plugins have been initialized.  It deals with
 379    the "base" device, and is used to prepare the runtime for dealing with a
 380    number of such devices (as implemented by some particular plugin).  If the
 381    argument device type D matches a previous call to the function, return the
 382    current base device, else shut the old device down and re-initialize with
 383    the new device type.  */
 384
 385 static struct gomp_device_descr *
 386 lazy_init (acc_device_t d)
 387 {
 388   if (base_dev)
 389     {
 390       /* Re-initializing the same device, do nothing.  */
 391       if (d == init_key)
 392         return base_dev;
 393
 394       acc_shutdown_1 (init_key);
 395     }
 396
 397   assert (!base_dev);
 398
 399   return acc_init_1 (d);
 400 }
 401
 402 /* Ensure that plugins are loaded, initialize and open the (default-numbered)
 403    device.  */
 404
 405 static void
 406 lazy_init_and_open (acc_device_t d)
 407 {
 408   if (!base_dev)
 409     gomp_init_targets_once ();
 410
 411   gomp_mutex_lock (&acc_device_lock);
 412
 413   base_dev = lazy_init (d);
 414
 415   lazy_open (-1);
 416
 417   gomp_mutex_unlock (&acc_device_lock);
 418 }
 419
 420 int
 421 acc_get_num_devices (acc_device_t d)
 422 {
 423   int n = 0;
 424   const struct gomp_device_descr *acc_dev;
 425
 426   if (d == acc_device_none)
 427     return 0;
 428
 429   if (!base_dev)
 430     gomp_init_targets_once ();
 431
 432   acc_dev = resolve_device (d);
 433   if (!acc_dev)
 434     return 0;
 435
 436   n = acc_dev->get_num_devices_func ();
 437   if (n < 0)
 438     n = 0;
 439
 440   return n;
 441 }
 442
 443 ialias (acc_get_num_devices)
 444
 445 void
 446 acc_set_device_type (acc_device_t d)
 447 {
 448   lazy_init_and_open (d);
 449 }
 450
 451 ialias (acc_set_device_type)
 452
 453 acc_device_t
 454 acc_get_device_type (void)
 455 {
 456   acc_device_t res = acc_device_none;
 457   const struct gomp_device_descr *dev;
 458
 459   if (base_dev)
 460     res = acc_device_type (base_dev->type);
 461   else
 462     {
 463       gomp_init_targets_once ();
 464
 465       dev = resolve_device (acc_device_default);
 466       res = acc_device_type (dev->type);
 467     }
 468
 469   assert (res != acc_device_default
 470           && res != acc_device_not_host);
 471
 472   return res;
 473 }
 474
 475 ialias (acc_get_device_type)
 476
 477 int
 478 acc_get_device_num (acc_device_t d)
 479 {
 480   const struct gomp_device_descr *dev;
 481   int num;
 482
 483   if (d >= _ACC_device_hwm)
 484     gomp_fatal ("device %u out of range", (unsigned)d);
 485
 486   if (!base_dev)
 487     gomp_init_targets_once ();
 488
 489   dev = resolve_device (d);
 490   if (!dev)
 491     gomp_fatal ("no devices of type %u", d);
 492
 493   /* We might not have called lazy_open for this host thread yet, in which case
 494      the get_device_num_func hook will return -1.  */
 495   num = dev->openacc.get_device_num_func ();
 496   if (num < 0)
 497     num = goacc_device_num;
 498
 499   return num;
 500 }
 501
 502 ialias (acc_get_device_num)
 503
 504 void
 505 acc_set_device_num (int n, acc_device_t d)
 506 {
 507   const struct gomp_device_descr *dev;
 508   int num_devices;
 509
 510   if (!base_dev)
 511     gomp_init_targets_once ();
 512
 513   if ((int) d == 0)
 514     {
 515       int i;
 516
 517       /* A device setting of zero sets all device types on the system to use
 518          the Nth instance of that device type.  Only attempt it for initialized
 519          devices though.  */
 520       for (i = acc_device_not_host + 1; i < _ACC_device_hwm; i++)
 521         {
 522           dev = resolve_device (d);
 523           if (dev && dev->is_initialized)
 524             dev->openacc.set_device_num_func (n);
 525         }
 526
 527       /* ...and for future calls to acc_init/acc_set_device_type, etc.  */
 528       goacc_device_num = n;
 529     }
 530   else
 531     {
 532       struct goacc_thread *thr = goacc_thread ();
 533
 534       gomp_mutex_lock (&acc_device_lock);
 535
 536       base_dev = lazy_init (d);
 537
 538       num_devices = base_dev->get_num_devices_func ();
 539
 540       if (n >= num_devices)
 541         gomp_fatal ("device %u out of range", n);
 542
 543       /* If we're changing the device number, de-associate this thread with
 544          the device (but don't close the device, since it may be in use by
 545          other threads).  */
 546       if (thr && thr->dev && n != thr->dev->target_id)
 547         thr->dev = NULL;
 548
 549       lazy_open (n);
 550
 551       gomp_mutex_unlock (&acc_device_lock);
 552     }
 553 }
 554
 555 ialias (acc_set_device_num)
 556
 557 int
 558 acc_on_device (acc_device_t dev)
 559 {
 560   struct goacc_thread *thr = goacc_thread ();
 561
 562   if (thr && thr->dev
 563       && acc_device_type (thr->dev->type) == acc_device_host_nonshm)
 564     return dev == acc_device_host_nonshm || dev == acc_device_not_host;
 565
 566   /* Just rely on the compiler builtin.  */
 567   return __builtin_acc_on_device (dev);
 568 }
 569 ialias (acc_on_device)
 570
 571 attribute_hidden void
 572 goacc_runtime_initialize (void)
 573 {
 574   gomp_mutex_init (&acc_device_lock);
 575
 576 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 577   pthread_key_create (&goacc_tls_key, NULL);
 578 #endif
 579
 580   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 581
 582   base_dev = NULL;
 583
 584   goacc_threads = NULL;
 585   gomp_mutex_init (&goacc_thread_lock);
 586 }
 587
 588 /* Compiler helper functions */
 589
 590 attribute_hidden void
 591 goacc_save_and_set_bind (acc_device_t d)
 592 {
 593   struct goacc_thread *thr = goacc_thread ();
 594
 595   assert (!thr->saved_bound_dev);
 596
 597   thr->saved_bound_dev = thr->dev;
 598   thr->dev = dispatchers[d];
 599 }
 600
 601 attribute_hidden void
 602 goacc_restore_bind (void)
 603 {
 604   struct goacc_thread *thr = goacc_thread ();
 605
 606   thr->dev = thr->saved_bound_dev;
 607   thr->saved_bound_dev = NULL;
 608 }
 609
 610 /* This is called from any OpenACC support function that may need to implicitly
 611    initialize the libgomp runtime.  On exit all such initialization will have
 612    been done, and both the global ACC_dev and the per-host-thread ACC_memmap
 613    pointers will be valid.  */
 614
 615 attribute_hidden void
 616 goacc_lazy_initialize (void)
 617 {
 618   struct goacc_thread *thr = goacc_thread ();
 619
 620   if (thr && thr->dev)
 621     return;
 622
 623   if (!base_dev)
 624     lazy_init_and_open (acc_device_default);
 625   else
 626     {
 627       gomp_mutex_lock (&acc_device_lock);
 628       lazy_open (-1);
 629       gomp_mutex_unlock (&acc_device_lock);
 630     }
 631 }