libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2015 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "oacc-int.h"
  31 #include "openacc.h"
  32 #include <assert.h>
  33 #include <stdlib.h>
  34 #include <strings.h>
  35 #include <stdbool.h>
  36 #include <string.h>
  37
  38 static gomp_mutex_t acc_device_lock;
  39
  40 /* The dispatch table for the current accelerator device.  This is global, so
  41    you can only have one type of device open at any given time in a program.
  42    This is the "base" device in that several devices that use the same
  43    dispatch table may be active concurrently: this one (the "zeroth") is used
  44    for overall initialisation/shutdown, and other instances -- not necessarily
  45    including this one -- may be opened and closed once the base device has
  46    been initialized.  */
  47 struct gomp_device_descr *base_dev;
  48
  49 #if defined HAVE_TLS || defined USE_EMUTLS
  50 __thread struct goacc_thread *goacc_tls_data;
  51 #else
  52 pthread_key_t goacc_tls_key;
  53 #endif
  54 static pthread_key_t goacc_cleanup_key;
  55
  56 /* Current dispatcher, and how it was initialized */
  57 static acc_device_t init_key = _ACC_device_hwm;
  58
  59 static struct goacc_thread *goacc_threads;
  60 static gomp_mutex_t goacc_thread_lock;
  61
  62 /* An array of dispatchers for device types, indexed by the type.  This array
  63    only references "base" devices, and other instances of the same type are
  64    found by simply indexing from each such device (which are stored linearly,
  65    grouped by device in target.c:devices).  */
  66 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  67
  68 attribute_hidden void
  69 goacc_register (struct gomp_device_descr *disp)
  70 {
  71   /* Only register the 0th device here.  */
  72   if (disp->target_id != 0)
  73     return;
  74
  75   gomp_mutex_lock (&acc_device_lock);
  76
  77   assert (acc_device_type (disp->type) != acc_device_none
  78           && acc_device_type (disp->type) != acc_device_default
  79           && acc_device_type (disp->type) != acc_device_not_host);
  80   assert (!dispatchers[disp->type]);
  81   dispatchers[disp->type] = disp;
  82
  83   gomp_mutex_unlock (&acc_device_lock);
  84 }
  85
  86 /* OpenACC names some things a little differently.  */
  87
  88 static const char *
  89 get_openacc_name (const char *name)
  90 {
  91   if (strcmp (name, "nvptx") == 0)
  92     return "nvidia";
  93   else
  94     return name;
  95 }
  96
  97 static struct gomp_device_descr *
  98 resolve_device (acc_device_t d)
  99 {
 100   acc_device_t d_arg = d;
 101
 102   switch (d)
 103     {
 104     case acc_device_default:
 105       {
 106         if (goacc_device_type)
 107           {
 108             /* Lookup the named device.  */
 109             while (++d != _ACC_device_hwm)
 110               if (dispatchers[d]
 111                   && !strcasecmp (goacc_device_type,
 112                                   get_openacc_name (dispatchers[d]->name))
 113                   && dispatchers[d]->get_num_devices_func () > 0)
 114                 goto found;
 115
 116             gomp_fatal ("device type %s not supported", goacc_device_type);
 117           }
 118
 119         /* No default device specified, so start scanning for any non-host
 120            device that is available.  */
 121         d = acc_device_not_host;
 122       }
 123       /* FALLTHROUGH */
 124
 125     case acc_device_not_host:
 126       /* Find the first available device after acc_device_not_host.  */
 127       while (++d != _ACC_device_hwm)
 128         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 129           goto found;
 130       if (d_arg == acc_device_default)
 131         {
 132           d = acc_device_host;
 133           goto found;
 134         }
 135       gomp_fatal ("no device found");
 136       break;
 137
 138     case acc_device_host:
 139       break;
 140
 141     default:
 142       if (d > _ACC_device_hwm)
 143         gomp_fatal ("device %u out of range", (unsigned)d);
 144       break;
 145     }
 146  found:
 147
 148   assert (d != acc_device_none
 149           && d != acc_device_default
 150           && d != acc_device_not_host);
 151
 152   return dispatchers[d];
 153 }
 154
 155 /* This is called when plugins have been initialized, and serves to call
 156    (indirectly) the target's device_init hook.  Calling multiple times without
 157    an intervening acc_shutdown_1 call is an error.  */
 158
 159 static struct gomp_device_descr *
 160 acc_init_1 (acc_device_t d)
 161 {
 162   struct gomp_device_descr *acc_dev;
 163
 164   acc_dev = resolve_device (d);
 165
 166   if (!acc_dev || acc_dev->get_num_devices_func () <= 0)
 167     gomp_fatal ("device %u not supported", (unsigned)d);
 168
 169   if (acc_dev->is_initialized)
 170     gomp_fatal ("device already active");
 171
 172   /* We need to remember what we were intialized as, to check shutdown etc.  */
 173   init_key = d;
 174
 175   gomp_init_device (acc_dev);
 176
 177   return acc_dev;
 178 }
 179
 180 static struct goacc_thread *
 181 goacc_new_thread (void)
 182 {
 183   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 184
 185 #if defined HAVE_TLS || defined USE_EMUTLS
 186   goacc_tls_data = thr;
 187 #else
 188   pthread_setspecific (goacc_tls_key, thr);
 189 #endif
 190
 191   pthread_setspecific (goacc_cleanup_key, thr);
 192
 193   gomp_mutex_lock (&goacc_thread_lock);
 194   thr->next = goacc_threads;
 195   goacc_threads = thr;
 196   gomp_mutex_unlock (&goacc_thread_lock);
 197
 198   return thr;
 199 }
 200
 201 static void
 202 goacc_destroy_thread (void *data)
 203 {
 204   struct goacc_thread *thr = data, *walk, *prev;
 205
 206   gomp_mutex_lock (&goacc_thread_lock);
 207
 208   if (thr)
 209     {
 210       if (base_dev && thr->target_tls)
 211         {
 212           base_dev->openacc.destroy_thread_data_func (thr->target_tls);
 213           thr->target_tls = NULL;
 214         }
 215
 216       assert (!thr->mapped_data);
 217
 218       /* Remove from thread list.  */
 219       for (prev = NULL, walk = goacc_threads; walk;
 220            prev = walk, walk = walk->next)
 221         if (walk == thr)
 222           {
 223             if (prev == NULL)
 224               goacc_threads = walk->next;
 225             else
 226               prev->next = walk->next;
 227
 228             free (thr);
 229
 230             break;
 231           }
 232
 233       assert (walk);
 234     }
 235
 236   gomp_mutex_unlock (&goacc_thread_lock);
 237 }
 238
 239 /* Open the ORD'th device of the currently-active type (base_dev must be
 240    initialised before calling).  If ORD is < 0, open the default-numbered
 241    device (set by the ACC_DEVICE_NUM environment variable or a call to
 242    acc_set_device_num), or leave any currently-opened device as is.  "Opening"
 243    consists of calling the device's open_device_func hook, and setting up
 244    thread-local data (maybe allocating, then initializing with information
 245    pertaining to the newly-opened or previously-opened device).  */
 246
 247 static void
 248 lazy_open (int ord)
 249 {
 250   struct goacc_thread *thr = goacc_thread ();
 251   struct gomp_device_descr *acc_dev;
 252
 253   if (thr && thr->dev)
 254     {
 255       assert (ord < 0 || ord == thr->dev->target_id);
 256       return;
 257     }
 258
 259   assert (base_dev);
 260
 261   if (ord < 0)
 262     ord = goacc_device_num;
 263
 264   /* The OpenACC 2.0 spec leaves the runtime's behaviour when an out-of-range
 265      device is requested as implementation-defined (4.2 ACC_DEVICE_NUM).
 266      We choose to raise an error in such a case.  */
 267   if (ord >= base_dev->get_num_devices_func ())
 268     gomp_fatal ("device %u does not exist", ord);
 269
 270   if (!thr)
 271     thr = goacc_new_thread ();
 272
 273   acc_dev = thr->dev = &base_dev[ord];
 274
 275   assert (acc_dev->target_id == ord);
 276
 277   thr->saved_bound_dev = NULL;
 278   thr->mapped_data = NULL;
 279
 280   if (!acc_dev->openacc.target_data)
 281     acc_dev->openacc.target_data = acc_dev->openacc.open_device_func (ord);
 282
 283   thr->target_tls
 284     = acc_dev->openacc.create_thread_data_func (acc_dev->openacc.target_data);
 285
 286   acc_dev->openacc.async_set_async_func (acc_async_sync);
 287
 288   struct gomp_memory_mapping *mem_map = &acc_dev->mem_map;
 289   gomp_mutex_lock (&mem_map->lock);
 290   if (!mem_map->is_initialized)
 291     gomp_init_tables (acc_dev, mem_map);
 292   gomp_mutex_unlock (&mem_map->lock);
 293 }
 294
 295 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 296    init/shutdown is per-process or per-thread.  We choose per-process.  */
 297
 298 void
 299 acc_init (acc_device_t d)
 300 {
 301   if (!base_dev)
 302     gomp_init_targets_once ();
 303
 304   gomp_mutex_lock (&acc_device_lock);
 305
 306   base_dev = acc_init_1 (d);
 307
 308   lazy_open (-1);
 309
 310   gomp_mutex_unlock (&acc_device_lock);
 311 }
 312
 313 ialias (acc_init)
 314
 315 static void
 316 acc_shutdown_1 (acc_device_t d)
 317 {
 318   struct goacc_thread *walk;
 319
 320   /* We don't check whether d matches the actual device found, because
 321      OpenACC 2.0 (3.2.12) says the parameters to the init and this
 322      call must match (for the shutdown call anyway, it's silent on
 323      others).  */
 324
 325   if (!base_dev)
 326     gomp_fatal ("no device initialized");
 327   if (d != init_key)
 328     gomp_fatal ("device %u(%u) is initialized",
 329                 (unsigned) init_key, (unsigned) base_dev->type);
 330
 331   gomp_mutex_lock (&goacc_thread_lock);
 332
 333   /* Free target-specific TLS data and close all devices.  */
 334   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 335     {
 336       if (walk->target_tls)
 337         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 338
 339       walk->target_tls = NULL;
 340
 341       /* This would mean the user is shutting down OpenACC in the middle of an
 342          "acc data" pragma.  Likely not intentional.  */
 343       if (walk->mapped_data)
 344         gomp_fatal ("shutdown in 'acc data' region");
 345
 346       if (walk->dev)
 347         {
 348           void *target_data = walk->dev->openacc.target_data;
 349           if (walk->dev->openacc.close_device_func (target_data) < 0)
 350             gomp_fatal ("failed to close device");
 351
 352           walk->dev->openacc.target_data = target_data = NULL;
 353
 354           struct gomp_memory_mapping *mem_map = &walk->dev->mem_map;
 355           gomp_mutex_lock (&mem_map->lock);
 356           gomp_free_memmap (mem_map);
 357           gomp_mutex_unlock (&mem_map->lock);
 358
 359           walk->dev = NULL;
 360         }
 361     }
 362
 363   gomp_mutex_unlock (&goacc_thread_lock);
 364
 365   gomp_fini_device (base_dev);
 366
 367   base_dev = NULL;
 368 }
 369
 370 void
 371 acc_shutdown (acc_device_t d)
 372 {
 373   gomp_mutex_lock (&acc_device_lock);
 374
 375   acc_shutdown_1 (d);
 376
 377   gomp_mutex_unlock (&acc_device_lock);
 378 }
 379
 380 ialias (acc_shutdown)
 381
 382 /* This function is called after plugins have been initialized.  It deals with
 383    the "base" device, and is used to prepare the runtime for dealing with a
 384    number of such devices (as implemented by some particular plugin).  If the
 385    argument device type D matches a previous call to the function, return the
 386    current base device, else shut the old device down and re-initialize with
 387    the new device type.  */
 388
 389 static struct gomp_device_descr *
 390 lazy_init (acc_device_t d)
 391 {
 392   if (base_dev)
 393     {
 394       /* Re-initializing the same device, do nothing.  */
 395       if (d == init_key)
 396         return base_dev;
 397
 398       acc_shutdown_1 (init_key);
 399     }
 400
 401   assert (!base_dev);
 402
 403   return acc_init_1 (d);
 404 }
 405
 406 /* Ensure that plugins are loaded, initialize and open the (default-numbered)
 407    device.  */
 408
 409 static void
 410 lazy_init_and_open (acc_device_t d)
 411 {
 412   if (!base_dev)
 413     gomp_init_targets_once ();
 414
 415   gomp_mutex_lock (&acc_device_lock);
 416
 417   base_dev = lazy_init (d);
 418
 419   lazy_open (-1);
 420
 421   gomp_mutex_unlock (&acc_device_lock);
 422 }
 423
 424 int
 425 acc_get_num_devices (acc_device_t d)
 426 {
 427   int n = 0;
 428   const struct gomp_device_descr *acc_dev;
 429
 430   if (d == acc_device_none)
 431     return 0;
 432
 433   if (!base_dev)
 434     gomp_init_targets_once ();
 435
 436   acc_dev = resolve_device (d);
 437   if (!acc_dev)
 438     return 0;
 439
 440   n = acc_dev->get_num_devices_func ();
 441   if (n < 0)
 442     n = 0;
 443
 444   return n;
 445 }
 446
 447 ialias (acc_get_num_devices)
 448
 449 void
 450 acc_set_device_type (acc_device_t d)
 451 {
 452   lazy_init_and_open (d);
 453 }
 454
 455 ialias (acc_set_device_type)
 456
 457 acc_device_t
 458 acc_get_device_type (void)
 459 {
 460   acc_device_t res = acc_device_none;
 461   const struct gomp_device_descr *dev;
 462
 463   if (base_dev)
 464     res = acc_device_type (base_dev->type);
 465   else
 466     {
 467       gomp_init_targets_once ();
 468
 469       dev = resolve_device (acc_device_default);
 470       res = acc_device_type (dev->type);
 471     }
 472
 473   assert (res != acc_device_default
 474           && res != acc_device_not_host);
 475
 476   return res;
 477 }
 478
 479 ialias (acc_get_device_type)
 480
 481 int
 482 acc_get_device_num (acc_device_t d)
 483 {
 484   const struct gomp_device_descr *dev;
 485   int num;
 486
 487   if (d >= _ACC_device_hwm)
 488     gomp_fatal ("device %u out of range", (unsigned)d);
 489
 490   if (!base_dev)
 491     gomp_init_targets_once ();
 492
 493   dev = resolve_device (d);
 494   if (!dev)
 495     gomp_fatal ("no devices of type %u", d);
 496
 497   /* We might not have called lazy_open for this host thread yet, in which case
 498      the get_device_num_func hook will return -1.  */
 499   num = dev->openacc.get_device_num_func ();
 500   if (num < 0)
 501     num = goacc_device_num;
 502
 503   return num;
 504 }
 505
 506 ialias (acc_get_device_num)
 507
 508 void
 509 acc_set_device_num (int n, acc_device_t d)
 510 {
 511   const struct gomp_device_descr *dev;
 512   int num_devices;
 513
 514   if (!base_dev)
 515     gomp_init_targets_once ();
 516
 517   if ((int) d == 0)
 518     {
 519       int i;
 520
 521       /* A device setting of zero sets all device types on the system to use
 522          the Nth instance of that device type.  Only attempt it for initialized
 523          devices though.  */
 524       for (i = acc_device_not_host + 1; i < _ACC_device_hwm; i++)
 525         {
 526           dev = resolve_device (d);
 527           if (dev && dev->is_initialized)
 528             dev->openacc.set_device_num_func (n);
 529         }
 530
 531       /* ...and for future calls to acc_init/acc_set_device_type, etc.  */
 532       goacc_device_num = n;
 533     }
 534   else
 535     {
 536       struct goacc_thread *thr = goacc_thread ();
 537
 538       gomp_mutex_lock (&acc_device_lock);
 539
 540       base_dev = lazy_init (d);
 541
 542       num_devices = base_dev->get_num_devices_func ();
 543
 544       if (n >= num_devices)
 545         gomp_fatal ("device %u out of range", n);
 546
 547       /* If we're changing the device number, de-associate this thread with
 548          the device (but don't close the device, since it may be in use by
 549          other threads).  */
 550       if (thr && thr->dev && n != thr->dev->target_id)
 551         thr->dev = NULL;
 552
 553       lazy_open (n);
 554
 555       gomp_mutex_unlock (&acc_device_lock);
 556     }
 557 }
 558
 559 ialias (acc_set_device_num)
 560
 561 int
 562 acc_on_device (acc_device_t dev)
 563 {
 564   struct goacc_thread *thr = goacc_thread ();
 565
 566   if (thr && thr->dev
 567       && acc_device_type (thr->dev->type) == acc_device_host_nonshm)
 568     return dev == acc_device_host_nonshm || dev == acc_device_not_host;
 569
 570   /* Just rely on the compiler builtin.  */
 571   return __builtin_acc_on_device (dev);
 572 }
 573
 574 ialias (acc_on_device)
 575
 576 attribute_hidden void
 577 goacc_runtime_initialize (void)
 578 {
 579   gomp_mutex_init (&acc_device_lock);
 580
 581 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 582   pthread_key_create (&goacc_tls_key, NULL);
 583 #endif
 584
 585   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 586
 587   base_dev = NULL;
 588
 589   goacc_threads = NULL;
 590   gomp_mutex_init (&goacc_thread_lock);
 591 }
 592
 593 /* Compiler helper functions */
 594
 595 attribute_hidden void
 596 goacc_save_and_set_bind (acc_device_t d)
 597 {
 598   struct goacc_thread *thr = goacc_thread ();
 599
 600   assert (!thr->saved_bound_dev);
 601
 602   thr->saved_bound_dev = thr->dev;
 603   thr->dev = dispatchers[d];
 604 }
 605
 606 attribute_hidden void
 607 goacc_restore_bind (void)
 608 {
 609   struct goacc_thread *thr = goacc_thread ();
 610
 611   thr->dev = thr->saved_bound_dev;
 612   thr->saved_bound_dev = NULL;
 613 }
 614
 615 /* This is called from any OpenACC support function that may need to implicitly
 616    initialize the libgomp runtime.  On exit all such initialization will have
 617    been done, and both the global ACC_dev and the per-host-thread ACC_memmap
 618    pointers will be valid.  */
 619
 620 attribute_hidden void
 621 goacc_lazy_initialize (void)
 622 {
 623   struct goacc_thread *thr = goacc_thread ();
 624
 625   if (thr && thr->dev)
 626     return;
 627
 628   if (!base_dev)
 629     lazy_init_and_open (acc_device_default);
 630   else
 631     {
 632       gomp_mutex_lock (&acc_device_lock);
 633       lazy_open (-1);
 634       gomp_mutex_unlock (&acc_device_lock);
 635     }
 636 }