libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2014 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "libgomp_target.h"
  31 #include "oacc-int.h"
  32 #include "openacc.h"
  33 #include <assert.h>
  34 #include <stdlib.h>
  35 #include <strings.h>
  36 #include <stdbool.h>
  37 #include <stdio.h>
  38 #include <string.h>
  39
  40 static gomp_mutex_t acc_device_lock;
  41
  42 /* The dispatch table for the current accelerator device.  This is global, so
  43    you can only have one type of device open at any given time in a program.
  44    This is the "base" device in that several devices that use the same
  45    dispatch table may be active concurrently: this one (the "zeroth") is used
  46    for overall initialisation/shutdown, and other instances -- not necessarily
  47    including this one -- may be opened and closed once the base device has
  48    been initialized.  */
  49 struct gomp_device_descr *base_dev;
  50
  51 #if defined HAVE_TLS || defined USE_EMUTLS
  52 __thread struct goacc_thread *goacc_tls_data;
  53 #else
  54 pthread_key_t goacc_tls_key;
  55 #endif
  56 static pthread_key_t goacc_cleanup_key;
  57
  58 /* Current dispatcher, and how it was initialized */
  59 static acc_device_t init_key = _ACC_device_hwm;
  60
  61 static struct goacc_thread *goacc_threads;
  62 static gomp_mutex_t goacc_thread_lock;
  63
  64 /* An array of dispatchers for device types, indexed by the type.  This array
  65    only references "base" devices, and other instances of the same type are
  66    found by simply indexing from each such device (which are stored linearly,
  67    grouped by device in target.c:devices).  */
  68 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  69
  70 attribute_hidden void
  71 goacc_register (struct gomp_device_descr *disp)
  72 {
  73   /* Only register the 0th device here.  */
  74   if (disp->target_id != 0)
  75     return;
  76
  77   gomp_mutex_lock (&acc_device_lock);
  78
  79   assert (acc_device_type (disp->type) != acc_device_none
  80           && acc_device_type (disp->type) != acc_device_default
  81           && acc_device_type (disp->type) != acc_device_not_host);
  82   assert (!dispatchers[disp->type]);
  83   dispatchers[disp->type] = disp;
  84
  85   gomp_mutex_unlock (&acc_device_lock);
  86 }
  87
  88 /* OpenACC names some things a little differently.  */
  89
  90 static const char *
  91 get_openacc_name (const char *name)
  92 {
  93   if (strcmp (name, "nvptx") == 0)
  94     return "nvidia";
  95   else
  96     return name;
  97 }
  98
  99 static struct gomp_device_descr *
 100 resolve_device (acc_device_t d)
 101 {
 102   acc_device_t d_arg = d;
 103
 104   switch (d)
 105     {
 106     case acc_device_default:
 107       {
 108         if (goacc_device_type)
 109           {
 110             /* Lookup the named device.  */
 111             while (++d != _ACC_device_hwm)
 112               if (dispatchers[d]
 113                   && !strcasecmp (goacc_device_type,
 114                                   get_openacc_name (dispatchers[d]->name))
 115                   && dispatchers[d]->get_num_devices_func () > 0)
 116                 goto found;
 117
 118             gomp_fatal ("device type %s not supported", goacc_device_type);
 119           }
 120
 121         /* No default device specified, so start scanning for any non-host
 122            device that is available.  */
 123         d = acc_device_not_host;
 124       }
 125       /* FALLTHROUGH */
 126
 127     case acc_device_not_host:
 128       /* Find the first available device after acc_device_not_host.  */
 129       while (++d != _ACC_device_hwm)
 130         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 131           goto found;
 132       if (d_arg == acc_device_default)
 133         {
 134           d = acc_device_host;
 135           goto found;
 136         }
 137       gomp_fatal ("no device found");
 138       break;
 139
 140     case acc_device_host:
 141       break;
 142
 143     default:
 144       if (d > _ACC_device_hwm)
 145         gomp_fatal ("device %u out of range", (unsigned)d);
 146       break;
 147     }
 148  found:
 149
 150   assert (d != acc_device_none
 151           && d != acc_device_default
 152           && d != acc_device_not_host);
 153
 154   return dispatchers[d];
 155 }
 156
 157 /* This is called when plugins have been initialized, and serves to call
 158    (indirectly) the target's device_init hook.  Calling multiple times without
 159    an intervening acc_shutdown_1 call is an error.  */
 160
 161 static struct gomp_device_descr *
 162 acc_init_1 (acc_device_t d)
 163 {
 164   struct gomp_device_descr *acc_dev;
 165
 166   acc_dev = resolve_device (d);
 167
 168   if (!acc_dev || acc_dev->get_num_devices_func () <= 0)
 169     gomp_fatal ("device %u not supported", (unsigned)d);
 170
 171   if (acc_dev->is_initialized)
 172     gomp_fatal ("device already active");
 173
 174   /* We need to remember what we were intialized as, to check shutdown etc.  */
 175   init_key = d;
 176
 177   gomp_init_device (acc_dev);
 178
 179   return acc_dev;
 180 }
 181
 182 static struct goacc_thread *
 183 goacc_new_thread (void)
 184 {
 185   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 186
 187 #if defined HAVE_TLS || defined USE_EMUTLS
 188   goacc_tls_data = thr;
 189 #else
 190   pthread_setspecific (goacc_tls_key, thr);
 191 #endif
 192
 193   pthread_setspecific (goacc_cleanup_key, thr);
 194
 195   gomp_mutex_lock (&goacc_thread_lock);
 196   thr->next = goacc_threads;
 197   goacc_threads = thr;
 198   gomp_mutex_unlock (&goacc_thread_lock);
 199
 200   return thr;
 201 }
 202
 203 static void
 204 goacc_destroy_thread (void *data)
 205 {
 206   struct goacc_thread *thr = data, *walk, *prev;
 207
 208   gomp_mutex_lock (&goacc_thread_lock);
 209
 210   if (thr)
 211     {
 212       if (base_dev && thr->target_tls)
 213         {
 214           base_dev->openacc.destroy_thread_data_func (thr->target_tls);
 215           thr->target_tls = NULL;
 216         }
 217
 218       assert (!thr->mapped_data);
 219
 220       /* Remove from thread list.  */
 221       for (prev = NULL, walk = goacc_threads; walk;
 222            prev = walk, walk = walk->next)
 223         if (walk == thr)
 224           {
 225             if (prev == NULL)
 226               goacc_threads = walk->next;
 227             else
 228               prev->next = walk->next;
 229
 230             free (thr);
 231
 232             break;
 233           }
 234
 235       assert (walk);
 236     }
 237
 238   gomp_mutex_unlock (&goacc_thread_lock);
 239 }
 240
 241 /* Open the ORD'th device of the currently-active type (base_dev must be
 242    initialised before calling).  If ORD is < 0, open the default-numbered
 243    device (set by the ACC_DEVICE_NUM environment variable or a call to
 244    acc_set_device_num), or leave any currently-opened device as is.  "Opening"
 245    consists of calling the device's open_device_func hook, and setting up
 246    thread-local data (maybe allocating, then initializing with information
 247    pertaining to the newly-opened or previously-opened device).  */
 248
 249 static void
 250 lazy_open (int ord)
 251 {
 252   struct goacc_thread *thr = goacc_thread ();
 253   struct gomp_device_descr *acc_dev;
 254
 255   if (thr && thr->dev)
 256     {
 257       assert (ord < 0 || ord == thr->dev->target_id);
 258       return;
 259     }
 260
 261   assert (base_dev);
 262
 263   if (ord < 0)
 264     ord = goacc_device_num;
 265
 266   /* The OpenACC 2.0 spec leaves the runtime's behaviour when an out-of-range
 267      device is requested as implementation-defined (4.2 ACC_DEVICE_NUM).
 268      We choose to raise an error in such a case.  */
 269   if (ord >= base_dev->get_num_devices_func ())
 270     gomp_fatal ("device %u does not exist", ord);
 271
 272   if (!thr)
 273     thr = goacc_new_thread ();
 274
 275   acc_dev = thr->dev = &base_dev[ord];
 276
 277   assert (acc_dev->target_id == ord);
 278
 279   thr->saved_bound_dev = NULL;
 280   thr->mapped_data = NULL;
 281
 282   if (!acc_dev->openacc.target_data)
 283     acc_dev->openacc.target_data = acc_dev->openacc.open_device_func (ord);
 284
 285   thr->target_tls
 286     = acc_dev->openacc.create_thread_data_func (acc_dev->openacc.target_data);
 287
 288   acc_dev->openacc.async_set_async_func (acc_async_sync);
 289
 290   struct gomp_memory_mapping *mem_map = &acc_dev->mem_map;
 291   gomp_mutex_lock (&mem_map->lock);
 292   if (!mem_map->is_initialized)
 293     gomp_init_tables (acc_dev, mem_map);
 294   gomp_mutex_unlock (&mem_map->lock);
 295 }
 296
 297 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 298    init/shutdown is per-process or per-thread.  We choose per-process.  */
 299
 300 void
 301 acc_init (acc_device_t d)
 302 {
 303   if (!base_dev)
 304     gomp_init_targets_once ();
 305
 306   gomp_mutex_lock (&acc_device_lock);
 307
 308   base_dev = acc_init_1 (d);
 309
 310   lazy_open (-1);
 311
 312   gomp_mutex_unlock (&acc_device_lock);
 313 }
 314
 315 ialias (acc_init)
 316
 317 static void
 318 acc_shutdown_1 (acc_device_t d)
 319 {
 320   struct goacc_thread *walk;
 321
 322   /* We don't check whether d matches the actual device found, because
 323      OpenACC 2.0 (3.2.12) says the parameters to the init and this
 324      call must match (for the shutdown call anyway, it's silent on
 325      others).  */
 326
 327   if (!base_dev)
 328     gomp_fatal ("no device initialized");
 329   if (d != init_key)
 330     gomp_fatal ("device %u(%u) is initialized",
 331                 (unsigned) init_key, (unsigned) base_dev->type);
 332
 333   gomp_mutex_lock (&goacc_thread_lock);
 334
 335   /* Free target-specific TLS data and close all devices.  */
 336   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 337     {
 338       if (walk->target_tls)
 339         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 340
 341       walk->target_tls = NULL;
 342
 343       /* This would mean the user is shutting down OpenACC in the middle of an
 344          "acc data" pragma.  Likely not intentional.  */
 345       if (walk->mapped_data)
 346         gomp_fatal ("shutdown in 'acc data' region");
 347
 348       if (walk->dev)
 349         {
 350           void *target_data = walk->dev->openacc.target_data;
 351           if (walk->dev->openacc.close_device_func (target_data) < 0)
 352             gomp_fatal ("failed to close device");
 353
 354           walk->dev->openacc.target_data = target_data = NULL;
 355
 356           struct gomp_memory_mapping *mem_map = &walk->dev->mem_map;
 357           gomp_mutex_lock (&mem_map->lock);
 358           gomp_free_memmap (mem_map);
 359           gomp_mutex_unlock (&mem_map->lock);
 360
 361           walk->dev = NULL;
 362         }
 363     }
 364
 365   gomp_mutex_unlock (&goacc_thread_lock);
 366
 367   gomp_fini_device (base_dev);
 368
 369   base_dev = NULL;
 370 }
 371
 372 void
 373 acc_shutdown (acc_device_t d)
 374 {
 375   gomp_mutex_lock (&acc_device_lock);
 376
 377   acc_shutdown_1 (d);
 378
 379   gomp_mutex_unlock (&acc_device_lock);
 380 }
 381
 382 ialias (acc_shutdown)
 383
 384 /* This function is called after plugins have been initialized.  It deals with
 385    the "base" device, and is used to prepare the runtime for dealing with a
 386    number of such devices (as implemented by some particular plugin).  If the
 387    argument device type D matches a previous call to the function, return the
 388    current base device, else shut the old device down and re-initialize with
 389    the new device type.  */
 390
 391 static struct gomp_device_descr *
 392 lazy_init (acc_device_t d)
 393 {
 394   if (base_dev)
 395     {
 396       /* Re-initializing the same device, do nothing.  */
 397       if (d == init_key)
 398         return base_dev;
 399
 400       acc_shutdown_1 (init_key);
 401     }
 402
 403   assert (!base_dev);
 404
 405   return acc_init_1 (d);
 406 }
 407
 408 /* Ensure that plugins are loaded, initialize and open the (default-numbered)
 409    device.  */
 410
 411 static void
 412 lazy_init_and_open (acc_device_t d)
 413 {
 414   if (!base_dev)
 415     gomp_init_targets_once ();
 416
 417   gomp_mutex_lock (&acc_device_lock);
 418
 419   base_dev = lazy_init (d);
 420
 421   lazy_open (-1);
 422
 423   gomp_mutex_unlock (&acc_device_lock);
 424 }
 425
 426 int
 427 acc_get_num_devices (acc_device_t d)
 428 {
 429   int n = 0;
 430   const struct gomp_device_descr *acc_dev;
 431
 432   if (d == acc_device_none)
 433     return 0;
 434
 435   if (!base_dev)
 436     gomp_init_targets_once ();
 437
 438   acc_dev = resolve_device (d);
 439   if (!acc_dev)
 440     return 0;
 441
 442   n = acc_dev->get_num_devices_func ();
 443   if (n < 0)
 444     n = 0;
 445
 446   return n;
 447 }
 448
 449 ialias (acc_get_num_devices)
 450
 451 void
 452 acc_set_device_type (acc_device_t d)
 453 {
 454   lazy_init_and_open (d);
 455 }
 456
 457 ialias (acc_set_device_type)
 458
 459 acc_device_t
 460 acc_get_device_type (void)
 461 {
 462   acc_device_t res = acc_device_none;
 463   const struct gomp_device_descr *dev;
 464
 465   if (base_dev)
 466     res = acc_device_type (base_dev->type);
 467   else
 468     {
 469       gomp_init_targets_once ();
 470
 471       dev = resolve_device (acc_device_default);
 472       res = acc_device_type (dev->type);
 473     }
 474
 475   assert (res != acc_device_default
 476           && res != acc_device_not_host);
 477
 478   return res;
 479 }
 480
 481 ialias (acc_get_device_type)
 482
 483 int
 484 acc_get_device_num (acc_device_t d)
 485 {
 486   const struct gomp_device_descr *dev;
 487   int num;
 488
 489   if (d >= _ACC_device_hwm)
 490     gomp_fatal ("device %u out of range", (unsigned)d);
 491
 492   if (!base_dev)
 493     gomp_init_targets_once ();
 494
 495   dev = resolve_device (d);
 496   if (!dev)
 497     gomp_fatal ("no devices of type %u", d);
 498
 499   /* We might not have called lazy_open for this host thread yet, in which case
 500      the get_device_num_func hook will return -1.  */
 501   num = dev->openacc.get_device_num_func ();
 502   if (num < 0)
 503     num = goacc_device_num;
 504
 505   return num;
 506 }
 507
 508 ialias (acc_get_device_num)
 509
 510 void
 511 acc_set_device_num (int n, acc_device_t d)
 512 {
 513   const struct gomp_device_descr *dev;
 514   int num_devices;
 515
 516   if (!base_dev)
 517     gomp_init_targets_once ();
 518
 519   if ((int) d == 0)
 520     {
 521       int i;
 522
 523       /* A device setting of zero sets all device types on the system to use
 524          the Nth instance of that device type.  Only attempt it for initialized
 525          devices though.  */
 526       for (i = acc_device_not_host + 1; i < _ACC_device_hwm; i++)
 527         {
 528           dev = resolve_device (d);
 529           if (dev && dev->is_initialized)
 530             dev->openacc.set_device_num_func (n);
 531         }
 532
 533       /* ...and for future calls to acc_init/acc_set_device_type, etc.  */
 534       goacc_device_num = n;
 535     }
 536   else
 537     {
 538       struct goacc_thread *thr = goacc_thread ();
 539
 540       gomp_mutex_lock (&acc_device_lock);
 541
 542       base_dev = lazy_init (d);
 543
 544       num_devices = base_dev->get_num_devices_func ();
 545
 546       if (n >= num_devices)
 547         gomp_fatal ("device %u out of range", n);
 548
 549       /* If we're changing the device number, de-associate this thread with
 550          the device (but don't close the device, since it may be in use by
 551          other threads).  */
 552       if (thr && thr->dev && n != thr->dev->target_id)
 553         thr->dev = NULL;
 554
 555       lazy_open (n);
 556
 557       gomp_mutex_unlock (&acc_device_lock);
 558     }
 559 }
 560
 561 ialias (acc_set_device_num)
 562
 563 int
 564 acc_on_device (acc_device_t dev)
 565 {
 566   struct goacc_thread *thr = goacc_thread ();
 567
 568   if (thr && thr->dev
 569       && acc_device_type (thr->dev->type) == acc_device_host_nonshm)
 570     return dev == acc_device_host_nonshm || dev == acc_device_not_host;
 571
 572   /* Just rely on the compiler builtin.  */
 573   return __builtin_acc_on_device (dev);
 574 }
 575 ialias (acc_on_device)
 576
 577 attribute_hidden void
 578 goacc_runtime_initialize (void)
 579 {
 580   gomp_mutex_init (&acc_device_lock);
 581
 582 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 583   pthread_key_create (&goacc_tls_key, NULL);
 584 #endif
 585
 586   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 587
 588   base_dev = NULL;
 589
 590   goacc_threads = NULL;
 591   gomp_mutex_init (&goacc_thread_lock);
 592 }
 593
 594 /* Compiler helper functions */
 595
 596 attribute_hidden void
 597 goacc_save_and_set_bind (acc_device_t d)
 598 {
 599   struct goacc_thread *thr = goacc_thread ();
 600
 601   assert (!thr->saved_bound_dev);
 602
 603   thr->saved_bound_dev = thr->dev;
 604   thr->dev = dispatchers[d];
 605 }
 606
 607 attribute_hidden void
 608 goacc_restore_bind (void)
 609 {
 610   struct goacc_thread *thr = goacc_thread ();
 611
 612   thr->dev = thr->saved_bound_dev;
 613   thr->saved_bound_dev = NULL;
 614 }
 615
 616 /* This is called from any OpenACC support function that may need to implicitly
 617    initialize the libgomp runtime.  On exit all such initialization will have
 618    been done, and both the global ACC_dev and the per-host-thread ACC_memmap
 619    pointers will be valid.  */
 620
 621 attribute_hidden void
 622 goacc_lazy_initialize (void)
 623 {
 624   struct goacc_thread *thr = goacc_thread ();
 625
 626   if (thr && thr->dev)
 627     return;
 628
 629   if (!base_dev)
 630     lazy_init_and_open (acc_device_default);
 631   else
 632     {
 633       gomp_mutex_lock (&acc_device_lock);
 634       lazy_open (-1);
 635       gomp_mutex_unlock (&acc_device_lock);
 636     }
 637 }