libgomp/oacc-parallel.c

   1 /* Copyright (C) 2013-2016 Free Software Foundation, Inc.
   2
   3    Contributed by Mentor Embedded.
   4
   5    This file is part of the GNU Offloading and Multi Processing Library
   6    (libgomp).
   7
   8    Libgomp is free software; you can redistribute it and/or modify it
   9    under the terms of the GNU General Public License as published by
  10    the Free Software Foundation; either version 3, or (at your option)
  11    any later version.
  12
  13    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  14    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  15    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  16    more details.
  17
  18    Under Section 7 of GPL version 3, you are granted additional
  19    permissions described in the GCC Runtime Library Exception, version
  20    3.1, as published by the Free Software Foundation.
  21
  22    You should have received a copy of the GNU General Public License and
  23    a copy of the GCC Runtime Library Exception along with this program;
  24    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  25    <http://www.gnu.org/licenses/>.  */
  26
  27 /* This file handles OpenACC constructs.  */
  28
  29 #include "openacc.h"
  30 #include "libgomp.h"
  31 #include "libgomp_g.h"
  32 #include "gomp-constants.h"
  33 #include "oacc-int.h"
  34 #ifdef HAVE_INTTYPES_H
  35 # include <inttypes.h>  /* For PRIu64.  */
  36 #endif
  37 #include <string.h>
  38 #include <stdarg.h>
  39 #include <assert.h>
  40
  41 static int
  42 find_pset (int pos, size_t mapnum, unsigned short *kinds)
  43 {
  44   if (pos + 1 >= mapnum)
  45     return 0;
  46
  47   unsigned char kind = kinds[pos+1] & 0xff;
  48
  49   return kind == GOMP_MAP_TO_PSET;
  50 }
  51
  52 static void goacc_wait (int async, int num_waits, va_list *ap);
  53
  54
  55 /* Launch a possibly offloaded function on DEVICE.  FN is the host fn
  56    address.  MAPNUM, HOSTADDRS, SIZES & KINDS  describe the memory
  57    blocks to be copied to/from the device.  Varadic arguments are
  58    keyed optional parameters terminated with a zero.  */
  59
  60 void
  61 GOACC_parallel_keyed (int device, void (*fn) (void *),
  62                       size_t mapnum, void **hostaddrs, size_t *sizes,
  63                       unsigned short *kinds, ...)
  64 {
  65   bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
  66   va_list ap;
  67   struct goacc_thread *thr;
  68   struct gomp_device_descr *acc_dev;
  69   struct target_mem_desc *tgt;
  70   void **devaddrs;
  71   unsigned int i;
  72   struct splay_tree_key_s k;
  73   splay_tree_key tgt_fn_key;
  74   void (*tgt_fn);
  75   int async = GOMP_ASYNC_SYNC;
  76   unsigned dims[GOMP_DIM_MAX];
  77   unsigned tag;
  78
  79 #ifdef HAVE_INTTYPES_H
  80   gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n",
  81               __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds);
  82 #else
  83   gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n",
  84               __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds);
  85 #endif
  86   goacc_lazy_initialize ();
  87
  88   thr = goacc_thread ();
  89   acc_dev = thr->dev;
  90
  91   /* Host fallback if "if" clause is false or if the current device is set to
  92      the host.  */
  93   if (host_fallback)
  94     {
  95       goacc_save_and_set_bind (acc_device_host);
  96       fn (hostaddrs);
  97       goacc_restore_bind ();
  98       return;
  99     }
 100   else if (acc_device_type (acc_dev->type) == acc_device_host)
 101     {
 102       fn (hostaddrs);
 103       return;
 104     }
 105
 106   va_start (ap, kinds);
 107   /* TODO: This will need amending when device_type is implemented.  */
 108   while ((tag = va_arg (ap, unsigned)) != 0)
 109     {
 110       if (GOMP_LAUNCH_DEVICE (tag))
 111         gomp_fatal ("device_type '%d' offload parameters, libgomp is too old",
 112                     GOMP_LAUNCH_DEVICE (tag));
 113
 114       switch (GOMP_LAUNCH_CODE (tag))
 115         {
 116         case GOMP_LAUNCH_DIM:
 117           {
 118             unsigned mask = GOMP_LAUNCH_OP (tag);
 119
 120             for (i = 0; i != GOMP_DIM_MAX; i++)
 121               if (mask & GOMP_DIM_MASK (i))
 122                 dims[i] = va_arg (ap, unsigned);
 123           }
 124           break;
 125
 126         case GOMP_LAUNCH_ASYNC:
 127           {
 128             /* Small constant values are encoded in the operand.  */
 129             async = GOMP_LAUNCH_OP (tag);
 130
 131             if (async == GOMP_LAUNCH_OP_MAX)
 132               async = va_arg (ap, unsigned);
 133             break;
 134           }
 135
 136         case GOMP_LAUNCH_WAIT:
 137           {
 138             unsigned num_waits = GOMP_LAUNCH_OP (tag);
 139
 140             if (num_waits)
 141               goacc_wait (async, num_waits, &ap);
 142             break;
 143           }
 144
 145         default:
 146           gomp_fatal ("unrecognized offload code '%d',"
 147                       " libgomp is too old", GOMP_LAUNCH_CODE (tag));
 148         }
 149     }
 150   va_end (ap);
 151
 152   acc_dev->openacc.async_set_async_func (async);
 153
 154   if (!(acc_dev->capabilities & GOMP_OFFLOAD_CAP_NATIVE_EXEC))
 155     {
 156       k.host_start = (uintptr_t) fn;
 157       k.host_end = k.host_start + 1;
 158       gomp_mutex_lock (&acc_dev->lock);
 159       tgt_fn_key = splay_tree_lookup (&acc_dev->mem_map, &k);
 160       gomp_mutex_unlock (&acc_dev->lock);
 161
 162       if (tgt_fn_key == NULL)
 163         gomp_fatal ("target function wasn't mapped");
 164
 165       tgt_fn = (void (*)) tgt_fn_key->tgt_offset;
 166     }
 167   else
 168     tgt_fn = (void (*)) fn;
 169
 170   tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true,
 171                        GOMP_MAP_VARS_OPENACC);
 172
 173   devaddrs = gomp_alloca (sizeof (void *) * mapnum);
 174   for (i = 0; i < mapnum; i++)
 175     devaddrs[i] = (void *) (tgt->list[i].key->tgt->tgt_start
 176                             + tgt->list[i].key->tgt_offset);
 177
 178   acc_dev->openacc.exec_func (tgt_fn, mapnum, hostaddrs, devaddrs,
 179                               async, dims, tgt);
 180
 181   /* If running synchronously, unmap immediately.  */
 182   if (async < acc_async_noval)
 183     gomp_unmap_vars (tgt, true);
 184   else
 185     {
 186       gomp_copy_from_async (tgt);
 187       acc_dev->openacc.register_async_cleanup_func (tgt);
 188     }
 189
 190   acc_dev->openacc.async_set_async_func (acc_async_sync);
 191 }
 192
 193 /* Legacy entry point, only provide host execution.  */
 194
 195 void
 196 GOACC_parallel (int device, void (*fn) (void *),
 197                 size_t mapnum, void **hostaddrs, size_t *sizes,
 198                 unsigned short *kinds,
 199                 int num_gangs, int num_workers, int vector_length,
 200                 int async, int num_waits, ...)
 201 {
 202   goacc_save_and_set_bind (acc_device_host);
 203   fn (hostaddrs);
 204   goacc_restore_bind ();
 205 }
 206
 207 void
 208 GOACC_data_start (int device, size_t mapnum,
 209                   void **hostaddrs, size_t *sizes, unsigned short *kinds)
 210 {
 211   bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
 212   struct target_mem_desc *tgt;
 213
 214 #ifdef HAVE_INTTYPES_H
 215   gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n",
 216               __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds);
 217 #else
 218   gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n",
 219               __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds);
 220 #endif
 221
 222   goacc_lazy_initialize ();
 223
 224   struct goacc_thread *thr = goacc_thread ();
 225   struct gomp_device_descr *acc_dev = thr->dev;
 226
 227   /* Host fallback or 'do nothing'.  */
 228   if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 229       || host_fallback)
 230     {
 231       tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true,
 232                            GOMP_MAP_VARS_OPENACC);
 233       tgt->prev = thr->mapped_data;
 234       thr->mapped_data = tgt;
 235
 236       return;
 237     }
 238
 239   gomp_debug (0, "  %s: prepare mappings\n", __FUNCTION__);
 240   tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true,
 241                        GOMP_MAP_VARS_OPENACC);
 242   gomp_debug (0, "  %s: mappings prepared\n", __FUNCTION__);
 243   tgt->prev = thr->mapped_data;
 244   thr->mapped_data = tgt;
 245 }
 246
 247 void
 248 GOACC_data_end (void)
 249 {
 250   struct goacc_thread *thr = goacc_thread ();
 251   struct target_mem_desc *tgt = thr->mapped_data;
 252
 253   gomp_debug (0, "  %s: restore mappings\n", __FUNCTION__);
 254   thr->mapped_data = tgt->prev;
 255   gomp_unmap_vars (tgt, true);
 256   gomp_debug (0, "  %s: mappings restored\n", __FUNCTION__);
 257 }
 258
 259 void
 260 GOACC_enter_exit_data (int device, size_t mapnum,
 261                        void **hostaddrs, size_t *sizes, unsigned short *kinds,
 262                        int async, int num_waits, ...)
 263 {
 264   struct goacc_thread *thr;
 265   struct gomp_device_descr *acc_dev;
 266   bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
 267   bool data_enter = false;
 268   size_t i;
 269
 270   goacc_lazy_initialize ();
 271
 272   thr = goacc_thread ();
 273   acc_dev = thr->dev;
 274
 275   if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 276       || host_fallback)
 277     return;
 278
 279   if (num_waits)
 280     {
 281       va_list ap;
 282
 283       va_start (ap, num_waits);
 284       goacc_wait (async, num_waits, &ap);
 285       va_end (ap);
 286     }
 287
 288   acc_dev->openacc.async_set_async_func (async);
 289
 290   /* Determine if this is an "acc enter data".  */
 291   for (i = 0; i < mapnum; ++i)
 292     {
 293       unsigned char kind = kinds[i] & 0xff;
 294
 295       if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET)
 296         continue;
 297
 298       if (kind == GOMP_MAP_FORCE_ALLOC
 299           || kind == GOMP_MAP_FORCE_PRESENT
 300           || kind == GOMP_MAP_FORCE_TO)
 301         {
 302           data_enter = true;
 303           break;
 304         }
 305
 306       if (kind == GOMP_MAP_FORCE_DEALLOC
 307           || kind == GOMP_MAP_FORCE_FROM)
 308         break;
 309
 310       gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
 311                       kind);
 312     }
 313
 314   if (data_enter)
 315     {
 316       for (i = 0; i < mapnum; i++)
 317         {
 318           unsigned char kind = kinds[i] & 0xff;
 319
 320           /* Scan for PSETs.  */
 321           int psets = find_pset (i, mapnum, kinds);
 322
 323           if (!psets)
 324             {
 325               switch (kind)
 326                 {
 327                 case GOMP_MAP_POINTER:
 328                   gomp_acc_insert_pointer (1, &hostaddrs[i], &sizes[i],
 329                                         &kinds[i]);
 330                   break;
 331                 case GOMP_MAP_FORCE_ALLOC:
 332                   acc_create (hostaddrs[i], sizes[i]);
 333                   break;
 334                 case GOMP_MAP_FORCE_PRESENT:
 335                   acc_present_or_copyin (hostaddrs[i], sizes[i]);
 336                   break;
 337                 case GOMP_MAP_FORCE_TO:
 338                   acc_present_or_copyin (hostaddrs[i], sizes[i]);
 339                   break;
 340                 default:
 341                   gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
 342                               kind);
 343                   break;
 344                 }
 345             }
 346           else
 347             {
 348               gomp_acc_insert_pointer (3, &hostaddrs[i], &sizes[i], &kinds[i]);
 349               /* Increment 'i' by two because OpenACC requires fortran
 350                  arrays to be contiguous, so each PSET is associated with
 351                  one of MAP_FORCE_ALLOC/MAP_FORCE_PRESET/MAP_FORCE_TO, and
 352                  one MAP_POINTER.  */
 353               i += 2;
 354             }
 355         }
 356     }
 357   else
 358     for (i = 0; i < mapnum; ++i)
 359       {
 360         unsigned char kind = kinds[i] & 0xff;
 361
 362         int psets = find_pset (i, mapnum, kinds);
 363
 364         if (!psets)
 365           {
 366             switch (kind)
 367               {
 368               case GOMP_MAP_POINTER:
 369                 gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff)
 370                                          == GOMP_MAP_FORCE_FROM,
 371                                          async, 1);
 372                 break;
 373               case GOMP_MAP_FORCE_DEALLOC:
 374                 acc_delete (hostaddrs[i], sizes[i]);
 375                 break;
 376               case GOMP_MAP_FORCE_FROM:
 377                 acc_copyout (hostaddrs[i], sizes[i]);
 378                 break;
 379               default:
 380                 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
 381                             kind);
 382                 break;
 383               }
 384           }
 385         else
 386           {
 387             gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff)
 388                                      == GOMP_MAP_FORCE_FROM, async, 3);
 389             /* See the above comment.  */
 390             i += 2;
 391           }
 392       }
 393
 394   acc_dev->openacc.async_set_async_func (acc_async_sync);
 395 }
 396
 397 static void
 398 goacc_wait (int async, int num_waits, va_list *ap)
 399 {
 400   struct goacc_thread *thr = goacc_thread ();
 401   struct gomp_device_descr *acc_dev = thr->dev;
 402
 403   while (num_waits--)
 404     {
 405       int qid = va_arg (*ap, int);
 406
 407       if (acc_async_test (qid))
 408         continue;
 409
 410       if (async == acc_async_sync)
 411         acc_wait (qid);
 412       else if (qid == async)
 413         ;/* If we're waiting on the same asynchronous queue as we're
 414             launching on, the queue itself will order work as
 415             required, so there's no need to wait explicitly.  */
 416       else
 417         acc_dev->openacc.async_wait_async_func (qid, async);
 418     }
 419 }
 420
 421 void
 422 GOACC_update (int device, size_t mapnum,
 423               void **hostaddrs, size_t *sizes, unsigned short *kinds,
 424               int async, int num_waits, ...)
 425 {
 426   bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
 427   size_t i;
 428
 429   goacc_lazy_initialize ();
 430
 431   struct goacc_thread *thr = goacc_thread ();
 432   struct gomp_device_descr *acc_dev = thr->dev;
 433
 434   if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 435       || host_fallback)
 436     return;
 437
 438   if (num_waits)
 439     {
 440       va_list ap;
 441
 442       va_start (ap, num_waits);
 443       goacc_wait (async, num_waits, &ap);
 444       va_end (ap);
 445     }
 446
 447   acc_dev->openacc.async_set_async_func (async);
 448
 449   for (i = 0; i < mapnum; ++i)
 450     {
 451       unsigned char kind = kinds[i] & 0xff;
 452
 453       switch (kind)
 454         {
 455         case GOMP_MAP_POINTER:
 456         case GOMP_MAP_TO_PSET:
 457           break;
 458
 459         case GOMP_MAP_FORCE_TO:
 460           acc_update_device (hostaddrs[i], sizes[i]);
 461           break;
 462
 463         case GOMP_MAP_FORCE_FROM:
 464           acc_update_self (hostaddrs[i], sizes[i]);
 465           break;
 466
 467         default:
 468           gomp_fatal (">>>> GOACC_update UNHANDLED kind 0x%.2x", kind);
 469           break;
 470         }
 471     }
 472
 473   acc_dev->openacc.async_set_async_func (acc_async_sync);
 474 }
 475
 476 void
 477 GOACC_wait (int async, int num_waits, ...)
 478 {
 479   if (num_waits)
 480     {
 481       va_list ap;
 482
 483       va_start (ap, num_waits);
 484       goacc_wait (async, num_waits, &ap);
 485       va_end (ap);
 486     }
 487   else if (async == acc_async_sync)
 488     acc_wait_all ();
 489   else if (async == acc_async_noval)
 490     goacc_thread ()->dev->openacc.async_wait_all_async_func (acc_async_noval);
 491 }
 492
 493 void
 494 GOACC_host_data (int device, size_t mapnum,
 495                  void **hostaddrs, size_t *sizes, unsigned short *kinds)
 496 {
 497   bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
 498   struct target_mem_desc *tgt;
 499
 500 #ifdef HAVE_INTTYPES_H
 501   gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n",
 502               __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds);
 503 #else
 504   gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n",
 505               __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds);
 506 #endif
 507
 508   goacc_lazy_initialize ();
 509
 510   struct goacc_thread *thr = goacc_thread ();
 511   struct gomp_device_descr *acc_dev = thr->dev;
 512
 513   /* Host fallback or 'do nothing'.  */
 514   if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 515       || host_fallback)
 516     {
 517       tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true,
 518                            GOMP_MAP_VARS_OPENACC);
 519       tgt->prev = thr->mapped_data;
 520       thr->mapped_data = tgt;
 521
 522       return;
 523     }
 524
 525   gomp_debug (0, "  %s: prepare mappings\n", __FUNCTION__);
 526   tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true,
 527                        GOMP_MAP_VARS_OPENACC);
 528   gomp_debug (0, "  %s: mappings prepared\n", __FUNCTION__);
 529   tgt->prev = thr->mapped_data;
 530   thr->mapped_data = tgt;
 531 }
 532
 533 int
 534 GOACC_get_num_threads (void)
 535 {
 536   return 1;
 537 }
 538
 539 int
 540 GOACC_get_thread_num (void)
 541 {
 542   return 0;
 543 }
 544
 545 void
 546 GOACC_declare (int device, size_t mapnum,
 547                void **hostaddrs, size_t *sizes, unsigned short *kinds)
 548 {
 549   int i;
 550
 551   for (i = 0; i < mapnum; i++)
 552     {
 553       unsigned char kind = kinds[i] & 0xff;
 554
 555       if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET)
 556         continue;
 557
 558       switch (kind)
 559         {
 560           case GOMP_MAP_FORCE_ALLOC:
 561           case GOMP_MAP_FORCE_DEALLOC:
 562           case GOMP_MAP_FORCE_FROM:
 563           case GOMP_MAP_FORCE_TO:
 564           case GOMP_MAP_POINTER:
 565             GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
 566                                    &kinds[i], 0, 0);
 567             break;
 568
 569           case GOMP_MAP_FORCE_DEVICEPTR:
 570             break;
 571
 572           case GOMP_MAP_ALLOC:
 573             if (!acc_is_present (hostaddrs[i], sizes[i]))
 574               GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
 575                                      &kinds[i], 0, 0);
 576             break;
 577
 578           case GOMP_MAP_TO:
 579             GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
 580                                    &kinds[i], 0, 0);
 581
 582             break;
 583
 584           case GOMP_MAP_FROM:
 585             kinds[i] = GOMP_MAP_FORCE_FROM;
 586             GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
 587                                    &kinds[i], 0, 0);
 588             break;
 589
 590           case GOMP_MAP_FORCE_PRESENT:
 591             if (!acc_is_present (hostaddrs[i], sizes[i]))
 592               gomp_fatal ("[%p,%ld] is not mapped", hostaddrs[i],
 593                           (unsigned long) sizes[i]);
 594             break;
 595
 596           default:
 597             assert (0);
 598             break;
 599         }
 600     }
 601 }