libgomp/oacc-mem.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2015 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "openacc.h"
  30 #include "config.h"
  31 #include "libgomp.h"
  32 #include "gomp-constants.h"
  33 #include "oacc-int.h"
  34 #include "splay-tree.h"
  35 #include <stdint.h>
  36 #include <assert.h>
  37
  38 /* Return block containing [H->S), or NULL if not contained.  */
  39
  40 static splay_tree_key
  41 lookup_host (struct gomp_device_descr *dev, void *h, size_t s)
  42 {
  43   struct splay_tree_key_s node;
  44   splay_tree_key key;
  45
  46   node.host_start = (uintptr_t) h;
  47   node.host_end = (uintptr_t) h + s;
  48
  49   gomp_mutex_lock (&dev->lock);
  50   key = splay_tree_lookup (&dev->mem_map, &node);
  51   gomp_mutex_unlock (&dev->lock);
  52
  53   return key;
  54 }
  55
  56 /* Return block containing [D->S), or NULL if not contained.
  57    The list isn't ordered by device address, so we have to iterate
  58    over the whole array.  This is not expected to be a common
  59    operation.  */
  60
  61 static splay_tree_key
  62 lookup_dev (struct target_mem_desc *tgt, void *d, size_t s)
  63 {
  64   int i;
  65   struct target_mem_desc *t;
  66
  67   if (!tgt)
  68     return NULL;
  69
  70   gomp_mutex_lock (&tgt->device_descr->lock);
  71
  72   for (t = tgt; t != NULL; t = t->prev)
  73     {
  74       if (t->tgt_start <= (uintptr_t) d && t->tgt_end >= (uintptr_t) d + s)
  75         break;
  76     }
  77
  78   gomp_mutex_unlock (&tgt->device_descr->lock);
  79
  80   if (!t)
  81     return NULL;
  82
  83   for (i = 0; i < t->list_count; i++)
  84     {
  85       void * offset;
  86
  87       splay_tree_key k = &t->array[i].key;
  88       offset = d - t->tgt_start + k->tgt_offset;
  89
  90       if (k->host_start + offset <= (void *) k->host_end)
  91         return k;
  92     }
  93
  94   return NULL;
  95 }
  96
  97 /* OpenACC is silent on how memory exhaustion is indicated.  We return
  98    NULL.  */
  99
 100 void *
 101 acc_malloc (size_t s)
 102 {
 103   if (!s)
 104     return NULL;
 105
 106   goacc_lazy_initialize ();
 107
 108   struct goacc_thread *thr = goacc_thread ();
 109
 110   return base_dev->alloc_func (thr->dev->target_id, s);
 111 }
 112
 113 /* OpenACC 2.0a (3.2.16) doesn't specify what to do in the event
 114    the device address is mapped. We choose to check if it mapped,
 115    and if it is, to unmap it. */
 116 void
 117 acc_free (void *d)
 118 {
 119   splay_tree_key k;
 120   struct goacc_thread *thr = goacc_thread ();
 121
 122   if (!d)
 123     return;
 124
 125   /* We don't have to call lazy open here, as the ptr value must have
 126      been returned by acc_malloc.  It's not permitted to pass NULL in
 127      (unless you got that null from acc_malloc).  */
 128   if ((k = lookup_dev (thr->dev->openacc.data_environ, d, 1)))
 129    {
 130      void *offset;
 131
 132      offset = d - k->tgt->tgt_start + k->tgt_offset;
 133
 134      acc_unmap_data ((void *)(k->host_start + offset));
 135    }
 136
 137   base_dev->free_func (thr->dev->target_id, d);
 138 }
 139
 140 void
 141 acc_memcpy_to_device (void *d, void *h, size_t s)
 142 {
 143   /* No need to call lazy open here, as the device pointer must have
 144      been obtained from a routine that did that.  */
 145   struct goacc_thread *thr = goacc_thread ();
 146
 147   base_dev->host2dev_func (thr->dev->target_id, d, h, s);
 148 }
 149
 150 void
 151 acc_memcpy_from_device (void *h, void *d, size_t s)
 152 {
 153   /* No need to call lazy open here, as the device pointer must have
 154      been obtained from a routine that did that.  */
 155   struct goacc_thread *thr = goacc_thread ();
 156
 157   base_dev->dev2host_func (thr->dev->target_id, h, d, s);
 158 }
 159
 160 /* Return the device pointer that corresponds to host data H.  Or NULL
 161    if no mapping.  */
 162
 163 void *
 164 acc_deviceptr (void *h)
 165 {
 166   splay_tree_key n;
 167   void *d;
 168   void *offset;
 169
 170   goacc_lazy_initialize ();
 171
 172   struct goacc_thread *thr = goacc_thread ();
 173
 174   n = lookup_host (thr->dev, h, 1);
 175
 176   if (!n)
 177     return NULL;
 178
 179   offset = h - n->host_start;
 180
 181   d = n->tgt->tgt_start + n->tgt_offset + offset;
 182
 183   return d;
 184 }
 185
 186 /* Return the host pointer that corresponds to device data D.  Or NULL
 187    if no mapping.  */
 188
 189 void *
 190 acc_hostptr (void *d)
 191 {
 192   splay_tree_key n;
 193   void *h;
 194   void *offset;
 195
 196   goacc_lazy_initialize ();
 197
 198   struct goacc_thread *thr = goacc_thread ();
 199
 200   n = lookup_dev (thr->dev->openacc.data_environ, d, 1);
 201
 202   if (!n)
 203     return NULL;
 204
 205   offset = d - n->tgt->tgt_start + n->tgt_offset;
 206
 207   h = n->host_start + offset;
 208
 209   return h;
 210 }
 211
 212 /* Return 1 if host data [H,+S] is present on the device.  */
 213
 214 int
 215 acc_is_present (void *h, size_t s)
 216 {
 217   splay_tree_key n;
 218
 219   if (!s || !h)
 220     return 0;
 221
 222   goacc_lazy_initialize ();
 223
 224   struct goacc_thread *thr = goacc_thread ();
 225   struct gomp_device_descr *acc_dev = thr->dev;
 226
 227   n = lookup_host (acc_dev, h, s);
 228
 229   if (n && ((uintptr_t)h < n->host_start
 230             || (uintptr_t)h + s > n->host_end
 231             || s > n->host_end - n->host_start))
 232     n = NULL;
 233
 234   return n != NULL;
 235 }
 236
 237 /* Create a mapping for host [H,+S] -> device [D,+S] */
 238
 239 void
 240 acc_map_data (void *h, void *d, size_t s)
 241 {
 242   struct target_mem_desc *tgt;
 243   size_t mapnum = 1;
 244   void *hostaddrs = h;
 245   void *devaddrs = d;
 246   size_t sizes = s;
 247   unsigned short kinds = GOMP_MAP_ALLOC;
 248
 249   goacc_lazy_initialize ();
 250
 251   struct goacc_thread *thr = goacc_thread ();
 252   struct gomp_device_descr *acc_dev = thr->dev;
 253
 254   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 255     {
 256       if (d != h)
 257         gomp_fatal ("cannot map data on shared-memory system");
 258
 259       tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true, false);
 260     }
 261   else
 262     {
 263       struct goacc_thread *thr = goacc_thread ();
 264
 265       if (!d || !h || !s)
 266         gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
 267                     (void *)h, (int)s, (void *)d, (int)s);
 268
 269       if (lookup_host (acc_dev, h, s))
 270         gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h,
 271                     (int)s);
 272
 273       if (lookup_dev (thr->dev->openacc.data_environ, d, s))
 274         gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d,
 275                     (int)s);
 276
 277       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, &devaddrs, &sizes,
 278                            &kinds, true, false);
 279     }
 280
 281   tgt->prev = acc_dev->openacc.data_environ;
 282   acc_dev->openacc.data_environ = tgt;
 283 }
 284
 285 void
 286 acc_unmap_data (void *h)
 287 {
 288   struct goacc_thread *thr = goacc_thread ();
 289   struct gomp_device_descr *acc_dev = thr->dev;
 290
 291   /* No need to call lazy open, as the address must have been mapped.  */
 292
 293   size_t host_size;
 294   splay_tree_key n = lookup_host (acc_dev, h, 1);
 295   struct target_mem_desc *t;
 296
 297   if (!n)
 298     gomp_fatal ("%p is not a mapped block", (void *)h);
 299
 300   host_size = n->host_end - n->host_start;
 301
 302   if (n->host_start != (uintptr_t) h)
 303     gomp_fatal ("[%p,%d] surrounds1 %p",
 304                 (void *) n->host_start, (int) host_size, (void *) h);
 305
 306   t = n->tgt;
 307
 308   if (t->refcount == 2)
 309     {
 310       struct target_mem_desc *tp;
 311
 312       /* This is the last reference, so pull the descriptor off the
 313          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 314          freeing the device memory. */
 315       t->tgt_end = 0;
 316       t->to_free = 0;
 317
 318       gomp_mutex_lock (&acc_dev->lock);
 319
 320       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 321            tp = t, t = t->prev)
 322         if (n->tgt == t)
 323           {
 324             if (tp)
 325               tp->prev = t->prev;
 326             else
 327               acc_dev->openacc.data_environ = t->prev;
 328
 329             break;
 330           }
 331
 332       gomp_mutex_unlock (&acc_dev->lock);
 333     }
 334
 335   gomp_unmap_vars (t, true);
 336 }
 337
 338 #define FLAG_PRESENT (1 << 0)
 339 #define FLAG_CREATE (1 << 1)
 340 #define FLAG_COPY (1 << 2)
 341
 342 static void *
 343 present_create_copy (unsigned f, void *h, size_t s)
 344 {
 345   void *d;
 346   splay_tree_key n;
 347
 348   if (!h || !s)
 349     gomp_fatal ("[%p,+%d] is a bad range", (void *)h, (int)s);
 350
 351   goacc_lazy_initialize ();
 352
 353   struct goacc_thread *thr = goacc_thread ();
 354   struct gomp_device_descr *acc_dev = thr->dev;
 355
 356   n = lookup_host (acc_dev, h, s);
 357   if (n)
 358     {
 359       /* Present. */
 360       d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 361
 362       if (!(f & FLAG_PRESENT))
 363         gomp_fatal ("[%p,+%d] already mapped to [%p,+%d]",
 364             (void *)h, (int)s, (void *)d, (int)s);
 365       if ((h + s) > (void *)n->host_end)
 366         gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 367     }
 368   else if (!(f & FLAG_CREATE))
 369     {
 370       gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 371     }
 372   else
 373     {
 374       struct target_mem_desc *tgt;
 375       size_t mapnum = 1;
 376       unsigned short kinds;
 377       void *hostaddrs = h;
 378
 379       if (f & FLAG_COPY)
 380         kinds = GOMP_MAP_TO;
 381       else
 382         kinds = GOMP_MAP_ALLOC;
 383
 384       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, NULL, &s, &kinds, true,
 385                            false);
 386
 387       gomp_mutex_lock (&acc_dev->lock);
 388
 389       d = tgt->to_free;
 390       tgt->prev = acc_dev->openacc.data_environ;
 391       acc_dev->openacc.data_environ = tgt;
 392
 393       gomp_mutex_unlock (&acc_dev->lock);
 394     }
 395
 396   return d;
 397 }
 398
 399 void *
 400 acc_create (void *h, size_t s)
 401 {
 402   return present_create_copy (FLAG_CREATE, h, s);
 403 }
 404
 405 void *
 406 acc_copyin (void *h, size_t s)
 407 {
 408   return present_create_copy (FLAG_CREATE | FLAG_COPY, h, s);
 409 }
 410
 411 void *
 412 acc_present_or_create (void *h, size_t s)
 413 {
 414   return present_create_copy (FLAG_PRESENT | FLAG_CREATE, h, s);
 415 }
 416
 417 void *
 418 acc_present_or_copyin (void *h, size_t s)
 419 {
 420   return present_create_copy (FLAG_PRESENT | FLAG_CREATE | FLAG_COPY, h, s);
 421 }
 422
 423 #define FLAG_COPYOUT (1 << 0)
 424
 425 static void
 426 delete_copyout (unsigned f, void *h, size_t s)
 427 {
 428   size_t host_size;
 429   splay_tree_key n;
 430   void *d;
 431   struct goacc_thread *thr = goacc_thread ();
 432   struct gomp_device_descr *acc_dev = thr->dev;
 433
 434   n = lookup_host (acc_dev, h, s);
 435
 436   /* No need to call lazy open, as the data must already have been
 437      mapped.  */
 438
 439   if (!n)
 440     gomp_fatal ("[%p,%d] is not mapped", (void *)h, (int)s);
 441
 442   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 443
 444   host_size = n->host_end - n->host_start;
 445
 446   if (n->host_start != (uintptr_t) h || host_size != s)
 447     gomp_fatal ("[%p,%d] surrounds2 [%p,+%d]",
 448                 (void *) n->host_start, (int) host_size, (void *) h, (int) s);
 449
 450   if (f & FLAG_COPYOUT)
 451     acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 452
 453   acc_unmap_data (h);
 454
 455   acc_dev->free_func (acc_dev->target_id, d);
 456 }
 457
 458 void
 459 acc_delete (void *h , size_t s)
 460 {
 461   delete_copyout (0, h, s);
 462 }
 463
 464 void acc_copyout (void *h, size_t s)
 465 {
 466   delete_copyout (FLAG_COPYOUT, h, s);
 467 }
 468
 469 static void
 470 update_dev_host (int is_dev, void *h, size_t s)
 471 {
 472   splay_tree_key n;
 473   void *d;
 474   struct goacc_thread *thr = goacc_thread ();
 475   struct gomp_device_descr *acc_dev = thr->dev;
 476
 477   n = lookup_host (acc_dev, h, s);
 478
 479   /* No need to call lazy open, as the data must already have been
 480      mapped.  */
 481
 482   if (!n)
 483     gomp_fatal ("[%p,%d] is not mapped", h, (int)s);
 484
 485   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 486
 487   if (is_dev)
 488     acc_dev->host2dev_func (acc_dev->target_id, d, h, s);
 489   else
 490     acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 491 }
 492
 493 void
 494 acc_update_device (void *h, size_t s)
 495 {
 496   update_dev_host (1, h, s);
 497 }
 498
 499 void
 500 acc_update_self (void *h, size_t s)
 501 {
 502   update_dev_host (0, h, s);
 503 }
 504
 505 void
 506 gomp_acc_insert_pointer (size_t mapnum, void **hostaddrs, size_t *sizes,
 507                          void *kinds)
 508 {
 509   struct target_mem_desc *tgt;
 510   struct goacc_thread *thr = goacc_thread ();
 511   struct gomp_device_descr *acc_dev = thr->dev;
 512
 513   gomp_debug (0, "  %s: prepare mappings\n", __FUNCTION__);
 514   tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs,
 515                        NULL, sizes, kinds, true, false);
 516   gomp_debug (0, "  %s: mappings prepared\n", __FUNCTION__);
 517   tgt->prev = acc_dev->openacc.data_environ;
 518   acc_dev->openacc.data_environ = tgt;
 519 }
 520
 521 void
 522 gomp_acc_remove_pointer (void *h, bool force_copyfrom, int async, int mapnum)
 523 {
 524   struct goacc_thread *thr = goacc_thread ();
 525   struct gomp_device_descr *acc_dev = thr->dev;
 526   splay_tree_key n;
 527   struct target_mem_desc *t;
 528   int minrefs = (mapnum == 1) ? 2 : 3;
 529
 530   n = lookup_host (acc_dev, h, 1);
 531
 532   if (!n)
 533     gomp_fatal ("%p is not a mapped block", (void *)h);
 534
 535   gomp_debug (0, "  %s: restore mappings\n", __FUNCTION__);
 536
 537   t = n->tgt;
 538
 539   struct target_mem_desc *tp;
 540
 541   gomp_mutex_lock (&acc_dev->lock);
 542
 543   if (t->refcount == minrefs)
 544     {
 545       /* This is the last reference, so pull the descriptor off the
 546          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 547          freeing the device memory. */
 548       t->tgt_end = 0;
 549       t->to_free = 0;
 550
 551       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 552            tp = t, t = t->prev)
 553         {
 554           if (n->tgt == t)
 555             {
 556               if (tp)
 557                 tp->prev = t->prev;
 558               else
 559                 acc_dev->openacc.data_environ = t->prev;
 560               break;
 561             }
 562         }
 563     }
 564
 565   if (force_copyfrom)
 566     t->list[0]->copy_from = 1;
 567
 568   gomp_mutex_unlock (&acc_dev->lock);
 569
 570   /* If running synchronously, unmap immediately.  */
 571   if (async < acc_async_noval)
 572     gomp_unmap_vars (t, true);
 573   else
 574     {
 575       gomp_copy_from_async (t);
 576       acc_dev->openacc.register_async_cleanup_func (t);
 577     }
 578
 579   gomp_debug (0, "  %s: mappings restored\n", __FUNCTION__);
 580 }