libgomp/oacc-mem.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2015 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "openacc.h"
  30 #include "config.h"
  31 #include "libgomp.h"
  32 #include "gomp-constants.h"
  33 #include "oacc-int.h"
  34 #include "splay-tree.h"
  35 #include <stdint.h>
  36 #include <assert.h>
  37
  38 /* Return block containing [H->S), or NULL if not contained.  */
  39
  40 static splay_tree_key
  41 lookup_host (struct gomp_memory_mapping *mem_map, void *h, size_t s)
  42 {
  43   struct splay_tree_key_s node;
  44   splay_tree_key key;
  45
  46   node.host_start = (uintptr_t) h;
  47   node.host_end = (uintptr_t) h + s;
  48
  49   gomp_mutex_lock (&mem_map->lock);
  50
  51   key = splay_tree_lookup (&mem_map->splay_tree, &node);
  52
  53   gomp_mutex_unlock (&mem_map->lock);
  54
  55   return key;
  56 }
  57
  58 /* Return block containing [D->S), or NULL if not contained.
  59    The list isn't ordered by device address, so we have to iterate
  60    over the whole array.  This is not expected to be a common
  61    operation.  */
  62
  63 static splay_tree_key
  64 lookup_dev (struct target_mem_desc *tgt, void *d, size_t s)
  65 {
  66   int i;
  67   struct target_mem_desc *t;
  68   struct gomp_memory_mapping *mem_map;
  69
  70   if (!tgt)
  71     return NULL;
  72
  73   mem_map = tgt->mem_map;
  74
  75   gomp_mutex_lock (&mem_map->lock);
  76
  77   for (t = tgt; t != NULL; t = t->prev)
  78     {
  79       if (t->tgt_start <= (uintptr_t) d && t->tgt_end >= (uintptr_t) d + s)
  80         break;
  81     }
  82
  83   gomp_mutex_unlock (&mem_map->lock);
  84
  85   if (!t)
  86     return NULL;
  87
  88   for (i = 0; i < t->list_count; i++)
  89     {
  90       void * offset;
  91
  92       splay_tree_key k = &t->array[i].key;
  93       offset = d - t->tgt_start + k->tgt_offset;
  94
  95       if (k->host_start + offset <= (void *) k->host_end)
  96         return k;
  97     }
  98
  99   return NULL;
 100 }
 101
 102 /* OpenACC is silent on how memory exhaustion is indicated.  We return
 103    NULL.  */
 104
 105 void *
 106 acc_malloc (size_t s)
 107 {
 108   if (!s)
 109     return NULL;
 110
 111   goacc_lazy_initialize ();
 112
 113   struct goacc_thread *thr = goacc_thread ();
 114
 115   return base_dev->alloc_func (thr->dev->target_id, s);
 116 }
 117
 118 /* OpenACC 2.0a (3.2.16) doesn't specify what to do in the event
 119    the device address is mapped. We choose to check if it mapped,
 120    and if it is, to unmap it. */
 121 void
 122 acc_free (void *d)
 123 {
 124   splay_tree_key k;
 125   struct goacc_thread *thr = goacc_thread ();
 126
 127   if (!d)
 128     return;
 129
 130   /* We don't have to call lazy open here, as the ptr value must have
 131      been returned by acc_malloc.  It's not permitted to pass NULL in
 132      (unless you got that null from acc_malloc).  */
 133   if ((k = lookup_dev (thr->dev->openacc.data_environ, d, 1)))
 134    {
 135      void *offset;
 136
 137      offset = d - k->tgt->tgt_start + k->tgt_offset;
 138
 139      acc_unmap_data ((void *)(k->host_start + offset));
 140    }
 141
 142   base_dev->free_func (thr->dev->target_id, d);
 143 }
 144
 145 void
 146 acc_memcpy_to_device (void *d, void *h, size_t s)
 147 {
 148   /* No need to call lazy open here, as the device pointer must have
 149      been obtained from a routine that did that.  */
 150   struct goacc_thread *thr = goacc_thread ();
 151
 152   base_dev->host2dev_func (thr->dev->target_id, d, h, s);
 153 }
 154
 155 void
 156 acc_memcpy_from_device (void *h, void *d, size_t s)
 157 {
 158   /* No need to call lazy open here, as the device pointer must have
 159      been obtained from a routine that did that.  */
 160   struct goacc_thread *thr = goacc_thread ();
 161
 162   base_dev->dev2host_func (thr->dev->target_id, h, d, s);
 163 }
 164
 165 /* Return the device pointer that corresponds to host data H.  Or NULL
 166    if no mapping.  */
 167
 168 void *
 169 acc_deviceptr (void *h)
 170 {
 171   splay_tree_key n;
 172   void *d;
 173   void *offset;
 174
 175   goacc_lazy_initialize ();
 176
 177   struct goacc_thread *thr = goacc_thread ();
 178
 179   n = lookup_host (&thr->dev->mem_map, h, 1);
 180
 181   if (!n)
 182     return NULL;
 183
 184   offset = h - n->host_start;
 185
 186   d = n->tgt->tgt_start + n->tgt_offset + offset;
 187
 188   return d;
 189 }
 190
 191 /* Return the host pointer that corresponds to device data D.  Or NULL
 192    if no mapping.  */
 193
 194 void *
 195 acc_hostptr (void *d)
 196 {
 197   splay_tree_key n;
 198   void *h;
 199   void *offset;
 200
 201   goacc_lazy_initialize ();
 202
 203   struct goacc_thread *thr = goacc_thread ();
 204
 205   n = lookup_dev (thr->dev->openacc.data_environ, d, 1);
 206
 207   if (!n)
 208     return NULL;
 209
 210   offset = d - n->tgt->tgt_start + n->tgt_offset;
 211
 212   h = n->host_start + offset;
 213
 214   return h;
 215 }
 216
 217 /* Return 1 if host data [H,+S] is present on the device.  */
 218
 219 int
 220 acc_is_present (void *h, size_t s)
 221 {
 222   splay_tree_key n;
 223
 224   if (!s || !h)
 225     return 0;
 226
 227   goacc_lazy_initialize ();
 228
 229   struct goacc_thread *thr = goacc_thread ();
 230   struct gomp_device_descr *acc_dev = thr->dev;
 231
 232   n = lookup_host (&acc_dev->mem_map, h, s);
 233
 234   if (n && ((uintptr_t)h < n->host_start
 235             || (uintptr_t)h + s > n->host_end
 236             || s > n->host_end - n->host_start))
 237     n = NULL;
 238
 239   return n != NULL;
 240 }
 241
 242 /* Create a mapping for host [H,+S] -> device [D,+S] */
 243
 244 void
 245 acc_map_data (void *h, void *d, size_t s)
 246 {
 247   struct target_mem_desc *tgt;
 248   size_t mapnum = 1;
 249   void *hostaddrs = h;
 250   void *devaddrs = d;
 251   size_t sizes = s;
 252   unsigned short kinds = GOMP_MAP_ALLOC;
 253
 254   goacc_lazy_initialize ();
 255
 256   struct goacc_thread *thr = goacc_thread ();
 257   struct gomp_device_descr *acc_dev = thr->dev;
 258
 259   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 260     {
 261       if (d != h)
 262         gomp_fatal ("cannot map data on shared-memory system");
 263
 264       tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true, false);
 265     }
 266   else
 267     {
 268       struct goacc_thread *thr = goacc_thread ();
 269
 270       if (!d || !h || !s)
 271         gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
 272                     (void *)h, (int)s, (void *)d, (int)s);
 273
 274       if (lookup_host (&acc_dev->mem_map, h, s))
 275         gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h,
 276                     (int)s);
 277
 278       if (lookup_dev (thr->dev->openacc.data_environ, d, s))
 279         gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d,
 280                     (int)s);
 281
 282       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, &devaddrs, &sizes,
 283                            &kinds, true, false);
 284     }
 285
 286   tgt->prev = acc_dev->openacc.data_environ;
 287   acc_dev->openacc.data_environ = tgt;
 288 }
 289
 290 void
 291 acc_unmap_data (void *h)
 292 {
 293   struct goacc_thread *thr = goacc_thread ();
 294   struct gomp_device_descr *acc_dev = thr->dev;
 295
 296   /* No need to call lazy open, as the address must have been mapped.  */
 297
 298   size_t host_size;
 299   splay_tree_key n = lookup_host (&acc_dev->mem_map, h, 1);
 300   struct target_mem_desc *t;
 301
 302   if (!n)
 303     gomp_fatal ("%p is not a mapped block", (void *)h);
 304
 305   host_size = n->host_end - n->host_start;
 306
 307   if (n->host_start != (uintptr_t) h)
 308     gomp_fatal ("[%p,%d] surrounds1 %p",
 309                 (void *) n->host_start, (int) host_size, (void *) h);
 310
 311   t = n->tgt;
 312
 313   if (t->refcount == 2)
 314     {
 315       struct target_mem_desc *tp;
 316
 317       /* This is the last reference, so pull the descriptor off the
 318          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 319          freeing the device memory. */
 320       t->tgt_end = 0;
 321       t->to_free = 0;
 322
 323       gomp_mutex_lock (&acc_dev->mem_map.lock);
 324
 325       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 326            tp = t, t = t->prev)
 327         if (n->tgt == t)
 328           {
 329             if (tp)
 330               tp->prev = t->prev;
 331             else
 332               acc_dev->openacc.data_environ = t->prev;
 333
 334             break;
 335           }
 336
 337       gomp_mutex_unlock (&acc_dev->mem_map.lock);
 338     }
 339
 340   gomp_unmap_vars (t, true);
 341 }
 342
 343 #define FLAG_PRESENT (1 << 0)
 344 #define FLAG_CREATE (1 << 1)
 345 #define FLAG_COPY (1 << 2)
 346
 347 static void *
 348 present_create_copy (unsigned f, void *h, size_t s)
 349 {
 350   void *d;
 351   splay_tree_key n;
 352
 353   if (!h || !s)
 354     gomp_fatal ("[%p,+%d] is a bad range", (void *)h, (int)s);
 355
 356   goacc_lazy_initialize ();
 357
 358   struct goacc_thread *thr = goacc_thread ();
 359   struct gomp_device_descr *acc_dev = thr->dev;
 360
 361   n = lookup_host (&acc_dev->mem_map, h, s);
 362   if (n)
 363     {
 364       /* Present. */
 365       d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 366
 367       if (!(f & FLAG_PRESENT))
 368         gomp_fatal ("[%p,+%d] already mapped to [%p,+%d]",
 369             (void *)h, (int)s, (void *)d, (int)s);
 370       if ((h + s) > (void *)n->host_end)
 371         gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 372     }
 373   else if (!(f & FLAG_CREATE))
 374     {
 375       gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 376     }
 377   else
 378     {
 379       struct target_mem_desc *tgt;
 380       size_t mapnum = 1;
 381       unsigned short kinds;
 382       void *hostaddrs = h;
 383
 384       if (f & FLAG_COPY)
 385         kinds = GOMP_MAP_TO;
 386       else
 387         kinds = GOMP_MAP_ALLOC;
 388
 389       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, NULL, &s, &kinds, true,
 390                            false);
 391
 392       gomp_mutex_lock (&acc_dev->mem_map.lock);
 393
 394       d = tgt->to_free;
 395       tgt->prev = acc_dev->openacc.data_environ;
 396       acc_dev->openacc.data_environ = tgt;
 397
 398       gomp_mutex_unlock (&acc_dev->mem_map.lock);
 399     }
 400
 401   return d;
 402 }
 403
 404 void *
 405 acc_create (void *h, size_t s)
 406 {
 407   return present_create_copy (FLAG_CREATE, h, s);
 408 }
 409
 410 void *
 411 acc_copyin (void *h, size_t s)
 412 {
 413   return present_create_copy (FLAG_CREATE | FLAG_COPY, h, s);
 414 }
 415
 416 void *
 417 acc_present_or_create (void *h, size_t s)
 418 {
 419   return present_create_copy (FLAG_PRESENT | FLAG_CREATE, h, s);
 420 }
 421
 422 void *
 423 acc_present_or_copyin (void *h, size_t s)
 424 {
 425   return present_create_copy (FLAG_PRESENT | FLAG_CREATE | FLAG_COPY, h, s);
 426 }
 427
 428 #define FLAG_COPYOUT (1 << 0)
 429
 430 static void
 431 delete_copyout (unsigned f, void *h, size_t s)
 432 {
 433   size_t host_size;
 434   splay_tree_key n;
 435   void *d;
 436   struct goacc_thread *thr = goacc_thread ();
 437   struct gomp_device_descr *acc_dev = thr->dev;
 438
 439   n = lookup_host (&acc_dev->mem_map, h, s);
 440
 441   /* No need to call lazy open, as the data must already have been
 442      mapped.  */
 443
 444   if (!n)
 445     gomp_fatal ("[%p,%d] is not mapped", (void *)h, (int)s);
 446
 447   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 448
 449   host_size = n->host_end - n->host_start;
 450
 451   if (n->host_start != (uintptr_t) h || host_size != s)
 452     gomp_fatal ("[%p,%d] surrounds2 [%p,+%d]",
 453                 (void *) n->host_start, (int) host_size, (void *) h, (int) s);
 454
 455   if (f & FLAG_COPYOUT)
 456     acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 457
 458   acc_unmap_data (h);
 459
 460   acc_dev->free_func (acc_dev->target_id, d);
 461 }
 462
 463 void
 464 acc_delete (void *h , size_t s)
 465 {
 466   delete_copyout (0, h, s);
 467 }
 468
 469 void acc_copyout (void *h, size_t s)
 470 {
 471   delete_copyout (FLAG_COPYOUT, h, s);
 472 }
 473
 474 static void
 475 update_dev_host (int is_dev, void *h, size_t s)
 476 {
 477   splay_tree_key n;
 478   void *d;
 479   struct goacc_thread *thr = goacc_thread ();
 480   struct gomp_device_descr *acc_dev = thr->dev;
 481
 482   n = lookup_host (&acc_dev->mem_map, h, s);
 483
 484   /* No need to call lazy open, as the data must already have been
 485      mapped.  */
 486
 487   if (!n)
 488     gomp_fatal ("[%p,%d] is not mapped", h, (int)s);
 489
 490   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 491
 492   if (is_dev)
 493     acc_dev->host2dev_func (acc_dev->target_id, d, h, s);
 494   else
 495     acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 496 }
 497
 498 void
 499 acc_update_device (void *h, size_t s)
 500 {
 501   update_dev_host (1, h, s);
 502 }
 503
 504 void
 505 acc_update_self (void *h, size_t s)
 506 {
 507   update_dev_host (0, h, s);
 508 }
 509
 510 void
 511 gomp_acc_insert_pointer (size_t mapnum, void **hostaddrs, size_t *sizes,
 512                          void *kinds)
 513 {
 514   struct target_mem_desc *tgt;
 515   struct goacc_thread *thr = goacc_thread ();
 516   struct gomp_device_descr *acc_dev = thr->dev;
 517
 518   gomp_debug (0, "  %s: prepare mappings\n", __FUNCTION__);
 519   tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs,
 520                        NULL, sizes, kinds, true, false);
 521   gomp_debug (0, "  %s: mappings prepared\n", __FUNCTION__);
 522   tgt->prev = acc_dev->openacc.data_environ;
 523   acc_dev->openacc.data_environ = tgt;
 524 }
 525
 526 void
 527 gomp_acc_remove_pointer (void *h, bool force_copyfrom, int async, int mapnum)
 528 {
 529   struct goacc_thread *thr = goacc_thread ();
 530   struct gomp_device_descr *acc_dev = thr->dev;
 531   splay_tree_key n;
 532   struct target_mem_desc *t;
 533   int minrefs = (mapnum == 1) ? 2 : 3;
 534
 535   n = lookup_host (&acc_dev->mem_map, h, 1);
 536
 537   if (!n)
 538     gomp_fatal ("%p is not a mapped block", (void *)h);
 539
 540   gomp_debug (0, "  %s: restore mappings\n", __FUNCTION__);
 541
 542   t = n->tgt;
 543
 544   struct target_mem_desc *tp;
 545
 546   gomp_mutex_lock (&acc_dev->mem_map.lock);
 547
 548   if (t->refcount == minrefs)
 549     {
 550       /* This is the last reference, so pull the descriptor off the
 551          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 552          freeing the device memory. */
 553       t->tgt_end = 0;
 554       t->to_free = 0;
 555
 556       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 557            tp = t, t = t->prev)
 558         {
 559           if (n->tgt == t)
 560             {
 561               if (tp)
 562                 tp->prev = t->prev;
 563               else
 564                 acc_dev->openacc.data_environ = t->prev;
 565               break;
 566             }
 567         }
 568     }
 569
 570   if (force_copyfrom)
 571     t->list[0]->copy_from = 1;
 572
 573   gomp_mutex_unlock (&acc_dev->mem_map.lock);
 574
 575   /* If running synchronously, unmap immediately.  */
 576   if (async < acc_async_noval)
 577     gomp_unmap_vars (t, true);
 578   else
 579     {
 580       gomp_copy_from_async (t);
 581       acc_dev->openacc.register_async_cleanup_func (t);
 582     }
 583
 584   gomp_debug (0, "  %s: mappings restored\n", __FUNCTION__);
 585 }