libgomp/oacc-mem.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2014 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU OpenMP Library (libgomp).
   8
   9    Libgomp is free software; you can redistribute it and/or modify it
  10    under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 3, or (at your option)
  12    any later version.
  13
  14    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  15    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  16    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  17    more details.
  18
  19    Under Section 7 of GPL version 3, you are granted additional
  20    permissions described in the GCC Runtime Library Exception, version
  21    3.1, as published by the Free Software Foundation.
  22
  23    You should have received a copy of the GNU General Public License and
  24    a copy of the GCC Runtime Library Exception along with this program;
  25    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  26    <http://www.gnu.org/licenses/>.  */
  27
  28 #include "openacc.h"
  29 #include "config.h"
  30 #include "libgomp.h"
  31 #include "gomp-constants.h"
  32 #include "libgomp_target.h"
  33 #include "oacc-int.h"
  34 #include <stdio.h>
  35 #include <stdint.h>
  36 #include <assert.h>
  37
  38 #include "splay-tree.h"
  39
  40 /* Return block containing [H->S), or NULL if not contained.  */
  41
  42 attribute_hidden splay_tree_key
  43 lookup_host (struct gomp_memory_mapping *mem_map, void *h, size_t s)
  44 {
  45   struct splay_tree_key_s node;
  46   splay_tree_key key;
  47
  48   node.host_start = (uintptr_t) h;
  49   node.host_end = (uintptr_t) h + s;
  50
  51   gomp_mutex_lock (&mem_map->lock);
  52
  53   key = splay_tree_lookup (&mem_map->splay_tree, &node);
  54
  55   gomp_mutex_unlock (&mem_map->lock);
  56
  57   return key;
  58 }
  59
  60 /* Return block containing [D->S), or NULL if not contained.
  61    The list isn't ordered by device address, so we have to iterate
  62    over the whole array.  This is not expected to be a common
  63    operation.  */
  64
  65 static splay_tree_key
  66 lookup_dev (struct target_mem_desc *tgt, void *d, size_t s)
  67 {
  68   int i;
  69   struct target_mem_desc *t;
  70   struct gomp_memory_mapping *mem_map;
  71
  72   if (!tgt)
  73     return NULL;
  74
  75   mem_map = tgt->mem_map;
  76
  77   gomp_mutex_lock (&mem_map->lock);
  78
  79   for (t = tgt; t != NULL; t = t->prev)
  80     {
  81       if (t->tgt_start <= (uintptr_t) d && t->tgt_end >= (uintptr_t) d + s)
  82         break;
  83     }
  84
  85   gomp_mutex_unlock (&mem_map->lock);
  86
  87   if (!t)
  88     return NULL;
  89
  90   for (i = 0; i < t->list_count; i++)
  91     {
  92       void * offset;
  93
  94       splay_tree_key k = &t->array[i].key;
  95       offset = d - t->tgt_start + k->tgt_offset;
  96
  97       if (k->host_start + offset <= (void *) k->host_end)
  98         return k;
  99     }
 100
 101   return NULL;
 102 }
 103
 104 /* OpenACC is silent on how memory exhaustion is indicated.  We return
 105    NULL.  */
 106
 107 void *
 108 acc_malloc (size_t s)
 109 {
 110   if (!s)
 111     return NULL;
 112
 113   ACC_lazy_initialize ();
 114
 115   struct goacc_thread *thr = goacc_thread ();
 116
 117   return base_dev->alloc_func (thr->dev->target_id, s);
 118 }
 119
 120 /* OpenACC 2.0a (3.2.16) doesn't specify what to do in the event
 121    the device address is mapped. We choose to check if it mapped,
 122    and if it is, to unmap it. */
 123 void
 124 acc_free (void *d)
 125 {
 126   splay_tree_key k;
 127   struct goacc_thread *thr = goacc_thread ();
 128
 129   if (!d)
 130     return;
 131
 132   /* We don't have to call lazy open here, as the ptr value must have
 133      been returned by acc_malloc.  It's not permitted to pass NULL in
 134      (unless you got that null from acc_malloc).  */
 135   if ((k = lookup_dev (thr->dev->openacc.data_environ, d, 1)))
 136    {
 137      void *offset;
 138
 139      offset = d - k->tgt->tgt_start + k->tgt_offset;
 140
 141      acc_unmap_data ((void *)(k->host_start + offset));
 142    }
 143
 144   base_dev->free_func (thr->dev->target_id, d);
 145 }
 146
 147 void
 148 acc_memcpy_to_device (void *d, void *h, size_t s)
 149 {
 150   /* No need to call lazy open here, as the device pointer must have
 151      been obtained from a routine that did that.  */
 152   struct goacc_thread *thr = goacc_thread ();
 153
 154   base_dev->host2dev_func (thr->dev->target_id, d, h, s);
 155 }
 156
 157 void
 158 acc_memcpy_from_device (void *h, void *d, size_t s)
 159 {
 160   /* No need to call lazy open here, as the device pointer must have
 161      been obtained from a routine that did that.  */
 162   struct goacc_thread *thr = goacc_thread ();
 163
 164   base_dev->dev2host_func (thr->dev->target_id, h, d, s);
 165 }
 166
 167 /* Return the device pointer that corresponds to host data H.  Or NULL
 168    if no mapping.  */
 169
 170 void *
 171 acc_deviceptr (void *h)
 172 {
 173   splay_tree_key n;
 174   void *d;
 175   void *offset;
 176
 177   ACC_lazy_initialize ();
 178
 179   struct goacc_thread *thr = goacc_thread ();
 180
 181   n = lookup_host (&thr->dev->mem_map, h, 1);
 182
 183   if (!n)
 184     return NULL;
 185
 186   offset = h - n->host_start;
 187
 188   d = n->tgt->tgt_start + n->tgt_offset + offset;
 189
 190   return d;
 191 }
 192
 193 /* Return the host pointer that corresponds to device data D.  Or NULL
 194    if no mapping.  */
 195
 196 void *
 197 acc_hostptr (void *d)
 198 {
 199   splay_tree_key n;
 200   void *h;
 201   void *offset;
 202
 203   ACC_lazy_initialize ();
 204
 205   struct goacc_thread *thr = goacc_thread ();
 206
 207   n = lookup_dev (thr->dev->openacc.data_environ, d, 1);
 208
 209   if (!n)
 210     return NULL;
 211
 212   offset = d - n->tgt->tgt_start + n->tgt_offset;
 213
 214   h = n->host_start + offset;
 215
 216   return h;
 217 }
 218
 219 /* Return 1 if host data [H,+S] is present on the device.  */
 220
 221 int
 222 acc_is_present (void *h, size_t s)
 223 {
 224   splay_tree_key n;
 225
 226   if (!s || !h)
 227     return 0;
 228
 229   ACC_lazy_initialize ();
 230
 231   struct goacc_thread *thr = goacc_thread ();
 232   struct gomp_device_descr *acc_dev = thr->dev;
 233
 234   n = lookup_host (&acc_dev->mem_map, h, s);
 235
 236   if (n && ((uintptr_t)h < n->host_start
 237             || (uintptr_t)h + s > n->host_end
 238             || s > n->host_end - n->host_start))
 239     n = NULL;
 240
 241   return n != NULL;
 242 }
 243
 244 /* Create a mapping for host [H,+S] -> device [D,+S] */
 245
 246 void
 247 acc_map_data (void *h, void *d, size_t s)
 248 {
 249   struct target_mem_desc *tgt;
 250   size_t mapnum = 1;
 251   void *hostaddrs = h;
 252   void *devaddrs = d;
 253   size_t sizes = s;
 254   unsigned short kinds = GOMP_MAP_ALLOC;
 255
 256   ACC_lazy_initialize ();
 257
 258   struct goacc_thread *thr = goacc_thread ();
 259   struct gomp_device_descr *acc_dev = thr->dev;
 260
 261   if (acc_dev->capabilities & TARGET_CAP_SHARED_MEM)
 262     {
 263       if (d != h)
 264         gomp_fatal ("cannot map data on shared-memory system");
 265
 266       tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true, false);
 267     }
 268   else
 269     {
 270       struct goacc_thread *thr = goacc_thread ();
 271
 272       if (!d || !h || !s)
 273         gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
 274                     (void *)h, (int)s, (void *)d, (int)s);
 275
 276       if (lookup_host (&acc_dev->mem_map, h, s))
 277         gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h,
 278                     (int)s);
 279
 280       if (lookup_dev (thr->dev->openacc.data_environ, d, s))
 281         gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d,
 282                     (int)s);
 283
 284       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, &devaddrs, &sizes,
 285                            &kinds, true, false);
 286     }
 287
 288   tgt->prev = acc_dev->openacc.data_environ;
 289   acc_dev->openacc.data_environ = tgt;
 290 }
 291
 292 void
 293 acc_unmap_data (void *h)
 294 {
 295   struct goacc_thread *thr = goacc_thread ();
 296   struct gomp_device_descr *acc_dev = thr->dev;
 297
 298   /* No need to call lazy open, as the address must have been mapped.  */
 299
 300   size_t host_size;
 301   splay_tree_key n = lookup_host (&acc_dev->mem_map, h, 1);
 302   struct target_mem_desc *t;
 303
 304   if (!n)
 305     gomp_fatal ("%p is not a mapped block", (void *)h);
 306
 307   host_size = n->host_end - n->host_start;
 308
 309   if (n->host_start != (uintptr_t) h)
 310     gomp_fatal ("[%p,%d] surrounds1 %p",
 311                 (void *) n->host_start, (int) host_size, (void *) h);
 312
 313   t = n->tgt;
 314
 315   if (t->refcount == 2)
 316     {
 317       struct target_mem_desc *tp;
 318
 319       /* This is the last reference, so pull the descriptor off the
 320          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 321          freeing the device memory. */
 322       t->tgt_end = 0;
 323       t->to_free = 0;
 324
 325       gomp_mutex_lock (&acc_dev->mem_map.lock);
 326
 327       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 328            tp = t, t = t->prev)
 329         if (n->tgt == t)
 330           {
 331             if (tp)
 332               tp->prev = t->prev;
 333             else
 334               acc_dev->openacc.data_environ = t->prev;
 335
 336             break;
 337           }
 338
 339       gomp_mutex_unlock (&acc_dev->mem_map.lock);
 340     }
 341
 342   gomp_unmap_vars (t, true);
 343 }
 344
 345 #define PCC_Present (1 << 0)
 346 #define PCC_Create (1 << 1)
 347 #define PCC_Copy (1 << 2)
 348
 349 attribute_hidden void *
 350 present_create_copy (unsigned f, void *h, size_t s)
 351 {
 352   void *d;
 353   splay_tree_key n;
 354
 355   if (!h || !s)
 356     gomp_fatal ("[%p,+%d] is a bad range", (void *)h, (int)s);
 357
 358   ACC_lazy_initialize ();
 359
 360   struct goacc_thread *thr = goacc_thread ();
 361   struct gomp_device_descr *acc_dev = thr->dev;
 362
 363   n = lookup_host (&acc_dev->mem_map, h, s);
 364   if (n)
 365     {
 366       /* Present. */
 367       d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 368
 369       if (!(f & PCC_Present))
 370         gomp_fatal ("[%p,+%d] already mapped to [%p,+%d]",
 371             (void *)h, (int)s, (void *)d, (int)s);
 372       if ((h + s) > (void *)n->host_end)
 373         gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 374     }
 375   else if (!(f & PCC_Create))
 376     {
 377       gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 378     }
 379   else
 380     {
 381       struct target_mem_desc *tgt;
 382       size_t mapnum = 1;
 383       unsigned short kinds;
 384       void *hostaddrs = h;
 385
 386       if (f & PCC_Copy)
 387         kinds = GOMP_MAP_ALLOC_TO;
 388       else
 389         kinds = GOMP_MAP_ALLOC;
 390
 391       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, NULL, &s, &kinds, true,
 392                            false);
 393
 394       gomp_mutex_lock (&acc_dev->mem_map.lock);
 395
 396       d = tgt->to_free;
 397       tgt->prev = acc_dev->openacc.data_environ;
 398       acc_dev->openacc.data_environ = tgt;
 399
 400       gomp_mutex_unlock (&acc_dev->mem_map.lock);
 401     }
 402
 403   return d;
 404 }
 405
 406 void *
 407 acc_create (void *h, size_t s)
 408 {
 409   return present_create_copy (PCC_Create, h, s);
 410 }
 411
 412 void *
 413 acc_copyin (void *h, size_t s)
 414 {
 415   return present_create_copy (PCC_Create | PCC_Copy, h, s);
 416 }
 417
 418 void *
 419 acc_present_or_create (void *h, size_t s)
 420 {
 421   return present_create_copy (PCC_Present | PCC_Create, h, s);
 422 }
 423
 424 void *
 425 acc_present_or_copyin (void *h, size_t s)
 426 {
 427   return present_create_copy (PCC_Present | PCC_Create | PCC_Copy, h, s);
 428 }
 429
 430 #define DC_Copyout (1 << 0)
 431
 432 static void
 433 delete_copyout (unsigned f, void *h, size_t s)
 434 {
 435   size_t host_size;
 436   splay_tree_key n;
 437   void *d;
 438   struct goacc_thread *thr = goacc_thread ();
 439   struct gomp_device_descr *acc_dev = thr->dev;
 440
 441   n = lookup_host (&acc_dev->mem_map, h, s);
 442
 443   /* No need to call lazy open, as the data must already have been
 444      mapped.  */
 445
 446   if (!n)
 447     gomp_fatal ("[%p,%d] is not mapped", (void *)h, (int)s);
 448
 449   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 450
 451   host_size = n->host_end - n->host_start;
 452
 453   if (n->host_start != (uintptr_t) h || host_size != s)
 454     gomp_fatal ("[%p,%d] surrounds2 [%p,+%d]",
 455                 (void *) n->host_start, (int) host_size, (void *) h, (int) s);
 456
 457   if (f & DC_Copyout)
 458     acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 459
 460   acc_unmap_data (h);
 461
 462   acc_dev->free_func (acc_dev->target_id, d);
 463 }
 464
 465 void
 466 acc_delete (void *h , size_t s)
 467 {
 468   delete_copyout (0, h, s);
 469 }
 470
 471 void acc_copyout (void *h, size_t s)
 472 {
 473   delete_copyout (DC_Copyout, h, s);
 474 }
 475
 476 static void
 477 update_dev_host (int is_dev, void *h, size_t s)
 478 {
 479   splay_tree_key n;
 480   void *d;
 481   struct goacc_thread *thr = goacc_thread ();
 482   struct gomp_device_descr *acc_dev = thr->dev;
 483
 484   n = lookup_host (&acc_dev->mem_map, h, s);
 485
 486   /* No need to call lazy open, as the data must already have been
 487      mapped.  */
 488
 489   if (!n)
 490     gomp_fatal ("[%p,%d] is not mapped", h, (int)s);
 491
 492   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 493
 494   if (is_dev)
 495     acc_dev->host2dev_func (acc_dev->target_id, d, h, s);
 496   else
 497     acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 498 }
 499
 500 void
 501 acc_update_device (void *h, size_t s)
 502 {
 503   update_dev_host (1, h, s);
 504 }
 505
 506 void
 507 acc_update_self (void *h, size_t s)
 508 {
 509   update_dev_host (0, h, s);
 510 }
 511
 512 void
 513 gomp_acc_insert_pointer (size_t mapnum, void **hostaddrs, size_t *sizes,
 514                          void *kinds)
 515 {
 516   struct target_mem_desc *tgt;
 517   struct goacc_thread *thr = goacc_thread ();
 518   struct gomp_device_descr *acc_dev = thr->dev;
 519
 520   gomp_notify ("  %s: prepare mappings\n", __FUNCTION__);
 521   tgt = gomp_map_vars ((struct gomp_device_descr *) acc_dev, mapnum, hostaddrs,
 522                        NULL, sizes, kinds, true, false);
 523   gomp_notify ("  %s: mappings prepared\n", __FUNCTION__);
 524   tgt->prev = acc_dev->openacc.data_environ;
 525   acc_dev->openacc.data_environ = tgt;
 526 }
 527
 528 void
 529 gomp_acc_remove_pointer (void *h, bool force_copyfrom, int async, int mapnum)
 530 {
 531   struct goacc_thread *thr = goacc_thread ();
 532   struct gomp_device_descr *acc_dev = thr->dev;
 533   splay_tree_key n;
 534   struct target_mem_desc *t;
 535   int minrefs = (mapnum == 1) ? 2 : 3;
 536
 537   n = lookup_host (&acc_dev->mem_map, h, 1);
 538
 539   if (!n)
 540     gomp_fatal ("%p is not a mapped block", (void *)h);
 541
 542   gomp_notify ("  %s: restore mappings\n", __FUNCTION__);
 543
 544   t = n->tgt;
 545
 546   struct target_mem_desc *tp;
 547
 548   gomp_mutex_lock (&acc_dev->mem_map.lock);
 549
 550   if (t->refcount == minrefs)
 551     {
 552       /* This is the last reference, so pull the descriptor off the
 553          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 554          freeing the device memory. */
 555       t->tgt_end = 0;
 556       t->to_free = 0;
 557
 558       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 559            tp = t, t = t->prev)
 560         {
 561           if (n->tgt == t)
 562             {
 563               if (tp)
 564                 tp->prev = t->prev;
 565               else
 566                 acc_dev->openacc.data_environ = t->prev;
 567               break;
 568             }
 569         }
 570     }
 571
 572   if (force_copyfrom)
 573     t->list[0]->copy_from = 1;
 574
 575   gomp_mutex_unlock (&acc_dev->mem_map.lock);
 576
 577   /* If running synchronously, unmap immediately.  */
 578   if (async < acc_async_noval)
 579     gomp_unmap_vars (t, true);
 580   else
 581     {
 582       gomp_copy_from_async (t);
 583       acc_dev->openacc.register_async_cleanup_func (t);
 584     }
 585
 586   gomp_notify ("  %s: mappings restored\n", __FUNCTION__);
 587 }