libgomp/oacc-mem.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2014 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU OpenMP Library (libgomp).
   8
   9    Libgomp is free software; you can redistribute it and/or modify it
  10    under the terms of the GNU General Public License as published by
  11    the Free Software Foundation; either version 3, or (at your option)
  12    any later version.
  13
  14    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  15    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  16    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  17    more details.
  18
  19    Under Section 7 of GPL version 3, you are granted additional
  20    permissions described in the GCC Runtime Library Exception, version
  21    3.1, as published by the Free Software Foundation.
  22
  23    You should have received a copy of the GNU General Public License and
  24    a copy of the GCC Runtime Library Exception along with this program;
  25    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  26    <http://www.gnu.org/licenses/>.  */
  27
  28 #include "openacc.h"
  29 #include "config.h"
  30 #include "libgomp.h"
  31 #include "gomp-constants.h"
  32 #include "target.h"
  33 #include "oacc-int.h"
  34 #include <stdio.h>
  35 #include <stdint.h>
  36 #include <assert.h>
  37
  38 #include "splay-tree.h"
  39
  40 /* Return block containing [H->S), or NULL if not contained.  */
  41
  42 attribute_hidden splay_tree_key
  43 lookup_host (struct gomp_memory_mapping *mem_map, void *h, size_t s)
  44 {
  45   struct splay_tree_key_s node;
  46   splay_tree_key key;
  47
  48   node.host_start = (uintptr_t) h;
  49   node.host_end = (uintptr_t) h + s;
  50
  51   gomp_mutex_lock (&mem_map->lock);
  52
  53   key = splay_tree_lookup (&mem_map->splay_tree, &node);
  54
  55   gomp_mutex_unlock (&mem_map->lock);
  56
  57   return key;
  58 }
  59
  60 /* Return block containing [D->S), or NULL if not contained.
  61    The list isn't ordered by device address, so we have to iterate
  62    over the whole array.  This is not expected to be a common
  63    operation.  */
  64
  65 static splay_tree_key
  66 lookup_dev (struct target_mem_desc *tgt, void *d, size_t s)
  67 {
  68   int i;
  69   struct target_mem_desc *t;
  70   struct gomp_memory_mapping *mem_map;
  71
  72   if (!tgt)
  73     return NULL;
  74
  75   mem_map = tgt->mem_map;
  76
  77   gomp_mutex_lock (&mem_map->lock);
  78
  79   for (t = tgt; t != NULL; t = t->prev)
  80     {
  81       if (t->tgt_start <= (uintptr_t) d && t->tgt_end >= (uintptr_t) d + s)
  82         break;
  83     }
  84
  85   gomp_mutex_unlock (&mem_map->lock);
  86
  87   if (!t)
  88     return NULL;
  89
  90   for (i = 0; i < t->list_count; i++)
  91     {
  92       void * offset;
  93
  94       splay_tree_key k = &t->array[i].key;
  95       offset = d - t->tgt_start + k->tgt_offset;
  96
  97       if (k->host_start + offset <= (void *) k->host_end)
  98         return k;
  99     }
 100
 101   return NULL;
 102 }
 103
 104 /* OpenACC is silent on how memory exhaustion is indicated.  We return
 105    NULL.  */
 106
 107 void *
 108 acc_malloc (size_t s)
 109 {
 110   if (!s)
 111     return NULL;
 112
 113   ACC_lazy_initialize ();
 114
 115   return base_dev->alloc_func (s);
 116 }
 117
 118 /* OpenACC 2.0a (3.2.16) doesn't specify what to do in the event
 119    the device address is mapped. We choose to check if it mapped,
 120    and if it is, to unmap it. */
 121 void
 122 acc_free (void *d)
 123 {
 124   splay_tree_key k;
 125   struct goacc_thread *thr = goacc_thread ();
 126
 127   if (!d)
 128     return;
 129
 130   /* We don't have to call lazy open here, as the ptr value must have
 131      been returned by acc_malloc.  It's not permitted to pass NULL in
 132      (unless you got that null from acc_malloc).  */
 133   if ((k = lookup_dev (thr->dev->openacc.data_environ, d, 1)))
 134    {
 135      void *offset;
 136
 137      offset = d - k->tgt->tgt_start + k->tgt_offset;
 138
 139      acc_unmap_data ((void *)(k->host_start + offset));
 140    }
 141
 142   base_dev->free_func (d);
 143 }
 144
 145 void
 146 acc_memcpy_to_device (void *d, void *h, size_t s)
 147 {
 148   /* No need to call lazy open here, as the device pointer must have
 149      been obtained from a routine that did that.  */
 150   base_dev->host2dev_func (d, h, s);
 151 }
 152
 153 void
 154 acc_memcpy_from_device (void *h, void *d, size_t s)
 155 {
 156   /* No need to call lazy open here, as the device pointer must have
 157      been obtained from a routine that did that.  */
 158   base_dev->dev2host_func (h, d, s);
 159 }
 160
 161 /* Return the device pointer that corresponds to host data H.  Or NULL
 162    if no mapping.  */
 163
 164 void *
 165 acc_deviceptr (void *h)
 166 {
 167   splay_tree_key n;
 168   void *d;
 169   void *offset;
 170
 171   ACC_lazy_initialize ();
 172
 173   struct goacc_thread *thr = goacc_thread ();
 174
 175   n = lookup_host (&thr->dev->mem_map, h, 1);
 176
 177   if (!n)
 178     return NULL;
 179
 180   offset = h - n->host_start;
 181
 182   d = n->tgt->tgt_start + n->tgt_offset + offset;
 183
 184   return d;
 185 }
 186
 187 /* Return the host pointer that corresponds to device data D.  Or NULL
 188    if no mapping.  */
 189
 190 void *
 191 acc_hostptr (void *d)
 192 {
 193   splay_tree_key n;
 194   void *h;
 195   void *offset;
 196
 197   ACC_lazy_initialize ();
 198
 199   struct goacc_thread *thr = goacc_thread ();
 200
 201   n = lookup_dev (thr->dev->openacc.data_environ, d, 1);
 202
 203   if (!n)
 204     return NULL;
 205
 206   offset = d - n->tgt->tgt_start + n->tgt_offset;
 207
 208   h = n->host_start + offset;
 209
 210   return h;
 211 }
 212
 213 /* Return 1 if host data [H,+S] is present on the device.  */
 214
 215 int
 216 acc_is_present (void *h, size_t s)
 217 {
 218   splay_tree_key n;
 219
 220   if (!s || !h)
 221     return 0;
 222
 223   ACC_lazy_initialize ();
 224
 225   struct goacc_thread *thr = goacc_thread ();
 226   struct gomp_device_descr *acc_dev = thr->dev;
 227
 228   n = lookup_host (&acc_dev->mem_map, h, s);
 229
 230   if (n && ((uintptr_t)h < n->host_start
 231             || (uintptr_t)h + s > n->host_end
 232             || s > n->host_end - n->host_start))
 233     n = NULL;
 234
 235   return n != NULL;
 236 }
 237
 238 /* Create a mapping for host [H,+S] -> device [D,+S] */
 239
 240 void
 241 acc_map_data (void *h, void *d, size_t s)
 242 {
 243   struct target_mem_desc *tgt;
 244   size_t mapnum = 1;
 245   void *hostaddrs = h;
 246   void *devaddrs = d;
 247   size_t sizes = s;
 248   unsigned short kinds = GOMP_MAP_ALLOC;
 249
 250   ACC_lazy_initialize ();
 251
 252   struct goacc_thread *thr = goacc_thread ();
 253   struct gomp_device_descr *acc_dev = thr->dev;
 254
 255   if (acc_dev->capabilities & TARGET_CAP_SHARED_MEM)
 256     {
 257       if (d != h)
 258         gomp_fatal ("cannot map data on shared-memory system");
 259
 260       tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true, false);
 261     }
 262   else
 263     {
 264       struct goacc_thread *thr = goacc_thread ();
 265
 266       if (!d || !h || !s)
 267         gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
 268                     (void *)h, (int)s, (void *)d, (int)s);
 269
 270       if (lookup_host (&acc_dev->mem_map, h, s))
 271         gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h,
 272                     (int)s);
 273
 274       if (lookup_dev (thr->dev->openacc.data_environ, d, s))
 275         gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d,
 276                     (int)s);
 277
 278       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, &devaddrs, &sizes,
 279                            &kinds, true, false);
 280     }
 281
 282   tgt->prev = acc_dev->openacc.data_environ;
 283   acc_dev->openacc.data_environ = tgt;
 284 }
 285
 286 void
 287 acc_unmap_data (void *h)
 288 {
 289   struct goacc_thread *thr = goacc_thread ();
 290   struct gomp_device_descr *acc_dev = thr->dev;
 291
 292   /* No need to call lazy open, as the address must have been mapped.  */
 293
 294   size_t host_size;
 295   splay_tree_key n = lookup_host (&acc_dev->mem_map, h, 1);
 296   struct target_mem_desc *t;
 297
 298   if (!n)
 299     gomp_fatal ("%p is not a mapped block", (void *)h);
 300
 301   host_size = n->host_end - n->host_start;
 302
 303   if (n->host_start != (uintptr_t) h)
 304     gomp_fatal ("[%p,%d] surrounds1 %p",
 305                 (void *) n->host_start, (int) host_size, (void *) h);
 306
 307   t = n->tgt;
 308
 309   if (t->refcount == 2)
 310     {
 311       struct target_mem_desc *tp;
 312
 313       /* This is the last reference, so pull the descriptor off the
 314          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 315          freeing the device memory. */
 316       t->tgt_end = 0;
 317       t->to_free = 0;
 318
 319       gomp_mutex_lock (&acc_dev->mem_map.lock);
 320
 321       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 322            tp = t, t = t->prev)
 323         if (n->tgt == t)
 324           {
 325             if (tp)
 326               tp->prev = t->prev;
 327             else
 328               acc_dev->openacc.data_environ = t->prev;
 329
 330             break;
 331           }
 332
 333       gomp_mutex_unlock (&acc_dev->mem_map.lock);
 334     }
 335
 336   gomp_unmap_vars (t, true);
 337 }
 338
 339 #define PCC_Present (1 << 0)
 340 #define PCC_Create (1 << 1)
 341 #define PCC_Copy (1 << 2)
 342
 343 attribute_hidden void *
 344 present_create_copy (unsigned f, void *h, size_t s)
 345 {
 346   void *d;
 347   splay_tree_key n;
 348
 349   if (!h || !s)
 350     gomp_fatal ("[%p,+%d] is a bad range", (void *)h, (int)s);
 351
 352   ACC_lazy_initialize ();
 353
 354   struct goacc_thread *thr = goacc_thread ();
 355   struct gomp_device_descr *acc_dev = thr->dev;
 356
 357   n = lookup_host (&acc_dev->mem_map, h, s);
 358   if (n)
 359     {
 360       /* Present. */
 361       d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 362
 363       if (!(f & PCC_Present))
 364         gomp_fatal ("[%p,+%d] already mapped to [%p,+%d]",
 365             (void *)h, (int)s, (void *)d, (int)s);
 366       if ((h + s) > (void *)n->host_end)
 367         gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 368     }
 369   else if (!(f & PCC_Create))
 370     {
 371       gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 372     }
 373   else
 374     {
 375       struct target_mem_desc *tgt;
 376       size_t mapnum = 1;
 377       unsigned short kinds;
 378       void *hostaddrs = h;
 379
 380       if (f & PCC_Copy)
 381         kinds = GOMP_MAP_ALLOC_TO;
 382       else
 383         kinds = GOMP_MAP_ALLOC;
 384
 385       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, NULL, &s, &kinds, true,
 386                            false);
 387
 388       gomp_mutex_lock (&acc_dev->mem_map.lock);
 389
 390       d = tgt->to_free;
 391       tgt->prev = acc_dev->openacc.data_environ;
 392       acc_dev->openacc.data_environ = tgt;
 393
 394       gomp_mutex_unlock (&acc_dev->mem_map.lock);
 395     }
 396
 397   return d;
 398 }
 399
 400 void *
 401 acc_create (void *h, size_t s)
 402 {
 403   return present_create_copy (PCC_Create, h, s);
 404 }
 405
 406 void *
 407 acc_copyin (void *h, size_t s)
 408 {
 409   return present_create_copy (PCC_Create | PCC_Copy, h, s);
 410 }
 411
 412 void *
 413 acc_present_or_create (void *h, size_t s)
 414 {
 415   return present_create_copy (PCC_Present | PCC_Create, h, s);
 416 }
 417
 418 void *
 419 acc_present_or_copyin (void *h, size_t s)
 420 {
 421   return present_create_copy (PCC_Present | PCC_Create | PCC_Copy, h, s);
 422 }
 423
 424 #define DC_Copyout (1 << 0)
 425
 426 static void
 427 delete_copyout (unsigned f, void *h, size_t s)
 428 {
 429   size_t host_size;
 430   splay_tree_key n;
 431   void *d;
 432   struct goacc_thread *thr = goacc_thread ();
 433   struct gomp_device_descr *acc_dev = thr->dev;
 434
 435   n = lookup_host (&acc_dev->mem_map, h, s);
 436
 437   /* No need to call lazy open, as the data must already have been
 438      mapped.  */
 439
 440   if (!n)
 441     gomp_fatal ("[%p,%d] is not mapped", (void *)h, (int)s);
 442
 443   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 444
 445   host_size = n->host_end - n->host_start;
 446
 447   if (n->host_start != (uintptr_t) h || host_size != s)
 448     gomp_fatal ("[%p,%d] surrounds2 [%p,+%d]",
 449                 (void *) n->host_start, (int) host_size, (void *) h, (int) s);
 450
 451   if (f & DC_Copyout)
 452     acc_dev->dev2host_func (h, d, s);
 453
 454   acc_unmap_data (h);
 455
 456   acc_dev->free_func (d);
 457 }
 458
 459 void
 460 acc_delete (void *h , size_t s)
 461 {
 462   delete_copyout (0, h, s);
 463 }
 464
 465 void acc_copyout (void *h, size_t s)
 466 {
 467   delete_copyout (DC_Copyout, h, s);
 468 }
 469
 470 static void
 471 update_dev_host (int is_dev, void *h, size_t s)
 472 {
 473   splay_tree_key n;
 474   void *d;
 475   struct goacc_thread *thr = goacc_thread ();
 476   struct gomp_device_descr *acc_dev = thr->dev;
 477
 478   n = lookup_host (&acc_dev->mem_map, h, s);
 479
 480   /* No need to call lazy open, as the data must already have been
 481      mapped.  */
 482
 483   if (!n)
 484     gomp_fatal ("[%p,%d] is not mapped", h, (int)s);
 485
 486   d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 487
 488   if (is_dev)
 489     acc_dev->host2dev_func (d, h, s);
 490   else
 491     acc_dev->dev2host_func (h, d, s);
 492 }
 493
 494 void
 495 acc_update_device (void *h, size_t s)
 496 {
 497   update_dev_host (1, h, s);
 498 }
 499
 500 void
 501 acc_update_self (void *h, size_t s)
 502 {
 503   update_dev_host (0, h, s);
 504 }
 505
 506 void
 507 gomp_acc_insert_pointer (size_t mapnum, void **hostaddrs, size_t *sizes,
 508                          void *kinds)
 509 {
 510   struct target_mem_desc *tgt;
 511   struct goacc_thread *thr = goacc_thread ();
 512   struct gomp_device_descr *acc_dev = thr->dev;
 513
 514   gomp_notify ("  %s: prepare mappings\n", __FUNCTION__);
 515   tgt = gomp_map_vars ((struct gomp_device_descr *) acc_dev, mapnum, hostaddrs,
 516                        NULL, sizes, kinds, true, false);
 517   gomp_notify ("  %s: mappings prepared\n", __FUNCTION__);
 518   tgt->prev = acc_dev->openacc.data_environ;
 519   acc_dev->openacc.data_environ = tgt;
 520 }
 521
 522 void
 523 gomp_acc_remove_pointer (void *h, bool force_copyfrom, int async, int mapnum)
 524 {
 525   struct goacc_thread *thr = goacc_thread ();
 526   struct gomp_device_descr *acc_dev = thr->dev;
 527   splay_tree_key n;
 528   struct target_mem_desc *t;
 529   int minrefs = (mapnum == 1) ? 2 : 3;
 530
 531   n = lookup_host (&acc_dev->mem_map, h, 1);
 532
 533   if (!n)
 534     gomp_fatal ("%p is not a mapped block", (void *)h);
 535
 536   gomp_notify ("  %s: restore mappings\n", __FUNCTION__);
 537
 538   t = n->tgt;
 539
 540   struct target_mem_desc *tp;
 541
 542   gomp_mutex_lock (&acc_dev->mem_map.lock);
 543
 544   if (t->refcount == minrefs)
 545     {
 546       /* This is the last reference, so pull the descriptor off the
 547          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 548          freeing the device memory. */
 549       t->tgt_end = 0;
 550       t->to_free = 0;
 551
 552       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 553            tp = t, t = t->prev)
 554         {
 555           if (n->tgt == t)
 556             {
 557               if (tp)
 558                 tp->prev = t->prev;
 559               else
 560                 acc_dev->openacc.data_environ = t->prev;
 561               break;
 562             }
 563         }
 564     }
 565
 566   if (force_copyfrom)
 567     t->list[0]->copy_from = 1;
 568
 569   gomp_mutex_unlock (&acc_dev->mem_map.lock);
 570
 571   /* If running synchronously, unmap immediately.  */
 572   if (async < acc_async_noval)
 573     gomp_unmap_vars (t, true);
 574   else
 575     {
 576       gomp_copy_from_async (t);
 577       acc_dev->openacc.register_async_cleanup_func (t);
 578     }
 579
 580   gomp_notify ("  %s: mappings restored\n", __FUNCTION__);
 581 }