tree-optimization/112450 - avoid AVX512 style masking for BImode masks

[official-gcc.git] / libgomp / oacc-mem.c

/* OpenACC Runtime initialization routines

   Copyright (C) 2013-2023 Free Software Foundation, Inc.

   Contributed by Mentor Embedded.

   This file is part of the GNU Offloading and Multi Processing Library
   (libgomp).

   Libgomp is free software; you can redistribute it and/or modify it
   under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3, or (at your option)
   any later version.

   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
   more details.

   Under Section 7 of GPL version 3, you are granted additional
   permissions described in the GCC Runtime Library Exception, version
   3.1, as published by the Free Software Foundation.

   You should have received a copy of the GNU General Public License and
   a copy of the GCC Runtime Library Exception along with this program;
   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
   <http://www.gnu.org/licenses/>.  */

#include "openacc.h"
#include "libgomp.h"
#include "gomp-constants.h"
#include "oacc-int.h"
#include <string.h>
#include <assert.h>

/* Return block containing [H->S), or NULL if not contained.  The device lock
   for DEV must be locked on entry, and remains locked on exit.  */

static splay_tree_key
lookup_host (struct gomp_device_descr *dev, void *h, size_t s)
{
  struct splay_tree_key_s node;
  splay_tree_key key;

  node.host_start = (uintptr_t) h;
  node.host_end = (uintptr_t) h + s;

  key = splay_tree_lookup (&dev->mem_map, &node);

  return key;
}

/* Helper for lookup_dev.  Iterate over splay tree.  */

static splay_tree_key
lookup_dev_1 (splay_tree_node node, uintptr_t d, size_t s)
{
  splay_tree_key key = &node->key;
  if (d >= key->tgt->tgt_start && d + s <= key->tgt->tgt_end)
    return key;

  key = NULL;
  if (node->left)
    key = lookup_dev_1 (node->left, d, s);
  if (!key && node->right)
    key = lookup_dev_1 (node->right, d, s);

  return key;
}

/* Return block containing [D->S), or NULL if not contained.

   This iterates over the splay tree.  This is not expected to be a common
   operation.

   The device lock associated with MEM_MAP must be locked on entry, and remains
   locked on exit.  */

static splay_tree_key
lookup_dev (splay_tree mem_map, void *d, size_t s)
{
  if (!mem_map || !mem_map->root)
    return NULL;

  return lookup_dev_1 (mem_map->root, (uintptr_t) d, s);
}


/* OpenACC is silent on how memory exhaustion is indicated.  We return
   NULL.  */

void *
acc_malloc (size_t s)
{
  if (!s)
    return NULL;

  goacc_lazy_initialize ();

  struct goacc_thread *thr = goacc_thread ();

  assert (thr->dev);

  if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return malloc (s);

  acc_prof_info prof_info;
  acc_api_info api_info;
  bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);

  void *res = thr->dev->alloc_func (thr->dev->target_id, s);

  if (profiling_p)
    {
      thr->prof_info = NULL;
      thr->api_info = NULL;
    }

  return res;
}

void
acc_free (void *d)
{
  splay_tree_key k;

  if (!d)
    return;

  struct goacc_thread *thr = goacc_thread ();

  assert (thr && thr->dev);

  struct gomp_device_descr *acc_dev = thr->dev;

  if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return free (d);

  acc_prof_info prof_info;
  acc_api_info api_info;
  bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);

  gomp_mutex_lock (&acc_dev->lock);

  /* We don't have to call lazy open here, as the ptr value must have
     been returned by acc_malloc.  It's not permitted to pass NULL in
     (unless you got that null from acc_malloc).  */
  if ((k = lookup_dev (&acc_dev->mem_map, d, 1)))
    {
      void *offset = d - k->tgt->tgt_start + k->tgt_offset;
      void *h = k->host_start + offset;
      size_t h_size = k->host_end - k->host_start;
      gomp_mutex_unlock (&acc_dev->lock);
      /* PR92503 "[OpenACC] Behavior of 'acc_free' if the memory space is still
         used in a mapping".  */
      gomp_fatal ("refusing to free device memory space at %p that is still"
                  " mapped at [%p,+%d]",
                  d, h, (int) h_size);
    }
  else
    gomp_mutex_unlock (&acc_dev->lock);

  if (!acc_dev->free_func (acc_dev->target_id, d))
    gomp_fatal ("error in freeing device memory in %s", __FUNCTION__);

  if (profiling_p)
    {
      thr->prof_info = NULL;
      thr->api_info = NULL;
    }
}

static void
memcpy_tofrom_device (bool from, void *d, void *h, size_t s, int async,
                      const char *libfnname)
{
  /* No need to call lazy open here, as the device pointer must have
     been obtained from a routine that did that.  */
  struct goacc_thread *thr = goacc_thread ();

  assert (thr && thr->dev);

  if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    {
      if (from)
        memmove (h, d, s);
      else
        memmove (d, h, s);
      return;
    }

  acc_prof_info prof_info;
  acc_api_info api_info;
  bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);
  if (profiling_p)
    {
      prof_info.async = async;
      prof_info.async_queue = prof_info.async;
    }

  goacc_aq aq = get_goacc_asyncqueue (async);
  if (from)
    gomp_copy_dev2host (thr->dev, aq, h, d, s);
  else
    gomp_copy_host2dev (thr->dev, aq, d, h, s, false, /* TODO: cbuf? */ NULL);

  if (profiling_p)
    {
      thr->prof_info = NULL;
      thr->api_info = NULL;
    }
}

void
acc_memcpy_to_device (void *d, void *h, size_t s)
{
  memcpy_tofrom_device (false, d, h, s, acc_async_sync, __FUNCTION__);
}

void
acc_memcpy_to_device_async (void *d, void *h, size_t s, int async)
{
  memcpy_tofrom_device (false, d, h, s, async, __FUNCTION__);
}

void
acc_memcpy_from_device (void *h, void *d, size_t s)
{
  memcpy_tofrom_device (true, d, h, s, acc_async_sync, __FUNCTION__);
}

void
acc_memcpy_from_device_async (void *h, void *d, size_t s, int async)
{
  memcpy_tofrom_device (true, d, h, s, async, __FUNCTION__);
}

/* Return the device pointer that corresponds to host data H.  Or NULL
   if no mapping.  */

void *
acc_deviceptr (void *h)
{
  splay_tree_key n;
  void *d;
  void *offset;

  goacc_lazy_initialize ();

  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *dev = thr->dev;

  if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return h;

  /* In the following, no OpenACC Profiling Interface events can possibly be
     generated.  */

  gomp_mutex_lock (&dev->lock);

  n = lookup_host (dev, h, 1);

  if (!n)
    {
      gomp_mutex_unlock (&dev->lock);
      return NULL;
    }

  offset = h - n->host_start;

  d = n->tgt->tgt_start + n->tgt_offset + offset;

  gomp_mutex_unlock (&dev->lock);

  return d;
}

/* Return the host pointer that corresponds to device data D.  Or NULL
   if no mapping.  */

void *
acc_hostptr (void *d)
{
  splay_tree_key n;
  void *h;
  void *offset;

  goacc_lazy_initialize ();

  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return d;

  /* In the following, no OpenACC Profiling Interface events can possibly be
     generated.  */

  gomp_mutex_lock (&acc_dev->lock);

  n = lookup_dev (&acc_dev->mem_map, d, 1);

  if (!n)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      return NULL;
    }

  offset = d - n->tgt->tgt_start + n->tgt_offset;

  h = n->host_start + offset;

  gomp_mutex_unlock (&acc_dev->lock);

  return h;
}

/* Return 1 if host data [H,+S] is present on the device.  */

int
acc_is_present (void *h, size_t s)
{
  splay_tree_key n;

  if (!s || !h)
    return 0;

  goacc_lazy_initialize ();

  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return h != NULL;

  /* In the following, no OpenACC Profiling Interface events can possibly be
     generated.  */

  gomp_mutex_lock (&acc_dev->lock);

  n = lookup_host (acc_dev, h, s);

  if (n && ((uintptr_t)h < n->host_start
            || (uintptr_t)h + s > n->host_end
            || s > n->host_end - n->host_start))
    n = NULL;

  gomp_mutex_unlock (&acc_dev->lock);

  return n != NULL;
}

/* Create a mapping for host [H,+S] -> device [D,+S] */

void
acc_map_data (void *h, void *d, size_t s)
{
  size_t mapnum = 1;
  void *hostaddrs = h;
  void *devaddrs = d;
  size_t sizes = s;
  unsigned short kinds = GOMP_MAP_ALLOC;

  goacc_lazy_initialize ();

  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    {
      if (d != h)
        gomp_fatal ("cannot map data on shared-memory system");
    }
  else
    {
      struct goacc_thread *thr = goacc_thread ();

      if (!d || !h || !s)
        gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
                    (void *)h, (int)s, (void *)d, (int)s);

      acc_prof_info prof_info;
      acc_api_info api_info;
      bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);

      gomp_mutex_lock (&acc_dev->lock);

      if (lookup_host (acc_dev, h, s))
        {
          gomp_mutex_unlock (&acc_dev->lock);
          gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h,
                      (int)s);
        }

      if (lookup_dev (&thr->dev->mem_map, d, s))
        {
          gomp_mutex_unlock (&acc_dev->lock);
          gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d,
                      (int)s);
        }

      gomp_mutex_unlock (&acc_dev->lock);

      struct target_mem_desc *tgt
        = goacc_map_vars (acc_dev, NULL, mapnum, &hostaddrs, &devaddrs, &sizes,
                          &kinds, true, GOMP_MAP_VARS_ENTER_DATA);
      assert (tgt);
      assert (tgt->list_count == 1);
      splay_tree_key n = tgt->list[0].key;
      assert (n);
      assert (n->refcount == 1);
      assert (n->dynamic_refcount == 0);
      /* Special reference counting behavior.  */
      n->refcount = REFCOUNT_INFINITY;

      if (profiling_p)
        {
          thr->prof_info = NULL;
          thr->api_info = NULL;
        }
    }
}

void
acc_unmap_data (void *h)
{
  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  /* No need to call lazy open, as the address must have been mapped.  */

  /* This is a no-op on shared-memory targets.  */
  if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return;

  acc_prof_info prof_info;
  acc_api_info api_info;
  bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);

  gomp_mutex_lock (&acc_dev->lock);

  splay_tree_key n = lookup_host (acc_dev, h, 1);

  if (!n)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("%p is not a mapped block", (void *)h);
    }

  size_t host_size = n->host_end - n->host_start;

  if (n->host_start != (uintptr_t) h)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("[%p,%d] surrounds %p",
                  (void *) n->host_start, (int) host_size, (void *) h);
    }
  /* TODO This currently doesn't catch 'REFCOUNT_INFINITY' usage different from
     'acc_map_data'.  Maybe 'dynamic_refcount' can be used for disambiguating
     the different 'REFCOUNT_INFINITY' cases, or simply separate
     'REFCOUNT_INFINITY' values per different usage ('REFCOUNT_ACC_MAP_DATA'
     etc.)?  */
  else if (n->refcount != REFCOUNT_INFINITY)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("refusing to unmap block [%p,+%d] that has not been mapped"
                  " by 'acc_map_data'",
                  (void *) h, (int) host_size);
    }

  struct target_mem_desc *tgt = n->tgt;

  if (tgt->refcount == REFCOUNT_INFINITY)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("cannot unmap target block");
    }

  /* Above, we've verified that the mapping must have been set up by
     'acc_map_data'.  */
  assert (tgt->refcount == 1);

  /* Nullifying these fields prevents 'gomp_unmap_tgt' via 'gomp_remove_var'
     from freeing the target memory.  */
  tgt->tgt_end = 0;
  tgt->to_free = NULL;

  bool is_tgt_unmapped = gomp_remove_var (acc_dev, n);
  assert (is_tgt_unmapped);

  gomp_mutex_unlock (&acc_dev->lock);

  if (profiling_p)
    {
      thr->prof_info = NULL;
      thr->api_info = NULL;
    }
}


/* Helper function to map a single dynamic data item, represented by a single
   mapping.  The acc_dev->lock should be held on entry, and remains locked on
   exit.  */

static void *
goacc_map_var_existing (struct gomp_device_descr *acc_dev, void *hostaddr,
                        size_t size, splay_tree_key n)
{
  assert (n);

  /* Present. */
  void *d = (void *) (n->tgt->tgt_start + n->tgt_offset + hostaddr
            - n->host_start);

  if (hostaddr + size > (void *) n->host_end)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("[%p,+%d] not mapped", hostaddr, (int) size);
    }

  assert (n->refcount != REFCOUNT_LINK);
  if (n->refcount != REFCOUNT_INFINITY)
    n->refcount++;
  n->dynamic_refcount++;

  return d;
}

/* Enter dynamic mapping for a single datum.  Return the device pointer.  */

static void *
goacc_enter_datum (void **hostaddrs, size_t *sizes, void *kinds, int async)
{
  void *d;
  splay_tree_key n;

  if (!hostaddrs[0] || !sizes[0])
    gomp_fatal ("[%p,+%d] is a bad range", hostaddrs[0], (int) sizes[0]);

  goacc_lazy_initialize ();

  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return hostaddrs[0];

  acc_prof_info prof_info;
  acc_api_info api_info;
  bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);
  if (profiling_p)
    {
      prof_info.async = async;
      prof_info.async_queue = prof_info.async;
    }

  gomp_mutex_lock (&acc_dev->lock);

  n = lookup_host (acc_dev, hostaddrs[0], sizes[0]);
  if (n)
    {
      d = goacc_map_var_existing (acc_dev, hostaddrs[0], sizes[0], n);
      gomp_mutex_unlock (&acc_dev->lock);
    }
  else
    {
      const size_t mapnum = 1;

      gomp_mutex_unlock (&acc_dev->lock);

      goacc_aq aq = get_goacc_asyncqueue (async);

      struct target_mem_desc *tgt
        = goacc_map_vars (acc_dev, aq, mapnum, hostaddrs, NULL, sizes,
                          kinds, true, GOMP_MAP_VARS_ENTER_DATA);
      assert (tgt);
      assert (tgt->list_count == 1);
      n = tgt->list[0].key;
      assert (n);
      assert (n->refcount == 1);
      assert (n->dynamic_refcount == 0);
      n->dynamic_refcount++;

      d = (void *) tgt->tgt_start;
    }

  if (profiling_p)
    {
      thr->prof_info = NULL;
      thr->api_info = NULL;
    }

  return d;
}

void *
acc_create (void *h, size_t s)
{
  unsigned short kinds[1] = { GOMP_MAP_ALLOC };
  return goacc_enter_datum (&h, &s, &kinds, acc_async_sync);
}

void
acc_create_async (void *h, size_t s, int async)
{
  unsigned short kinds[1] = { GOMP_MAP_ALLOC };
  goacc_enter_datum (&h, &s, &kinds, async);
}

/* acc_present_or_create used to be what acc_create is now.  */
/* acc_pcreate is acc_present_or_create by a different name.  */
#ifdef HAVE_ATTRIBUTE_ALIAS
strong_alias (acc_create, acc_present_or_create)
strong_alias (acc_create, acc_pcreate)
#else
void *
acc_present_or_create (void *h, size_t s)
{
  return acc_create (h, s);
}

void *
acc_pcreate (void *h, size_t s)
{
  return acc_create (h, s);
}
#endif

void *
acc_copyin (void *h, size_t s)
{
  unsigned short kinds[1] = { GOMP_MAP_TO };
  return goacc_enter_datum (&h, &s, &kinds, acc_async_sync);
}

void
acc_copyin_async (void *h, size_t s, int async)
{
  unsigned short kinds[1] = { GOMP_MAP_TO };
  goacc_enter_datum (&h, &s, &kinds, async);
}

/* acc_present_or_copyin used to be what acc_copyin is now.  */
/* acc_pcopyin is acc_present_or_copyin by a different name.  */
#ifdef HAVE_ATTRIBUTE_ALIAS
strong_alias (acc_copyin, acc_present_or_copyin)
strong_alias (acc_copyin, acc_pcopyin)
#else
void *
acc_present_or_copyin (void *h, size_t s)
{
  return acc_copyin (h, s);
}

void *
acc_pcopyin (void *h, size_t s)
{
  return acc_copyin (h, s);
}
#endif


/* Helper function to unmap a single data item.  Device lock should be held on
   entry, and remains locked on exit.  */

static void
goacc_exit_datum_1 (struct gomp_device_descr *acc_dev, void *h, size_t s,
                    unsigned short kind, splay_tree_key n, goacc_aq aq)
{
  assert (kind != GOMP_MAP_DETACH
          && kind != GOMP_MAP_FORCE_DETACH);

  if ((uintptr_t) h < n->host_start || (uintptr_t) h + s > n->host_end)
    {
      size_t host_size = n->host_end - n->host_start;
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("[%p,+%d] outside mapped block [%p,+%d]",
                  (void *) h, (int) s, (void *) n->host_start, (int) host_size);
    }

  bool finalize = (kind == GOMP_MAP_FORCE_FROM
                   || kind == GOMP_MAP_DELETE);

  assert (n->refcount != REFCOUNT_LINK);
  if (n->refcount != REFCOUNT_INFINITY
      && n->refcount < n->dynamic_refcount)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("Dynamic reference counting assert fail\n");
    }

  if (finalize)
    {
      if (n->refcount != REFCOUNT_INFINITY)
        n->refcount -= n->dynamic_refcount;
      n->dynamic_refcount = 0;
    }
  else if (n->dynamic_refcount)
    {
      if (n->refcount != REFCOUNT_INFINITY)
        n->refcount--;
      n->dynamic_refcount--;
    }

  if (n->refcount == 0)
    {
      bool copyout = (kind == GOMP_MAP_FROM
                      || kind == GOMP_MAP_FORCE_FROM);
      if (copyout)
        {
          void *d = (void *) (n->tgt->tgt_start + n->tgt_offset
                              + (uintptr_t) h - n->host_start);
          gomp_copy_dev2host (acc_dev, aq, h, d, s);
        }

      if (aq)
        /* TODO We can't do the 'is_tgt_unmapped' checking -- see the
           'gomp_unref_tgt' comment in
           <http://mid.mail-archive.com/878snl36eu.fsf@euler.schwinge.homeip.net>;
           PR92881.  */
        gomp_remove_var_async (acc_dev, n, aq);
      else
        {
          size_t num_mappings = 0;
          /* If the target_mem_desc represents a single data mapping, we can
             check that it is freed when this splay tree key's refcount reaches
             zero.  Otherwise (e.g. for a 'GOMP_MAP_STRUCT' mapping with
             multiple members), fall back to skipping the test.  */
          for (size_t l_i = 0; l_i < n->tgt->list_count; ++l_i)
            if (n->tgt->list[l_i].key
                && !n->tgt->list[l_i].is_attach)
              ++num_mappings;
          bool is_tgt_unmapped = gomp_remove_var (acc_dev, n);
          assert (is_tgt_unmapped || num_mappings > 1);
        }
    }
}


/* Exit a dynamic mapping for a single variable.  */

static void
goacc_exit_datum (void *h, size_t s, unsigned short kind, int async)
{
  /* No need to call lazy open, as the data must already have been
     mapped.  */

  kind &= 0xff;

  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return;

  acc_prof_info prof_info;
  acc_api_info api_info;
  bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);
  if (profiling_p)
    {
      prof_info.async = async;
      prof_info.async_queue = prof_info.async;
    }

  gomp_mutex_lock (&acc_dev->lock);

  splay_tree_key n = lookup_host (acc_dev, h, s);
  /* Non-present data is a no-op: PR92726, RP92970, PR92984.  */
  if (n)
    {
      goacc_aq aq = get_goacc_asyncqueue (async);
      goacc_exit_datum_1 (acc_dev, h, s, kind, n, aq);
    }

  gomp_mutex_unlock (&acc_dev->lock);

  if (profiling_p)
    {
      thr->prof_info = NULL;
      thr->api_info = NULL;
    }
}

void
acc_delete (void *h , size_t s)
{
  goacc_exit_datum (h, s, GOMP_MAP_RELEASE, acc_async_sync);
}

void
acc_delete_async (void *h , size_t s, int async)
{
  goacc_exit_datum (h, s, GOMP_MAP_RELEASE, async);
}

void
acc_delete_finalize (void *h , size_t s)
{
  goacc_exit_datum (h, s, GOMP_MAP_DELETE, acc_async_sync);
}

void
acc_delete_finalize_async (void *h , size_t s, int async)
{
  goacc_exit_datum (h, s, GOMP_MAP_DELETE, async);
}

void
acc_copyout (void *h, size_t s)
{
  goacc_exit_datum (h, s, GOMP_MAP_FROM, acc_async_sync);
}

void
acc_copyout_async (void *h, size_t s, int async)
{
  goacc_exit_datum (h, s, GOMP_MAP_FROM, async);
}

void
acc_copyout_finalize (void *h, size_t s)
{
  goacc_exit_datum (h, s, GOMP_MAP_FORCE_FROM, acc_async_sync);
}

void
acc_copyout_finalize_async (void *h, size_t s, int async)
{
  goacc_exit_datum (h, s, GOMP_MAP_FORCE_FROM, async);
}

static void
update_dev_host (int is_dev, void *h, size_t s, int async)
{
  splay_tree_key n;
  void *d;

  goacc_lazy_initialize ();

  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return;

  /* Fortran optional arguments that are non-present result in a
     NULL host address here.  This can safely be ignored as it is
     not possible to 'update' a non-present optional argument.  */
  if (h == NULL)
    return;

  acc_prof_info prof_info;
  acc_api_info api_info;
  bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);
  if (profiling_p)
    {
      prof_info.async = async;
      prof_info.async_queue = prof_info.async;
    }

  gomp_mutex_lock (&acc_dev->lock);

  n = lookup_host (acc_dev, h, s);

  if (!n)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("[%p,%d] is not mapped", h, (int)s);
    }

  d = (void *) (n->tgt->tgt_start + n->tgt_offset
                + (uintptr_t) h - n->host_start);

  goacc_aq aq = get_goacc_asyncqueue (async);

  if (is_dev)
    gomp_copy_host2dev (acc_dev, aq, d, h, s, false, /* TODO: cbuf? */ NULL);
  else
    gomp_copy_dev2host (acc_dev, aq, h, d, s);

  gomp_mutex_unlock (&acc_dev->lock);

  if (profiling_p)
    {
      thr->prof_info = NULL;
      thr->api_info = NULL;
    }
}

void
acc_update_device (void *h, size_t s)
{
  update_dev_host (1, h, s, acc_async_sync);
}

void
acc_update_device_async (void *h, size_t s, int async)
{
  update_dev_host (1, h, s, async);
}

void
acc_update_self (void *h, size_t s)
{
  update_dev_host (0, h, s, acc_async_sync);
}

void
acc_update_self_async (void *h, size_t s, int async)
{
  update_dev_host (0, h, s, async);
}

void
acc_attach_async (void **hostaddr, int async)
{
  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;
  goacc_aq aq = get_goacc_asyncqueue (async);

  struct splay_tree_key_s cur_node;
  splay_tree_key n;

  if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return;

  gomp_mutex_lock (&acc_dev->lock);

  cur_node.host_start = (uintptr_t) hostaddr;
  cur_node.host_end = cur_node.host_start + sizeof (void *);
  n = splay_tree_lookup (&acc_dev->mem_map, &cur_node);

  if (n == NULL)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("struct not mapped for acc_attach");
    }

  gomp_attach_pointer (acc_dev, aq, &acc_dev->mem_map, n, (uintptr_t) hostaddr,
                       0, NULL, false);

  gomp_mutex_unlock (&acc_dev->lock);
}

void
acc_attach (void **hostaddr)
{
  acc_attach_async (hostaddr, acc_async_sync);
}

static void
goacc_detach_internal (void **hostaddr, int async, bool finalize)
{
  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;
  struct splay_tree_key_s cur_node;
  splay_tree_key n;
  struct goacc_asyncqueue *aq = get_goacc_asyncqueue (async);

  if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
    return;

  gomp_mutex_lock (&acc_dev->lock);

  cur_node.host_start = (uintptr_t) hostaddr;
  cur_node.host_end = cur_node.host_start + sizeof (void *);
  n = splay_tree_lookup (&acc_dev->mem_map, &cur_node);

  if (n == NULL)
    {
      gomp_mutex_unlock (&acc_dev->lock);
      gomp_fatal ("struct not mapped for acc_detach");
    }

  gomp_detach_pointer (acc_dev, aq, n, (uintptr_t) hostaddr, finalize, NULL);

  gomp_mutex_unlock (&acc_dev->lock);
}

void
acc_detach (void **hostaddr)
{
  goacc_detach_internal (hostaddr, acc_async_sync, false);
}

void
acc_detach_async (void **hostaddr, int async)
{
  goacc_detach_internal (hostaddr, async, false);
}

void
acc_detach_finalize (void **hostaddr)
{
  goacc_detach_internal (hostaddr, acc_async_sync, true);
}

void
acc_detach_finalize_async (void **hostaddr, int async)
{
  goacc_detach_internal (hostaddr, async, true);
}

/* Some types of (pointer) variables use several consecutive mappings, which
   must be treated as a group for enter/exit data directives.  This function
   returns the last mapping in such a group (inclusive), or POS for singleton
   mappings.  */

static int
find_group_last (int pos, size_t mapnum, size_t *sizes, unsigned short *kinds)
{
  unsigned char kind0 = kinds[pos] & 0xff;
  int first_pos = pos;

  switch (kind0)
    {
    case GOMP_MAP_TO_PSET:
      if (pos + 1 < mapnum
          && (kinds[pos + 1] & 0xff) == GOMP_MAP_ATTACH)
        return pos + 1;

      while (pos + 1 < mapnum
             && (kinds[pos + 1] & 0xff) == GOMP_MAP_POINTER)
        pos++;
      /* We expect at least one GOMP_MAP_POINTER (if not a single
         GOMP_MAP_ATTACH) after a GOMP_MAP_TO_PSET.  */
      assert (pos > first_pos);
      break;

    case GOMP_MAP_STRUCT:
      pos += sizes[pos];
      break;

    case GOMP_MAP_POINTER:
    case GOMP_MAP_ALWAYS_POINTER:
      /* These mappings are only expected after some other mapping.  If we
         see one by itself, something has gone wrong.  */
      gomp_fatal ("unexpected mapping");
      break;

    case GOMP_MAP_ATTACH:
      break;

    default:
      /* GOMP_MAP_ALWAYS_POINTER can only appear directly after some other
         mapping.  */
      if (pos + 1 < mapnum)
        {
          unsigned char kind1 = kinds[pos + 1] & 0xff;
          if (kind1 == GOMP_MAP_ALWAYS_POINTER)
            return pos + 1;
        }

      /* We can have a single GOMP_MAP_ATTACH mapping after a to/from
         mapping.  */
      if (pos + 1 < mapnum
          && (kinds[pos + 1] & 0xff) == GOMP_MAP_ATTACH)
        return pos + 1;

      /* We can have zero or more GOMP_MAP_POINTER mappings after a to/from
         (etc.) mapping.  */
      while (pos + 1 < mapnum
             && (kinds[pos + 1] & 0xff) == GOMP_MAP_POINTER)
        pos++;
    }

  return pos;
}

/* Map variables for OpenACC "enter data".  We can't just call
   goacc_map_vars once, because individual mapped variables might have
   "exit data" called for them at different times.  */

static void
goacc_enter_data_internal (struct gomp_device_descr *acc_dev, size_t mapnum,
                           void **hostaddrs, size_t *sizes,
                           unsigned short *kinds, goacc_aq aq)
{
  gomp_mutex_lock (&acc_dev->lock);

  for (size_t i = 0; i < mapnum; i++)
    {
      splay_tree_key n;
      size_t group_last = find_group_last (i, mapnum, sizes, kinds);
      bool struct_p = false;
      size_t size, groupnum = (group_last - i) + 1;

      switch (kinds[i] & 0xff)
        {
        case GOMP_MAP_STRUCT:
          {
            size = (uintptr_t) hostaddrs[group_last] + sizes[group_last]
                   - (uintptr_t) hostaddrs[i];
            struct_p = true;
          }
          break;

        case GOMP_MAP_ATTACH:
          size = sizeof (void *);
          break;

        default:
          size = sizes[i];
        }

      n = lookup_host (acc_dev, hostaddrs[i], size);

      if (n && struct_p)
        {
          for (size_t j = i + 1; j <= group_last; j++)
            {
              struct splay_tree_key_s cur_node;
              cur_node.host_start = (uintptr_t) hostaddrs[j];
              cur_node.host_end = cur_node.host_start + sizes[j];
              splay_tree_key n2
                = splay_tree_lookup (&acc_dev->mem_map, &cur_node);
              if (!n2
                  || n2->tgt != n->tgt
                  || n2->host_start - n->host_start
                     != n2->tgt_offset - n->tgt_offset)
                {
                  gomp_mutex_unlock (&acc_dev->lock);
                  gomp_fatal ("Trying to map into device [%p..%p) structure "
                              "element when other mapped elements from the "
                              "same structure weren't mapped together with "
                              "it", (void *) cur_node.host_start,
                              (void *) cur_node.host_end);
                }
            }
          /* This is a special case because we must increment the refcount by
             the number of mapped struct elements, rather than by one.  */
          if (n->refcount != REFCOUNT_INFINITY)
            n->refcount += groupnum - 1;
          n->dynamic_refcount += groupnum - 1;
        }
      else if (n && groupnum == 1)
        {
          void *h = hostaddrs[i];
          size_t s = sizes[i];

          if ((kinds[i] & 0xff) == GOMP_MAP_ATTACH)
            {
              gomp_attach_pointer (acc_dev, aq, &acc_dev->mem_map, n,
                                   (uintptr_t) h, s, NULL, false);
              /* OpenACC 'attach'/'detach' doesn't affect structured/dynamic
                 reference counts ('n->refcount', 'n->dynamic_refcount').  */
            }
          else
            goacc_map_var_existing (acc_dev, h, s, n);
        }
      else if (n && groupnum > 1)
        {
          assert (n->refcount != REFCOUNT_LINK);

          for (size_t j = i + 1; j <= group_last; j++)
            if ((kinds[j] & 0xff) == GOMP_MAP_ATTACH)
              {
                splay_tree_key m
                  = lookup_host (acc_dev, hostaddrs[j], sizeof (void *));
                gomp_attach_pointer (acc_dev, aq, &acc_dev->mem_map, m,
                                     (uintptr_t) hostaddrs[j], sizes[j], NULL,
                                     false);
              }

          bool processed = false;

          struct target_mem_desc *tgt = n->tgt;

          /* Minimal OpenACC variant corresponding to PR96668
             "[OpenMP] Re-mapping allocated but previously unallocated
             allocatable does not work" 'libgomp/target.c' changes, so that
             OpenACC 'declare' code à la PR106643
             "[gfortran + OpenACC] Allocate in module causes refcount error"
             has a chance to work.  */
          if ((kinds[i] & 0xff) == GOMP_MAP_TO_PSET
              && tgt->list_count == 0)
            {
              /* 'declare target'.  */
              assert (n->refcount == REFCOUNT_INFINITY);

              for (size_t k = 1; k < groupnum; k++)
                {
                  /* The only thing we expect to see here.  */
                  assert ((kinds[i + k] & 0xff) == GOMP_MAP_POINTER);
                }

              /* Let 'goacc_map_vars' -> 'gomp_map_vars_internal' handle
                 this.  */
              gomp_mutex_unlock (&acc_dev->lock);
              struct target_mem_desc *tgt_
                = goacc_map_vars (acc_dev, aq, groupnum, &hostaddrs[i], NULL,
                                  &sizes[i], &kinds[i], true,
                                  GOMP_MAP_VARS_ENTER_DATA);
              assert (tgt_ == NULL);
              gomp_mutex_lock (&acc_dev->lock);

              /* Given that 'goacc_exit_data_internal'/'goacc_exit_datum_1'
                 will always see 'n->refcount == REFCOUNT_INFINITY',
                 there's no need to adjust 'n->dynamic_refcount' here.  */

              processed = true;
            }
          else
            assert (n->refcount != REFCOUNT_INFINITY);

          for (size_t j = 0; j < tgt->list_count; j++)
            if (tgt->list[j].key == n)
              {
                /* We are processing a group of mappings (e.g.
                   [GOMP_MAP_TO, GOMP_MAP_TO_PSET, GOMP_MAP_POINTER]).
                   Find the right group in the target_mem_desc's variable
                   list, and increment the refcounts for each item in that
                   group.  */
                for (size_t k = 0; k < groupnum; k++)
                  if (j + k < tgt->list_count
                      && tgt->list[j + k].key
                      && !tgt->list[j + k].is_attach)
                    {
                      tgt->list[j + k].key->refcount++;
                      tgt->list[j + k].key->dynamic_refcount++;
                    }
                processed = true;
                break;
              }

          if (!processed)
            {
              gomp_mutex_unlock (&acc_dev->lock);
              gomp_fatal ("dynamic refcount incrementing failed for "
                          "pointer/pset");
            }
        }
      else if (hostaddrs[i])
        {
          /* The data is not mapped already.  Map it now, unless the first
             member in the group has a NULL pointer (e.g. a non-present
             optional parameter).  */
          gomp_mutex_unlock (&acc_dev->lock);

          struct target_mem_desc *tgt
            = goacc_map_vars (acc_dev, aq, groupnum, &hostaddrs[i], NULL,
                              &sizes[i], &kinds[i], true,
                              GOMP_MAP_VARS_ENTER_DATA);
          assert (tgt);

          gomp_mutex_lock (&acc_dev->lock);

          for (size_t j = 0; j < tgt->list_count; j++)
            {
              n = tgt->list[j].key;
              if (n && !tgt->list[j].is_attach)
                n->dynamic_refcount++;
            }
        }

      i = group_last;
    }

  gomp_mutex_unlock (&acc_dev->lock);
}

/* Unmap variables for OpenACC "exit data".  */

static void
goacc_exit_data_internal (struct gomp_device_descr *acc_dev, size_t mapnum,
                          void **hostaddrs, size_t *sizes,
                          unsigned short *kinds, goacc_aq aq)
{
  gomp_mutex_lock (&acc_dev->lock);

  /* Handle "detach" before copyback/deletion of mapped data.  */
  for (size_t i = 0; i < mapnum; ++i)
    {
      unsigned char kind = kinds[i] & 0xff;
      bool finalize = false;
      switch (kind)
        {
        case GOMP_MAP_FORCE_DETACH:
          finalize = true;
          /* Fallthrough.  */

        case GOMP_MAP_DETACH:
          {
            struct splay_tree_key_s cur_node;
            uintptr_t hostaddr = (uintptr_t) hostaddrs[i];
            cur_node.host_start = hostaddr;
            cur_node.host_end = cur_node.host_start + sizeof (void *);
            splay_tree_key n
              = splay_tree_lookup (&acc_dev->mem_map, &cur_node);

            if (n == NULL)
              {
                gomp_mutex_unlock (&acc_dev->lock);
                gomp_fatal ("struct not mapped for detach operation");
              }

            gomp_detach_pointer (acc_dev, aq, n, hostaddr, finalize, NULL);
          }
          break;
        default:
          ;
        }
    }

  for (size_t i = 0; i < mapnum; ++i)
    {
      unsigned char kind = kinds[i] & 0xff;

      switch (kind)
        {
        case GOMP_MAP_FROM:
        case GOMP_MAP_FORCE_FROM:
        case GOMP_MAP_TO_PSET:
        case GOMP_MAP_POINTER:
        case GOMP_MAP_DELETE:
        case GOMP_MAP_RELEASE:
          {
            struct splay_tree_key_s cur_node;
            size_t size;
            if (kind == GOMP_MAP_POINTER)
              size = sizeof (void *);
            else
              size = sizes[i];
            cur_node.host_start = (uintptr_t) hostaddrs[i];
            cur_node.host_end = cur_node.host_start + size;
            splay_tree_key n
              = splay_tree_lookup (&acc_dev->mem_map, &cur_node);

            if (n == NULL)
              continue;

            goacc_exit_datum_1 (acc_dev, hostaddrs[i], size, kind, n, aq);
          }
          break;

        case GOMP_MAP_STRUCT:
          /* Skip the 'GOMP_MAP_STRUCT' itself, and use the regular processing
             for all its entries.  This special handling exists for GCC 10.1
             compatibility; afterwards, we're not generating these no-op
             'GOMP_MAP_STRUCT's anymore.  */
          break;

        case GOMP_MAP_DETACH:
        case GOMP_MAP_FORCE_DETACH:
          /* OpenACC 'attach'/'detach' doesn't affect structured/dynamic
             reference counts ('n->refcount', 'n->dynamic_refcount').  */
          break;

        default:
          gomp_fatal (">>>> goacc_exit_data_internal UNHANDLED kind 0x%.2x",
                          kind);
        }
    }

  gomp_mutex_unlock (&acc_dev->lock);
}

static void
goacc_enter_exit_data_internal (int flags_m, size_t mapnum, void **hostaddrs,
                                size_t *sizes, unsigned short *kinds,
                                bool data_enter, int async, int num_waits,
                                va_list *ap)
{
  int flags = GOACC_FLAGS_UNMARSHAL (flags_m);

  struct goacc_thread *thr;
  struct gomp_device_descr *acc_dev;

  goacc_lazy_initialize ();

  thr = goacc_thread ();
  acc_dev = thr->dev;

  bool profiling_p = GOACC_PROFILING_DISPATCH_P (true);

  acc_prof_info prof_info;
  if (profiling_p)
    {
      thr->prof_info = &prof_info;

      prof_info.event_type
        = data_enter ? acc_ev_enter_data_start : acc_ev_exit_data_start;
      prof_info.valid_bytes = _ACC_PROF_INFO_VALID_BYTES;
      prof_info.version = _ACC_PROF_INFO_VERSION;
      prof_info.device_type = acc_device_type (acc_dev->type);
      prof_info.device_number = acc_dev->target_id;
      prof_info.thread_id = -1;
      prof_info.async = async;
      prof_info.async_queue = prof_info.async;
      prof_info.src_file = NULL;
      prof_info.func_name = NULL;
      prof_info.line_no = -1;
      prof_info.end_line_no = -1;
      prof_info.func_line_no = -1;
      prof_info.func_end_line_no = -1;
    }
  acc_event_info enter_exit_data_event_info;
  if (profiling_p)
    {
      enter_exit_data_event_info.other_event.event_type
        = prof_info.event_type;
      enter_exit_data_event_info.other_event.valid_bytes
        = _ACC_OTHER_EVENT_INFO_VALID_BYTES;
      enter_exit_data_event_info.other_event.parent_construct
        = data_enter ? acc_construct_enter_data : acc_construct_exit_data;
      enter_exit_data_event_info.other_event.implicit = 0;
      enter_exit_data_event_info.other_event.tool_info = NULL;
    }
  acc_api_info api_info;
  if (profiling_p)
    {
      thr->api_info = &api_info;

      api_info.device_api = acc_device_api_none;
      api_info.valid_bytes = _ACC_API_INFO_VALID_BYTES;
      api_info.device_type = prof_info.device_type;
      api_info.vendor = -1;
      api_info.device_handle = NULL;
      api_info.context_handle = NULL;
      api_info.async_handle = NULL;
    }

  if (profiling_p)
    goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info,
                              &api_info);

  if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
      || (flags & GOACC_FLAG_HOST_FALLBACK))
    {
      prof_info.device_type = acc_device_host;
      api_info.device_type = prof_info.device_type;

      goto out_prof;
    }

  if (num_waits)
    goacc_wait (async, num_waits, ap);

  goacc_aq aq = get_goacc_asyncqueue (async);

  if (data_enter)
    goacc_enter_data_internal (acc_dev, mapnum, hostaddrs, sizes, kinds, aq);
  else
    goacc_exit_data_internal (acc_dev, mapnum, hostaddrs, sizes, kinds, aq);

 out_prof:
  if (profiling_p)
    {
      prof_info.event_type
        = data_enter ? acc_ev_enter_data_end : acc_ev_exit_data_end;
      enter_exit_data_event_info.other_event.event_type = prof_info.event_type;
      goacc_profiling_dispatch (&prof_info, &enter_exit_data_event_info,
                                &api_info);

      thr->prof_info = NULL;
      thr->api_info = NULL;
    }
}

/* Legacy entry point (GCC 11 and earlier).  */

void
GOACC_enter_exit_data (int flags_m, size_t mapnum, void **hostaddrs,
                       size_t *sizes, unsigned short *kinds, int async,
                       int num_waits, ...)
{
  /* Determine if this is an OpenACC "enter data".  */
  bool data_enter = false;
  for (size_t i = 0; i < mapnum; ++i)
    {
      unsigned char kind = kinds[i] & 0xff;

      if (kind == GOMP_MAP_POINTER
          || kind == GOMP_MAP_TO_PSET
          || kind == GOMP_MAP_STRUCT)
        continue;

      if (kind == GOMP_MAP_FORCE_ALLOC
          || kind == GOMP_MAP_FORCE_PRESENT
          || kind == GOMP_MAP_ATTACH
          || kind == GOMP_MAP_FORCE_TO
          || kind == GOMP_MAP_TO
          || kind == GOMP_MAP_ALLOC)
        {
          data_enter = true;
          break;
        }

      if (kind == GOMP_MAP_RELEASE
          || kind == GOMP_MAP_DELETE
          || kind == GOMP_MAP_DETACH
          || kind == GOMP_MAP_FORCE_DETACH
          || kind == GOMP_MAP_FROM
          || kind == GOMP_MAP_FORCE_FROM)
        break;

      gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
                      kind);
    }

  va_list ap;
  va_start (ap, num_waits);
  goacc_enter_exit_data_internal (flags_m, mapnum, hostaddrs, sizes, kinds,
                                  data_enter, async, num_waits, &ap);
  va_end (ap);
}

void
GOACC_enter_data (int flags_m, size_t mapnum, void **hostaddrs,
                  size_t *sizes, unsigned short *kinds, int async,
                  int num_waits, ...)
{
  va_list ap;
  va_start (ap, num_waits);
  goacc_enter_exit_data_internal (flags_m, mapnum, hostaddrs, sizes, kinds,
                                  true, async, num_waits, &ap);
  va_end (ap);
}

void
GOACC_exit_data (int flags_m, size_t mapnum, void **hostaddrs,
                 size_t *sizes, unsigned short *kinds, int async,
                 int num_waits, ...)
{
  va_list ap;
  va_start (ap, num_waits);
  goacc_enter_exit_data_internal (flags_m, mapnum, hostaddrs, sizes, kinds,
                                  false, async, num_waits, &ap);
  va_end (ap);
}

void
GOACC_declare (int flags_m, size_t mapnum,
               void **hostaddrs, size_t *sizes, unsigned short *kinds)
{
  for (size_t i = 0; i < mapnum; i++)
    {
      unsigned char kind = kinds[i] & 0xff;

      if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET)
        continue;

      switch (kind)
        {
        case GOMP_MAP_ALLOC:
          if (acc_is_present (hostaddrs[i], sizes[i]))
            continue;
          /* FALLTHRU */
        case GOMP_MAP_FORCE_ALLOC:
        case GOMP_MAP_TO:
        case GOMP_MAP_FORCE_TO:
          goacc_enter_exit_data_internal (flags_m, 1, &hostaddrs[i], &sizes[i],
                                          &kinds[i], true, GOMP_ASYNC_SYNC, 0, NULL);
          break;

        case GOMP_MAP_FROM:
        case GOMP_MAP_FORCE_FROM:
        case GOMP_MAP_RELEASE:
        case GOMP_MAP_DELETE:
          goacc_enter_exit_data_internal (flags_m, 1, &hostaddrs[i], &sizes[i],
                                          &kinds[i], false, GOMP_ASYNC_SYNC, 0, NULL);
          break;

        case GOMP_MAP_FORCE_DEVICEPTR:
          break;

        case GOMP_MAP_FORCE_PRESENT:
          if (!acc_is_present (hostaddrs[i], sizes[i]))
            gomp_fatal ("[%p,%ld] is not mapped", hostaddrs[i],
                        (unsigned long) sizes[i]);
          break;

        default:
          assert (0);
          break;
        }
    }
}