1 /* OpenACC Runtime initialization routines
3 Copyright (C) 2013-2024 Free Software Foundation, Inc.
5 Contributed by Mentor Embedded.
7 This file is part of the GNU Offloading and Multi Processing Library
10 Libgomp is free software; you can redistribute it and/or modify it
11 under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 3, or (at your option)
15 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
20 Under Section 7 of GPL version 3, you are granted additional
21 permissions described in the GCC Runtime Library Exception, version
22 3.1, as published by the Free Software Foundation.
24 You should have received a copy of the GNU General Public License and
25 a copy of the GCC Runtime Library Exception along with this program;
26 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
27 <http://www.gnu.org/licenses/>. */
31 #include "gomp-constants.h"
36 /* Return block containing [H->S), or NULL if not contained. The device lock
37 for DEV must be locked on entry, and remains locked on exit. */
40 lookup_host (struct gomp_device_descr
*dev
, void *h
, size_t s
)
42 struct splay_tree_key_s node
;
45 node
.host_start
= (uintptr_t) h
;
46 node
.host_end
= (uintptr_t) h
+ s
;
48 key
= splay_tree_lookup (&dev
->mem_map
, &node
);
53 /* Helper for lookup_dev. Iterate over splay tree. */
56 lookup_dev_1 (splay_tree_node node
, uintptr_t d
, size_t s
)
58 splay_tree_key key
= &node
->key
;
59 if (d
>= key
->tgt
->tgt_start
&& d
+ s
<= key
->tgt
->tgt_end
)
64 key
= lookup_dev_1 (node
->left
, d
, s
);
65 if (!key
&& node
->right
)
66 key
= lookup_dev_1 (node
->right
, d
, s
);
71 /* Return block containing [D->S), or NULL if not contained.
73 This iterates over the splay tree. This is not expected to be a common
76 The device lock associated with MEM_MAP must be locked on entry, and remains
80 lookup_dev (splay_tree mem_map
, void *d
, size_t s
)
82 if (!mem_map
|| !mem_map
->root
)
85 return lookup_dev_1 (mem_map
->root
, (uintptr_t) d
, s
);
89 /* OpenACC is silent on how memory exhaustion is indicated. We return
98 goacc_lazy_initialize ();
100 struct goacc_thread
*thr
= goacc_thread ();
104 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
107 acc_prof_info prof_info
;
108 acc_api_info api_info
;
109 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
111 void *res
= thr
->dev
->alloc_func (thr
->dev
->target_id
, s
);
115 thr
->prof_info
= NULL
;
116 thr
->api_info
= NULL
;
130 struct goacc_thread
*thr
= goacc_thread ();
132 assert (thr
&& thr
->dev
);
134 struct gomp_device_descr
*acc_dev
= thr
->dev
;
136 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
139 acc_prof_info prof_info
;
140 acc_api_info api_info
;
141 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
143 gomp_mutex_lock (&acc_dev
->lock
);
145 /* We don't have to call lazy open here, as the ptr value must have
146 been returned by acc_malloc. It's not permitted to pass NULL in
147 (unless you got that null from acc_malloc). */
148 if ((k
= lookup_dev (&acc_dev
->mem_map
, d
, 1)))
150 void *offset
= d
- k
->tgt
->tgt_start
+ k
->tgt_offset
;
151 void *h
= k
->host_start
+ offset
;
152 size_t h_size
= k
->host_end
- k
->host_start
;
153 gomp_mutex_unlock (&acc_dev
->lock
);
154 /* PR92503 "[OpenACC] Behavior of 'acc_free' if the memory space is still
155 used in a mapping". */
156 gomp_fatal ("refusing to free device memory space at %p that is still"
157 " mapped at [%p,+%d]",
161 gomp_mutex_unlock (&acc_dev
->lock
);
163 if (!acc_dev
->free_func (acc_dev
->target_id
, d
))
164 gomp_fatal ("error in freeing device memory in %s", __FUNCTION__
);
168 thr
->prof_info
= NULL
;
169 thr
->api_info
= NULL
;
174 memcpy_tofrom_device (bool from
, void *d
, void *h
, size_t s
, int async
,
175 const char *libfnname
)
177 /* No need to call lazy open here, as the device pointer must have
178 been obtained from a routine that did that. */
179 struct goacc_thread
*thr
= goacc_thread ();
181 assert (thr
&& thr
->dev
);
183 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
192 acc_prof_info prof_info
;
193 acc_api_info api_info
;
194 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
197 prof_info
.async
= async
;
198 prof_info
.async_queue
= prof_info
.async
;
201 goacc_aq aq
= get_goacc_asyncqueue (async
);
203 gomp_copy_dev2host (thr
->dev
, aq
, h
, d
, s
);
205 gomp_copy_host2dev (thr
->dev
, aq
, d
, h
, s
, false, /* TODO: cbuf? */ NULL
);
209 thr
->prof_info
= NULL
;
210 thr
->api_info
= NULL
;
215 acc_memcpy_to_device (void *d
, void *h
, size_t s
)
217 memcpy_tofrom_device (false, d
, h
, s
, acc_async_sync
, __FUNCTION__
);
221 acc_memcpy_to_device_async (void *d
, void *h
, size_t s
, int async
)
223 memcpy_tofrom_device (false, d
, h
, s
, async
, __FUNCTION__
);
227 acc_memcpy_from_device (void *h
, void *d
, size_t s
)
229 memcpy_tofrom_device (true, d
, h
, s
, acc_async_sync
, __FUNCTION__
);
233 acc_memcpy_from_device_async (void *h
, void *d
, size_t s
, int async
)
235 memcpy_tofrom_device (true, d
, h
, s
, async
, __FUNCTION__
);
238 /* Return the device pointer that corresponds to host data H. Or NULL
242 acc_deviceptr (void *h
)
248 goacc_lazy_initialize ();
250 struct goacc_thread
*thr
= goacc_thread ();
251 struct gomp_device_descr
*dev
= thr
->dev
;
253 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
256 /* In the following, no OpenACC Profiling Interface events can possibly be
259 gomp_mutex_lock (&dev
->lock
);
261 n
= lookup_host (dev
, h
, 1);
265 gomp_mutex_unlock (&dev
->lock
);
269 offset
= h
- n
->host_start
;
271 d
= n
->tgt
->tgt_start
+ n
->tgt_offset
+ offset
;
273 gomp_mutex_unlock (&dev
->lock
);
278 /* Return the host pointer that corresponds to device data D. Or NULL
282 acc_hostptr (void *d
)
288 goacc_lazy_initialize ();
290 struct goacc_thread
*thr
= goacc_thread ();
291 struct gomp_device_descr
*acc_dev
= thr
->dev
;
293 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
296 /* In the following, no OpenACC Profiling Interface events can possibly be
299 gomp_mutex_lock (&acc_dev
->lock
);
301 n
= lookup_dev (&acc_dev
->mem_map
, d
, 1);
305 gomp_mutex_unlock (&acc_dev
->lock
);
309 offset
= d
- n
->tgt
->tgt_start
+ n
->tgt_offset
;
311 h
= n
->host_start
+ offset
;
313 gomp_mutex_unlock (&acc_dev
->lock
);
318 /* Return 1 if host data [H,+S] is present on the device. */
321 acc_is_present (void *h
, size_t s
)
328 goacc_lazy_initialize ();
330 struct goacc_thread
*thr
= goacc_thread ();
331 struct gomp_device_descr
*acc_dev
= thr
->dev
;
333 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
336 /* In the following, no OpenACC Profiling Interface events can possibly be
339 gomp_mutex_lock (&acc_dev
->lock
);
341 n
= lookup_host (acc_dev
, h
, s
);
343 if (n
&& ((uintptr_t)h
< n
->host_start
344 || (uintptr_t)h
+ s
> n
->host_end
345 || s
> n
->host_end
- n
->host_start
))
348 gomp_mutex_unlock (&acc_dev
->lock
);
353 /* Create a mapping for host [H,+S] -> device [D,+S] */
356 acc_map_data (void *h
, void *d
, size_t s
)
362 unsigned short kinds
= GOMP_MAP_ALLOC
;
364 goacc_lazy_initialize ();
366 struct goacc_thread
*thr
= goacc_thread ();
367 struct gomp_device_descr
*acc_dev
= thr
->dev
;
369 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
372 gomp_fatal ("cannot map data on shared-memory system");
376 struct goacc_thread
*thr
= goacc_thread ();
379 gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
380 (void *)h
, (int)s
, (void *)d
, (int)s
);
382 acc_prof_info prof_info
;
383 acc_api_info api_info
;
384 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
386 gomp_mutex_lock (&acc_dev
->lock
);
388 if (lookup_host (acc_dev
, h
, s
))
390 gomp_mutex_unlock (&acc_dev
->lock
);
391 gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h
,
395 if (lookup_dev (&thr
->dev
->mem_map
, d
, s
))
397 gomp_mutex_unlock (&acc_dev
->lock
);
398 gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d
,
402 gomp_mutex_unlock (&acc_dev
->lock
);
404 struct target_mem_desc
*tgt
405 = goacc_map_vars (acc_dev
, NULL
, mapnum
, &hostaddrs
, &devaddrs
, &sizes
,
406 &kinds
, true, GOMP_MAP_VARS_ENTER_DATA
);
408 assert (tgt
->list_count
== 1);
409 splay_tree_key n
= tgt
->list
[0].key
;
411 assert (n
->refcount
== 1);
412 assert (n
->dynamic_refcount
== 0);
413 /* Special reference counting behavior. */
414 n
->refcount
= REFCOUNT_ACC_MAP_DATA
;
415 n
->dynamic_refcount
= 1;
419 thr
->prof_info
= NULL
;
420 thr
->api_info
= NULL
;
426 acc_unmap_data (void *h
)
428 struct goacc_thread
*thr
= goacc_thread ();
429 struct gomp_device_descr
*acc_dev
= thr
->dev
;
431 /* No need to call lazy open, as the address must have been mapped. */
433 /* This is a no-op on shared-memory targets. */
434 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
437 acc_prof_info prof_info
;
438 acc_api_info api_info
;
439 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
441 gomp_mutex_lock (&acc_dev
->lock
);
443 splay_tree_key n
= lookup_host (acc_dev
, h
, 1);
447 gomp_mutex_unlock (&acc_dev
->lock
);
448 gomp_fatal ("%p is not a mapped block", (void *)h
);
451 size_t host_size
= n
->host_end
- n
->host_start
;
453 if (n
->host_start
!= (uintptr_t) h
)
455 gomp_mutex_unlock (&acc_dev
->lock
);
456 gomp_fatal ("[%p,%d] surrounds %p",
457 (void *) n
->host_start
, (int) host_size
, (void *) h
);
459 else if (n
->refcount
!= REFCOUNT_ACC_MAP_DATA
)
461 gomp_mutex_unlock (&acc_dev
->lock
);
462 gomp_fatal ("refusing to unmap block [%p,+%d] that has not been mapped"
463 " by 'acc_map_data'",
464 (void *) h
, (int) host_size
);
467 /* This should hold for all mappings created by acc_map_data. We are however
468 removing this mapping in a "finalize" manner, so dynamic_refcount > 1 does
470 assert (n
->dynamic_refcount
>= 1);
472 struct target_mem_desc
*tgt
= n
->tgt
;
474 /* Above, we've verified that the mapping must have been set up by
476 assert (tgt
->refcount
== 1);
478 /* Nullifying these fields prevents 'gomp_unmap_tgt' via 'gomp_remove_var'
479 from freeing the target memory. */
483 bool is_tgt_unmapped
= gomp_remove_var (acc_dev
, n
);
484 assert (is_tgt_unmapped
);
486 gomp_mutex_unlock (&acc_dev
->lock
);
490 thr
->prof_info
= NULL
;
491 thr
->api_info
= NULL
;
496 /* Helper function to map a single dynamic data item, represented by a single
497 mapping. The acc_dev->lock should be held on entry, and remains locked on
501 goacc_map_var_existing (struct gomp_device_descr
*acc_dev
, void *hostaddr
,
502 size_t size
, splay_tree_key n
)
507 void *d
= (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
+ hostaddr
510 if (hostaddr
+ size
> (void *) n
->host_end
)
512 gomp_mutex_unlock (&acc_dev
->lock
);
513 gomp_fatal ("[%p,+%d] not mapped", hostaddr
, (int) size
);
516 assert (n
->refcount
!= REFCOUNT_LINK
);
517 if (n
->refcount
!= REFCOUNT_INFINITY
518 && n
->refcount
!= REFCOUNT_ACC_MAP_DATA
)
520 n
->dynamic_refcount
++;
525 /* Enter dynamic mapping for a single datum. Return the device pointer. */
528 goacc_enter_datum (void **hostaddrs
, size_t *sizes
, void *kinds
, int async
)
533 if (!hostaddrs
[0] || !sizes
[0])
534 gomp_fatal ("[%p,+%d] is a bad range", hostaddrs
[0], (int) sizes
[0]);
536 goacc_lazy_initialize ();
538 struct goacc_thread
*thr
= goacc_thread ();
539 struct gomp_device_descr
*acc_dev
= thr
->dev
;
541 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
544 acc_prof_info prof_info
;
545 acc_api_info api_info
;
546 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
549 prof_info
.async
= async
;
550 prof_info
.async_queue
= prof_info
.async
;
553 gomp_mutex_lock (&acc_dev
->lock
);
555 n
= lookup_host (acc_dev
, hostaddrs
[0], sizes
[0]);
558 d
= goacc_map_var_existing (acc_dev
, hostaddrs
[0], sizes
[0], n
);
559 gomp_mutex_unlock (&acc_dev
->lock
);
563 const size_t mapnum
= 1;
565 gomp_mutex_unlock (&acc_dev
->lock
);
567 goacc_aq aq
= get_goacc_asyncqueue (async
);
569 struct target_mem_desc
*tgt
570 = goacc_map_vars (acc_dev
, aq
, mapnum
, hostaddrs
, NULL
, sizes
,
571 kinds
, true, GOMP_MAP_VARS_ENTER_DATA
);
573 assert (tgt
->list_count
== 1);
574 n
= tgt
->list
[0].key
;
576 assert (n
->refcount
== 1);
577 assert (n
->dynamic_refcount
== 0);
578 n
->dynamic_refcount
++;
580 d
= (void *) tgt
->tgt_start
;
585 thr
->prof_info
= NULL
;
586 thr
->api_info
= NULL
;
593 acc_create (void *h
, size_t s
)
595 unsigned short kinds
[1] = { GOMP_MAP_ALLOC
};
596 return goacc_enter_datum (&h
, &s
, &kinds
, acc_async_sync
);
600 acc_create_async (void *h
, size_t s
, int async
)
602 unsigned short kinds
[1] = { GOMP_MAP_ALLOC
};
603 goacc_enter_datum (&h
, &s
, &kinds
, async
);
606 /* acc_present_or_create used to be what acc_create is now. */
607 /* acc_pcreate is acc_present_or_create by a different name. */
608 #ifdef HAVE_ATTRIBUTE_ALIAS
609 strong_alias (acc_create
, acc_present_or_create
)
610 strong_alias (acc_create
, acc_pcreate
)
613 acc_present_or_create (void *h
, size_t s
)
615 return acc_create (h
, s
);
619 acc_pcreate (void *h
, size_t s
)
621 return acc_create (h
, s
);
626 acc_copyin (void *h
, size_t s
)
628 unsigned short kinds
[1] = { GOMP_MAP_TO
};
629 return goacc_enter_datum (&h
, &s
, &kinds
, acc_async_sync
);
633 acc_copyin_async (void *h
, size_t s
, int async
)
635 unsigned short kinds
[1] = { GOMP_MAP_TO
};
636 goacc_enter_datum (&h
, &s
, &kinds
, async
);
639 /* acc_present_or_copyin used to be what acc_copyin is now. */
640 /* acc_pcopyin is acc_present_or_copyin by a different name. */
641 #ifdef HAVE_ATTRIBUTE_ALIAS
642 strong_alias (acc_copyin
, acc_present_or_copyin
)
643 strong_alias (acc_copyin
, acc_pcopyin
)
646 acc_present_or_copyin (void *h
, size_t s
)
648 return acc_copyin (h
, s
);
652 acc_pcopyin (void *h
, size_t s
)
654 return acc_copyin (h
, s
);
659 /* Helper function to unmap a single data item. Device lock should be held on
660 entry, and remains locked on exit. */
663 goacc_exit_datum_1 (struct gomp_device_descr
*acc_dev
, void *h
, size_t s
,
664 unsigned short kind
, splay_tree_key n
, goacc_aq aq
)
666 assert (kind
!= GOMP_MAP_DETACH
667 && kind
!= GOMP_MAP_FORCE_DETACH
);
669 if ((uintptr_t) h
< n
->host_start
|| (uintptr_t) h
+ s
> n
->host_end
)
671 size_t host_size
= n
->host_end
- n
->host_start
;
672 gomp_mutex_unlock (&acc_dev
->lock
);
673 gomp_fatal ("[%p,+%d] outside mapped block [%p,+%d]",
674 (void *) h
, (int) s
, (void *) n
->host_start
, (int) host_size
);
677 bool finalize
= (kind
== GOMP_MAP_FORCE_FROM
678 || kind
== GOMP_MAP_DELETE
);
680 assert (n
->refcount
!= REFCOUNT_LINK
);
681 if (n
->refcount
!= REFCOUNT_INFINITY
682 && n
->refcount
!= REFCOUNT_ACC_MAP_DATA
683 && n
->refcount
< n
->dynamic_refcount
)
685 gomp_mutex_unlock (&acc_dev
->lock
);
686 gomp_fatal ("Dynamic reference counting assert fail\n");
689 if (n
->refcount
== REFCOUNT_ACC_MAP_DATA
)
693 /* Mappings created by acc_map_data are returned to initial
694 dynamic_refcount of 1. Can only be deleted by acc_unmap_data. */
695 n
->dynamic_refcount
= 1;
697 else if (n
->dynamic_refcount
)
699 /* When mapping is created by acc_map_data, dynamic_refcount must be
700 maintained at >= 1. */
701 if (n
->dynamic_refcount
> 1)
702 n
->dynamic_refcount
--;
707 if (n
->refcount
!= REFCOUNT_INFINITY
)
708 n
->refcount
-= n
->dynamic_refcount
;
709 n
->dynamic_refcount
= 0;
711 else if (n
->dynamic_refcount
)
713 if (n
->refcount
!= REFCOUNT_INFINITY
)
715 n
->dynamic_refcount
--;
718 if (n
->refcount
== 0)
720 bool copyout
= (kind
== GOMP_MAP_FROM
721 || kind
== GOMP_MAP_FORCE_FROM
);
724 void *d
= (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
725 + (uintptr_t) h
- n
->host_start
);
726 gomp_copy_dev2host (acc_dev
, aq
, h
, d
, s
);
730 /* TODO We can't do the 'is_tgt_unmapped' checking -- see the
731 'gomp_unref_tgt' comment in
732 <http://mid.mail-archive.com/878snl36eu.fsf@euler.schwinge.homeip.net>;
734 gomp_remove_var_async (acc_dev
, n
, aq
);
737 size_t num_mappings
= 0;
738 /* If the target_mem_desc represents a single data mapping, we can
739 check that it is freed when this splay tree key's refcount reaches
740 zero. Otherwise (e.g. for a 'GOMP_MAP_STRUCT' mapping with
741 multiple members), fall back to skipping the test. */
742 for (size_t l_i
= 0; l_i
< n
->tgt
->list_count
; ++l_i
)
743 if (n
->tgt
->list
[l_i
].key
744 && !n
->tgt
->list
[l_i
].is_attach
)
746 bool is_tgt_unmapped
= gomp_remove_var (acc_dev
, n
);
747 assert (is_tgt_unmapped
|| num_mappings
> 1);
753 /* Exit a dynamic mapping for a single variable. */
756 goacc_exit_datum (void *h
, size_t s
, unsigned short kind
, int async
)
758 /* No need to call lazy open, as the data must already have been
763 struct goacc_thread
*thr
= goacc_thread ();
764 struct gomp_device_descr
*acc_dev
= thr
->dev
;
766 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
769 acc_prof_info prof_info
;
770 acc_api_info api_info
;
771 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
774 prof_info
.async
= async
;
775 prof_info
.async_queue
= prof_info
.async
;
778 gomp_mutex_lock (&acc_dev
->lock
);
780 splay_tree_key n
= lookup_host (acc_dev
, h
, s
);
781 /* Non-present data is a no-op: PR92726, RP92970, PR92984. */
784 goacc_aq aq
= get_goacc_asyncqueue (async
);
785 goacc_exit_datum_1 (acc_dev
, h
, s
, kind
, n
, aq
);
788 gomp_mutex_unlock (&acc_dev
->lock
);
792 thr
->prof_info
= NULL
;
793 thr
->api_info
= NULL
;
798 acc_delete (void *h
, size_t s
)
800 goacc_exit_datum (h
, s
, GOMP_MAP_RELEASE
, acc_async_sync
);
804 acc_delete_async (void *h
, size_t s
, int async
)
806 goacc_exit_datum (h
, s
, GOMP_MAP_RELEASE
, async
);
810 acc_delete_finalize (void *h
, size_t s
)
812 goacc_exit_datum (h
, s
, GOMP_MAP_DELETE
, acc_async_sync
);
816 acc_delete_finalize_async (void *h
, size_t s
, int async
)
818 goacc_exit_datum (h
, s
, GOMP_MAP_DELETE
, async
);
822 acc_copyout (void *h
, size_t s
)
824 goacc_exit_datum (h
, s
, GOMP_MAP_FROM
, acc_async_sync
);
828 acc_copyout_async (void *h
, size_t s
, int async
)
830 goacc_exit_datum (h
, s
, GOMP_MAP_FROM
, async
);
834 acc_copyout_finalize (void *h
, size_t s
)
836 goacc_exit_datum (h
, s
, GOMP_MAP_FORCE_FROM
, acc_async_sync
);
840 acc_copyout_finalize_async (void *h
, size_t s
, int async
)
842 goacc_exit_datum (h
, s
, GOMP_MAP_FORCE_FROM
, async
);
846 update_dev_host (int is_dev
, void *h
, size_t s
, int async
)
851 goacc_lazy_initialize ();
853 struct goacc_thread
*thr
= goacc_thread ();
854 struct gomp_device_descr
*acc_dev
= thr
->dev
;
856 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
859 /* Fortran optional arguments that are non-present result in a
860 NULL host address here. This can safely be ignored as it is
861 not possible to 'update' a non-present optional argument. */
865 acc_prof_info prof_info
;
866 acc_api_info api_info
;
867 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
870 prof_info
.async
= async
;
871 prof_info
.async_queue
= prof_info
.async
;
874 gomp_mutex_lock (&acc_dev
->lock
);
876 n
= lookup_host (acc_dev
, h
, s
);
880 gomp_mutex_unlock (&acc_dev
->lock
);
881 gomp_fatal ("[%p,%d] is not mapped", h
, (int)s
);
884 d
= (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
885 + (uintptr_t) h
- n
->host_start
);
887 goacc_aq aq
= get_goacc_asyncqueue (async
);
890 gomp_copy_host2dev (acc_dev
, aq
, d
, h
, s
, false, /* TODO: cbuf? */ NULL
);
892 gomp_copy_dev2host (acc_dev
, aq
, h
, d
, s
);
894 gomp_mutex_unlock (&acc_dev
->lock
);
898 thr
->prof_info
= NULL
;
899 thr
->api_info
= NULL
;
904 acc_update_device (void *h
, size_t s
)
906 update_dev_host (1, h
, s
, acc_async_sync
);
910 acc_update_device_async (void *h
, size_t s
, int async
)
912 update_dev_host (1, h
, s
, async
);
916 acc_update_self (void *h
, size_t s
)
918 update_dev_host (0, h
, s
, acc_async_sync
);
922 acc_update_self_async (void *h
, size_t s
, int async
)
924 update_dev_host (0, h
, s
, async
);
928 acc_attach_async (void **hostaddr
, int async
)
930 struct goacc_thread
*thr
= goacc_thread ();
931 struct gomp_device_descr
*acc_dev
= thr
->dev
;
932 goacc_aq aq
= get_goacc_asyncqueue (async
);
934 struct splay_tree_key_s cur_node
;
937 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
940 gomp_mutex_lock (&acc_dev
->lock
);
942 cur_node
.host_start
= (uintptr_t) hostaddr
;
943 cur_node
.host_end
= cur_node
.host_start
+ sizeof (void *);
944 n
= splay_tree_lookup (&acc_dev
->mem_map
, &cur_node
);
948 gomp_mutex_unlock (&acc_dev
->lock
);
949 gomp_fatal ("struct not mapped for acc_attach");
952 gomp_attach_pointer (acc_dev
, aq
, &acc_dev
->mem_map
, n
, (uintptr_t) hostaddr
,
955 gomp_mutex_unlock (&acc_dev
->lock
);
959 acc_attach (void **hostaddr
)
961 acc_attach_async (hostaddr
, acc_async_sync
);
965 goacc_detach_internal (void **hostaddr
, int async
, bool finalize
)
967 struct goacc_thread
*thr
= goacc_thread ();
968 struct gomp_device_descr
*acc_dev
= thr
->dev
;
969 struct splay_tree_key_s cur_node
;
971 struct goacc_asyncqueue
*aq
= get_goacc_asyncqueue (async
);
973 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
976 gomp_mutex_lock (&acc_dev
->lock
);
978 cur_node
.host_start
= (uintptr_t) hostaddr
;
979 cur_node
.host_end
= cur_node
.host_start
+ sizeof (void *);
980 n
= splay_tree_lookup (&acc_dev
->mem_map
, &cur_node
);
984 gomp_mutex_unlock (&acc_dev
->lock
);
985 gomp_fatal ("struct not mapped for acc_detach");
988 gomp_detach_pointer (acc_dev
, aq
, n
, (uintptr_t) hostaddr
, finalize
, NULL
);
990 gomp_mutex_unlock (&acc_dev
->lock
);
994 acc_detach (void **hostaddr
)
996 goacc_detach_internal (hostaddr
, acc_async_sync
, false);
1000 acc_detach_async (void **hostaddr
, int async
)
1002 goacc_detach_internal (hostaddr
, async
, false);
1006 acc_detach_finalize (void **hostaddr
)
1008 goacc_detach_internal (hostaddr
, acc_async_sync
, true);
1012 acc_detach_finalize_async (void **hostaddr
, int async
)
1014 goacc_detach_internal (hostaddr
, async
, true);
1017 /* Some types of (pointer) variables use several consecutive mappings, which
1018 must be treated as a group for enter/exit data directives. This function
1019 returns the last mapping in such a group (inclusive), or POS for singleton
1023 find_group_last (int pos
, size_t mapnum
, size_t *sizes
, unsigned short *kinds
)
1025 unsigned char kind0
= kinds
[pos
] & 0xff;
1026 int first_pos
= pos
;
1030 case GOMP_MAP_TO_PSET
:
1031 if (pos
+ 1 < mapnum
1032 && (kinds
[pos
+ 1] & 0xff) == GOMP_MAP_ATTACH
)
1035 while (pos
+ 1 < mapnum
1036 && (kinds
[pos
+ 1] & 0xff) == GOMP_MAP_POINTER
)
1038 /* We expect at least one GOMP_MAP_POINTER (if not a single
1039 GOMP_MAP_ATTACH) after a GOMP_MAP_TO_PSET. */
1040 assert (pos
> first_pos
);
1043 case GOMP_MAP_STRUCT
:
1044 case GOMP_MAP_STRUCT_UNORD
:
1048 case GOMP_MAP_POINTER
:
1049 case GOMP_MAP_ALWAYS_POINTER
:
1050 /* These mappings are only expected after some other mapping. If we
1051 see one by itself, something has gone wrong. */
1052 gomp_fatal ("unexpected mapping");
1055 case GOMP_MAP_ATTACH
:
1059 /* GOMP_MAP_ALWAYS_POINTER can only appear directly after some other
1061 if (pos
+ 1 < mapnum
)
1063 unsigned char kind1
= kinds
[pos
+ 1] & 0xff;
1064 if (kind1
== GOMP_MAP_ALWAYS_POINTER
)
1068 /* We can have a single GOMP_MAP_ATTACH mapping after a to/from
1070 if (pos
+ 1 < mapnum
1071 && (kinds
[pos
+ 1] & 0xff) == GOMP_MAP_ATTACH
)
1074 /* We can have zero or more GOMP_MAP_POINTER mappings after a to/from
1076 while (pos
+ 1 < mapnum
1077 && (kinds
[pos
+ 1] & 0xff) == GOMP_MAP_POINTER
)
1084 /* Map variables for OpenACC "enter data". We can't just call
1085 goacc_map_vars once, because individual mapped variables might have
1086 "exit data" called for them at different times. */
1089 goacc_enter_data_internal (struct gomp_device_descr
*acc_dev
, size_t mapnum
,
1090 void **hostaddrs
, size_t *sizes
,
1091 unsigned short *kinds
, goacc_aq aq
)
1093 gomp_mutex_lock (&acc_dev
->lock
);
1095 for (size_t i
= 0; i
< mapnum
; i
++)
1098 size_t group_last
= find_group_last (i
, mapnum
, sizes
, kinds
);
1099 bool struct_p
= false;
1100 size_t size
, groupnum
= (group_last
- i
) + 1;
1102 switch (kinds
[i
] & 0xff)
1104 case GOMP_MAP_STRUCT
:
1105 case GOMP_MAP_STRUCT_UNORD
:
1107 size
= (uintptr_t) hostaddrs
[group_last
] + sizes
[group_last
]
1108 - (uintptr_t) hostaddrs
[i
];
1113 case GOMP_MAP_ATTACH
:
1114 size
= sizeof (void *);
1121 n
= lookup_host (acc_dev
, hostaddrs
[i
], size
);
1125 for (size_t j
= i
+ 1; j
<= group_last
; j
++)
1127 struct splay_tree_key_s cur_node
;
1128 cur_node
.host_start
= (uintptr_t) hostaddrs
[j
];
1129 cur_node
.host_end
= cur_node
.host_start
+ sizes
[j
];
1131 = splay_tree_lookup (&acc_dev
->mem_map
, &cur_node
);
1133 || n2
->tgt
!= n
->tgt
1134 || n2
->host_start
- n
->host_start
1135 != n2
->tgt_offset
- n
->tgt_offset
)
1137 gomp_mutex_unlock (&acc_dev
->lock
);
1138 gomp_fatal ("Trying to map into device [%p..%p) structure "
1139 "element when other mapped elements from the "
1140 "same structure weren't mapped together with "
1141 "it", (void *) cur_node
.host_start
,
1142 (void *) cur_node
.host_end
);
1145 /* This is a special case because we must increment the refcount by
1146 the number of mapped struct elements, rather than by one. */
1147 if (n
->refcount
!= REFCOUNT_INFINITY
1148 && n
->refcount
!= REFCOUNT_ACC_MAP_DATA
)
1149 n
->refcount
+= groupnum
- 1;
1150 n
->dynamic_refcount
+= groupnum
- 1;
1152 else if (n
&& groupnum
== 1)
1154 void *h
= hostaddrs
[i
];
1155 size_t s
= sizes
[i
];
1157 if ((kinds
[i
] & 0xff) == GOMP_MAP_ATTACH
)
1159 gomp_attach_pointer (acc_dev
, aq
, &acc_dev
->mem_map
, n
,
1160 (uintptr_t) h
, s
, NULL
, false);
1161 /* OpenACC 'attach'/'detach' doesn't affect structured/dynamic
1162 reference counts ('n->refcount', 'n->dynamic_refcount'). */
1165 goacc_map_var_existing (acc_dev
, h
, s
, n
);
1167 else if (n
&& groupnum
> 1)
1169 assert (n
->refcount
!= REFCOUNT_LINK
);
1171 for (size_t j
= i
+ 1; j
<= group_last
; j
++)
1172 if ((kinds
[j
] & 0xff) == GOMP_MAP_ATTACH
)
1175 = lookup_host (acc_dev
, hostaddrs
[j
], sizeof (void *));
1176 gomp_attach_pointer (acc_dev
, aq
, &acc_dev
->mem_map
, m
,
1177 (uintptr_t) hostaddrs
[j
], sizes
[j
], NULL
,
1181 bool processed
= false;
1183 struct target_mem_desc
*tgt
= n
->tgt
;
1185 /* Minimal OpenACC variant corresponding to PR96668
1186 "[OpenMP] Re-mapping allocated but previously unallocated
1187 allocatable does not work" 'libgomp/target.c' changes, so that
1188 OpenACC 'declare' code à la PR106643
1189 "[gfortran + OpenACC] Allocate in module causes refcount error"
1190 has a chance to work. */
1191 if ((kinds
[i
] & 0xff) == GOMP_MAP_TO_PSET
1192 && tgt
->list_count
== 0)
1194 /* 'declare target'. */
1195 assert (n
->refcount
== REFCOUNT_INFINITY
);
1197 for (size_t k
= 1; k
< groupnum
; k
++)
1199 /* The only thing we expect to see here. */
1200 assert ((kinds
[i
+ k
] & 0xff) == GOMP_MAP_POINTER
);
1203 /* Let 'goacc_map_vars' -> 'gomp_map_vars_internal' handle
1205 gomp_mutex_unlock (&acc_dev
->lock
);
1206 struct target_mem_desc
*tgt_
1207 = goacc_map_vars (acc_dev
, aq
, groupnum
, &hostaddrs
[i
], NULL
,
1208 &sizes
[i
], &kinds
[i
], true,
1209 GOMP_MAP_VARS_ENTER_DATA
);
1210 assert (tgt_
== NULL
);
1211 gomp_mutex_lock (&acc_dev
->lock
);
1213 /* Given that 'goacc_exit_data_internal'/'goacc_exit_datum_1'
1214 will always see 'n->refcount == REFCOUNT_INFINITY',
1215 there's no need to adjust 'n->dynamic_refcount' here. */
1220 assert (n
->refcount
!= REFCOUNT_INFINITY
1221 && n
->refcount
!= REFCOUNT_ACC_MAP_DATA
);
1223 for (size_t j
= 0; j
< tgt
->list_count
; j
++)
1224 if (tgt
->list
[j
].key
== n
)
1226 /* We are processing a group of mappings (e.g.
1227 [GOMP_MAP_TO, GOMP_MAP_TO_PSET, GOMP_MAP_POINTER]).
1228 Find the right group in the target_mem_desc's variable
1229 list, and increment the refcounts for each item in that
1231 for (size_t k
= 0; k
< groupnum
; k
++)
1232 if (j
+ k
< tgt
->list_count
1233 && tgt
->list
[j
+ k
].key
1234 && !tgt
->list
[j
+ k
].is_attach
)
1236 tgt
->list
[j
+ k
].key
->refcount
++;
1237 tgt
->list
[j
+ k
].key
->dynamic_refcount
++;
1245 gomp_mutex_unlock (&acc_dev
->lock
);
1246 gomp_fatal ("dynamic refcount incrementing failed for "
1250 else if (hostaddrs
[i
])
1252 /* The data is not mapped already. Map it now, unless the first
1253 member in the group has a NULL pointer (e.g. a non-present
1254 optional parameter). */
1255 gomp_mutex_unlock (&acc_dev
->lock
);
1257 struct target_mem_desc
*tgt
1258 = goacc_map_vars (acc_dev
, aq
, groupnum
, &hostaddrs
[i
], NULL
,
1259 &sizes
[i
], &kinds
[i
], true,
1260 GOMP_MAP_VARS_ENTER_DATA
);
1263 gomp_mutex_lock (&acc_dev
->lock
);
1265 for (size_t j
= 0; j
< tgt
->list_count
; j
++)
1267 n
= tgt
->list
[j
].key
;
1268 if (n
&& !tgt
->list
[j
].is_attach
)
1269 n
->dynamic_refcount
++;
1276 gomp_mutex_unlock (&acc_dev
->lock
);
1279 /* Unmap variables for OpenACC "exit data". */
1282 goacc_exit_data_internal (struct gomp_device_descr
*acc_dev
, size_t mapnum
,
1283 void **hostaddrs
, size_t *sizes
,
1284 unsigned short *kinds
, goacc_aq aq
)
1286 gomp_mutex_lock (&acc_dev
->lock
);
1288 /* Handle "detach" before copyback/deletion of mapped data. */
1289 for (size_t i
= 0; i
< mapnum
; ++i
)
1291 unsigned char kind
= kinds
[i
] & 0xff;
1292 bool finalize
= false;
1295 case GOMP_MAP_FORCE_DETACH
:
1299 case GOMP_MAP_DETACH
:
1301 struct splay_tree_key_s cur_node
;
1302 uintptr_t hostaddr
= (uintptr_t) hostaddrs
[i
];
1303 cur_node
.host_start
= hostaddr
;
1304 cur_node
.host_end
= cur_node
.host_start
+ sizeof (void *);
1306 = splay_tree_lookup (&acc_dev
->mem_map
, &cur_node
);
1310 gomp_mutex_unlock (&acc_dev
->lock
);
1311 gomp_fatal ("struct not mapped for detach operation");
1314 gomp_detach_pointer (acc_dev
, aq
, n
, hostaddr
, finalize
, NULL
);
1322 for (size_t i
= 0; i
< mapnum
; ++i
)
1324 unsigned char kind
= kinds
[i
] & 0xff;
1329 case GOMP_MAP_FORCE_FROM
:
1330 case GOMP_MAP_TO_PSET
:
1331 case GOMP_MAP_POINTER
:
1332 case GOMP_MAP_DELETE
:
1333 case GOMP_MAP_RELEASE
:
1335 struct splay_tree_key_s cur_node
;
1337 if (kind
== GOMP_MAP_POINTER
)
1338 size
= sizeof (void *);
1341 cur_node
.host_start
= (uintptr_t) hostaddrs
[i
];
1342 cur_node
.host_end
= cur_node
.host_start
+ size
;
1344 = splay_tree_lookup (&acc_dev
->mem_map
, &cur_node
);
1349 goacc_exit_datum_1 (acc_dev
, hostaddrs
[i
], size
, kind
, n
, aq
);
1353 case GOMP_MAP_STRUCT
:
1354 case GOMP_MAP_STRUCT_UNORD
:
1355 /* Skip the 'GOMP_MAP_STRUCT' itself, and use the regular processing
1356 for all its entries. This special handling exists for GCC 10.1
1357 compatibility; afterwards, we're not generating these no-op
1358 'GOMP_MAP_STRUCT's anymore. */
1361 case GOMP_MAP_DETACH
:
1362 case GOMP_MAP_FORCE_DETACH
:
1363 /* OpenACC 'attach'/'detach' doesn't affect structured/dynamic
1364 reference counts ('n->refcount', 'n->dynamic_refcount'). */
1368 gomp_fatal (">>>> goacc_exit_data_internal UNHANDLED kind 0x%.2x",
1373 gomp_mutex_unlock (&acc_dev
->lock
);
1377 goacc_enter_exit_data_internal (int flags_m
, size_t mapnum
, void **hostaddrs
,
1378 size_t *sizes
, unsigned short *kinds
,
1379 bool data_enter
, int async
, int num_waits
,
1382 int flags
= GOACC_FLAGS_UNMARSHAL (flags_m
);
1384 struct goacc_thread
*thr
;
1385 struct gomp_device_descr
*acc_dev
;
1387 goacc_lazy_initialize ();
1389 thr
= goacc_thread ();
1392 bool profiling_p
= GOACC_PROFILING_DISPATCH_P (true);
1394 acc_prof_info prof_info
;
1397 thr
->prof_info
= &prof_info
;
1399 prof_info
.event_type
1400 = data_enter
? acc_ev_enter_data_start
: acc_ev_exit_data_start
;
1401 prof_info
.valid_bytes
= _ACC_PROF_INFO_VALID_BYTES
;
1402 prof_info
.version
= _ACC_PROF_INFO_VERSION
;
1403 prof_info
.device_type
= acc_device_type (acc_dev
->type
);
1404 prof_info
.device_number
= acc_dev
->target_id
;
1405 prof_info
.thread_id
= -1;
1406 prof_info
.async
= async
;
1407 prof_info
.async_queue
= prof_info
.async
;
1408 prof_info
.src_file
= NULL
;
1409 prof_info
.func_name
= NULL
;
1410 prof_info
.line_no
= -1;
1411 prof_info
.end_line_no
= -1;
1412 prof_info
.func_line_no
= -1;
1413 prof_info
.func_end_line_no
= -1;
1415 acc_event_info enter_exit_data_event_info
;
1418 enter_exit_data_event_info
.other_event
.event_type
1419 = prof_info
.event_type
;
1420 enter_exit_data_event_info
.other_event
.valid_bytes
1421 = _ACC_OTHER_EVENT_INFO_VALID_BYTES
;
1422 enter_exit_data_event_info
.other_event
.parent_construct
1423 = data_enter
? acc_construct_enter_data
: acc_construct_exit_data
;
1424 enter_exit_data_event_info
.other_event
.implicit
= 0;
1425 enter_exit_data_event_info
.other_event
.tool_info
= NULL
;
1427 acc_api_info api_info
;
1430 thr
->api_info
= &api_info
;
1432 api_info
.device_api
= acc_device_api_none
;
1433 api_info
.valid_bytes
= _ACC_API_INFO_VALID_BYTES
;
1434 api_info
.device_type
= prof_info
.device_type
;
1435 api_info
.vendor
= -1;
1436 api_info
.device_handle
= NULL
;
1437 api_info
.context_handle
= NULL
;
1438 api_info
.async_handle
= NULL
;
1442 goacc_profiling_dispatch (&prof_info
, &enter_exit_data_event_info
,
1445 if ((acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
1446 || (flags
& GOACC_FLAG_HOST_FALLBACK
))
1448 prof_info
.device_type
= acc_device_host
;
1449 api_info
.device_type
= prof_info
.device_type
;
1455 goacc_wait (async
, num_waits
, ap
);
1457 goacc_aq aq
= get_goacc_asyncqueue (async
);
1460 goacc_enter_data_internal (acc_dev
, mapnum
, hostaddrs
, sizes
, kinds
, aq
);
1462 goacc_exit_data_internal (acc_dev
, mapnum
, hostaddrs
, sizes
, kinds
, aq
);
1467 prof_info
.event_type
1468 = data_enter
? acc_ev_enter_data_end
: acc_ev_exit_data_end
;
1469 enter_exit_data_event_info
.other_event
.event_type
= prof_info
.event_type
;
1470 goacc_profiling_dispatch (&prof_info
, &enter_exit_data_event_info
,
1473 thr
->prof_info
= NULL
;
1474 thr
->api_info
= NULL
;
1478 /* Legacy entry point (GCC 11 and earlier). */
1481 GOACC_enter_exit_data (int flags_m
, size_t mapnum
, void **hostaddrs
,
1482 size_t *sizes
, unsigned short *kinds
, int async
,
1485 /* Determine if this is an OpenACC "enter data". */
1486 bool data_enter
= false;
1487 for (size_t i
= 0; i
< mapnum
; ++i
)
1489 unsigned char kind
= kinds
[i
] & 0xff;
1491 if (kind
== GOMP_MAP_POINTER
1492 || kind
== GOMP_MAP_TO_PSET
1493 || kind
== GOMP_MAP_STRUCT
1494 || kind
== GOMP_MAP_STRUCT_UNORD
)
1497 if (kind
== GOMP_MAP_FORCE_ALLOC
1498 || kind
== GOMP_MAP_FORCE_PRESENT
1499 || kind
== GOMP_MAP_ATTACH
1500 || kind
== GOMP_MAP_FORCE_TO
1501 || kind
== GOMP_MAP_TO
1502 || kind
== GOMP_MAP_ALLOC
)
1508 if (kind
== GOMP_MAP_RELEASE
1509 || kind
== GOMP_MAP_DELETE
1510 || kind
== GOMP_MAP_DETACH
1511 || kind
== GOMP_MAP_FORCE_DETACH
1512 || kind
== GOMP_MAP_FROM
1513 || kind
== GOMP_MAP_FORCE_FROM
)
1516 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
1521 va_start (ap
, num_waits
);
1522 goacc_enter_exit_data_internal (flags_m
, mapnum
, hostaddrs
, sizes
, kinds
,
1523 data_enter
, async
, num_waits
, &ap
);
1528 GOACC_enter_data (int flags_m
, size_t mapnum
, void **hostaddrs
,
1529 size_t *sizes
, unsigned short *kinds
, int async
,
1533 va_start (ap
, num_waits
);
1534 goacc_enter_exit_data_internal (flags_m
, mapnum
, hostaddrs
, sizes
, kinds
,
1535 true, async
, num_waits
, &ap
);
1540 GOACC_exit_data (int flags_m
, size_t mapnum
, void **hostaddrs
,
1541 size_t *sizes
, unsigned short *kinds
, int async
,
1545 va_start (ap
, num_waits
);
1546 goacc_enter_exit_data_internal (flags_m
, mapnum
, hostaddrs
, sizes
, kinds
,
1547 false, async
, num_waits
, &ap
);
1552 GOACC_declare (int flags_m
, size_t mapnum
,
1553 void **hostaddrs
, size_t *sizes
, unsigned short *kinds
)
1555 for (size_t i
= 0; i
< mapnum
; i
++)
1557 unsigned char kind
= kinds
[i
] & 0xff;
1559 if (kind
== GOMP_MAP_POINTER
|| kind
== GOMP_MAP_TO_PSET
)
1564 case GOMP_MAP_ALLOC
:
1565 if (acc_is_present (hostaddrs
[i
], sizes
[i
]))
1568 case GOMP_MAP_FORCE_ALLOC
:
1570 case GOMP_MAP_FORCE_TO
:
1571 goacc_enter_exit_data_internal (flags_m
, 1, &hostaddrs
[i
], &sizes
[i
],
1572 &kinds
[i
], true, GOMP_ASYNC_SYNC
, 0, NULL
);
1576 case GOMP_MAP_FORCE_FROM
:
1577 case GOMP_MAP_RELEASE
:
1578 case GOMP_MAP_DELETE
:
1579 goacc_enter_exit_data_internal (flags_m
, 1, &hostaddrs
[i
], &sizes
[i
],
1580 &kinds
[i
], false, GOMP_ASYNC_SYNC
, 0, NULL
);
1583 case GOMP_MAP_FORCE_DEVICEPTR
:
1586 case GOMP_MAP_FORCE_PRESENT
:
1587 if (!acc_is_present (hostaddrs
[i
], sizes
[i
]))
1588 gomp_fatal ("[%p,%ld] is not mapped", hostaddrs
[i
],
1589 (unsigned long) sizes
[i
]);