| blob | cab08c43550d7e7872f466c510f5b9aac91c7f20 |
1 /*
2 Copyright (c) 2014-2016 Intel Corporation. All Rights Reserved.
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
8 * Redistributions of source code must retain the above copyright
9 notice, this list of conditions and the following disclaimer.
10 * Redistributions in binary form must reproduce the above copyright
11 notice, this list of conditions and the following disclaimer in the
12 documentation and/or other materials provided with the distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
31 // Forward declaration as the following 2 functions are declared as friend
32 // in offload_engine.h.
33 // CLANG does not like static to been after friend declaration.
38 #ifdef MYO_SUPPORT
40 #endif
42 #include <malloc.h>
43 #ifndef TARGET_WINNT
44 #include <alloca.h>
45 #include <elf.h>
47 #include <errno.h>
48 #include <fcntl.h>
49 #include <stdlib.h>
50 #include <string.h>
51 #include <sys/stat.h>
52 #include <sys/types.h>
54 #include <algorithm>
55 #include <bitset>
56 #include <iostream>
58 #if defined(HOST_WINNT)
60 #else
62 #endif
64 #define GET_OFFLOAD_NUMBER(timer_data) \
65 timer_data? timer_data->offload_number : 0
70 #ifdef TARGET_WINNT
71 // Windows does not support imports from libraries without actually
72 // including them as dependence. We don't want to include in the
73 // dependence since is it used only for Fortran when traceback is enabled.
74 // Chose to implement it with GetProcAddress.
75 #define FORTRAN_TRACE_BACK win_for__continue_traceback
77 {
78 HINSTANCE hDLL;
87 }
92 }
93 }
97 }
99 }
103 #define FORTRAN_TRACE_BACK for__continue_traceback
105 // for__continue_traceback is provided as a dummy to resolve link time symbols
106 // for C/C++ programs. For Fortran the actual fortran library function in
107 // libifcore.so is used.
108 #pragma weak for__continue_traceback
110 {
114 }
118 #ifdef TARGET_WINNT
119 // Small subset of ELF declarations for Windows which is needed to compile
120 // this file. ELF header is used to understand what binary type is contained
121 // in the target image - shared library or executable.
128 #define EI_NIDENT 16
130 #define ET_EXEC 2
131 #define ET_DYN 3
134 {
136 Elf64_Half e_type;
137 Elf64_Half e_machine;
138 Elf64_Word e_version;
139 Elf64_Addr e_entry;
140 Elf64_Off e_phoff;
141 Elf64_Off e_shoff;
142 Elf64_Word e_flags;
143 Elf64_Half e_ehsize;
144 Elf64_Half e_phentsize;
145 Elf64_Half e_phnum;
146 Elf64_Half e_shentsize;
147 Elf64_Half e_shnum;
148 Elf64_Half e_shstrndx;
152 // Host console and file logging
161 // DMA channel count used by COI and set via
162 // OFFLOAD_DMA_CHANNEL_COUNT environment variable
165 // Trace information
170 "INOUT"
171 };
197 };
201 pthread_key_t mic_thread_key;
202 MicEnvVar mic_env_vars;
205 // MIC_STACKSIZE
208 // MIC_BUFFERSIZE
211 // Preallocated 4K page memory size for buffers on MIC
214 // Preallocated 2M page memory size for buffers on MIC
218 // LD_LIBRARY_PATH for KNC
221 // LD_LIBRARY_PATH for KNL
225 // MIC_PROXY_IO
228 // MIC_PROXY_FS_ROOT
231 // Threshold for creating buffers with large pages. Buffer is created
232 // with large pages hint if its size exceeds the threshold value.
233 // By default large pages are disabled right now (by setting default
234 // value for threshold to MAX) due to HSD 4114629.
247 // device initialization type
251 // active wait
255 // wait even for asynchronous offload
256 // true for now still the performance issue with COI is not fixed
260 // OMP_DEFAULT_DEVICE
264 //OFFLOAD_PARALLEL_COPY
268 //Use COI interface for noncontiguous transfer if it exists.
273 // The list of pending target libraries
280 // Set of OffloadDescriptors of asynchronous offloads that are not destroyed
283 // Target executable
285 // is true if last loaded image is dll
287 // is true if myo library is loaded when dll is loaded
290 // Print readable offload flags
293 OffloadFlags offload_flags
294 )
295 {
296 // Sized big enough for all flag names
304 }
308 }
312 }
313 }
315 // Print readable varDesc flags
318 varDescFlags offload_flags
319 )
320 {
321 // Sized big enough for all flag names
329 }
334 }
339 }
344 }
349 }
354 }
359 }
364 }
369 }
374 }
379 }
384 }
389 }
394 }
399 }
404 }
408 }
412 }
413 }
416 {
420 }
423 }
431 }
436 }
438 }
441 }
443 }
446 {
447 // special case for the 'process died' error
450 }
456 }
457 /* fallthru */
478 }
479 }
482 }
485 {
498 }
499 }
501 // is_targetptr == 0 && is_prealloc == 0 - allocation of pointer data;
502 // is_targetptr == 1 && is_prealloc == 0 - allocation of target memory:
503 // allocate memory at target; use its value as base in target table.
504 // is_targetptr == 1 && is_prealloc == 1 - use preallocated target memory:
505 // base - is address at target of preallocated memory; use its value as
506 // base in target table.
517 bool pin
518 )
519 {
520 // total length of base
523 COIBUFFER targptr_buf;
524 COIRESULT res;
528 // create buffer with large pages if data length exceeds
529 // large page threshold
532 }
533 // Allocate memory at target for targetptr without preallocated as we need
534 // its address as base argument in call to m_device.insert_ptr_data
538 length,
539 COI_BUFFER_OPENCL,
540 buffer_flags,
548 }
551 }
553 }
560 }
563 }
565 }
566 }
572 // add new entry
577 // if ptr_data is found just need to check it for overlapping
581 }
583 // If association is not found we must create it.
588 }
596 // align should be a power of 2
599 // offset within mic_buffer. Can do offset optimization
600 // only when source address alignment satisfies requested
601 // alignment on the target (cq172736).
605 }
606 }
608 // buffer size and flags
611 // For targetptr there is no CPU buffer
613 // create CPU buffer
616 c_offload_create_buf_host,
617 "Creating buffer from source memory %p, "
620 // result is not checked because we can continue without cpu
621 // buffer. In this case we will use COIBufferRead/Write
622 // instead of COIBufferCopy.
625 COI_BUFFER_OPENCL,
627 base,
631 }
633 // create MIC buffer
637 c_offload_create_buf_mic,
638 "Creating buffer from sink memory: "
641 buffer_flags);
643 COI_BUFFER_NORMAL,
644 COI_SINK_MEMORY,
645 base,
652 }
655 }
658 }
659 }
662 }
666 c_offload_create_buf_mic,
667 "Creating buffer for sink: size %lld, offset %d, "
671 COI_BUFFER_NORMAL,
672 buffer_flags,
680 }
683 }
686 }
687 }
690 // make buffer valid on the device.
693 COI_BUFFER_VALID,
694 COI_BUFFER_NO_MOVE,
699 }
702 }
705 }
708 COI_PROCESS_SOURCE,
709 COI_BUFFER_INVALID,
710 COI_BUFFER_NO_MOVE,
715 }
718 }
721 }
722 }
723 }
726 }
735 // This is not a new entry. Make sure that provided address range fits
736 // into existing one.
743 }
745 // if the entry is associated with static data it may not have buffers
746 // created because they are created on demand.
749 }
750 }
753 }
761 bool report_error
762 )
763 {
764 // total length of base
771 // find existing association in pointer table
779 }
782 }
788 // make sure that provided address range fits into existing one
796 }
801 }
803 // if the entry is associated with static data it may not have buffers
804 // created because they are created on demand.
807 }
810 }
815 )
816 {
822 // find existing association in pointer table
825 // MIC address should have been assigned.
826 // For now assume does not exist and get the addr
827 // if ((ptr_data == 0) || ptr_data->mic_addr) {
833 }
841 }
842 }
848 }
851 {
860 COI_BUFFER_OPENCL,
870 }
872 }
873 }
881 COI_BUFFER_NORMAL,
882 COI_SINK_MEMORY,
891 }
893 }
894 }
897 }
900 {
907 }
910 }
912 }
913 }
915 }
918 COIBUFFER targ_buf,
919 uint64_t size
920 )
921 {
925 COIRESULT res;
929 targ_buf,
931 ptr,
932 size,
933 COI_COPY_UNSPECIFIED,
940 }
942 }
944 }
948 PersistData *cur_el
949 )
950 {
959 }
970 {
971 //mutex_locker_t locker(stack_alloc_lock);
981 "stack_begin=%p, routine_id=%d, buf_size=%d,"
995 // this stack data must be destroyed
998 erase++;
1000 }
1001 }
1004 // this stack data must be destroyed
1005 // because the current function is a dynamic sibling
1008 erase++;
1011 }
1014 // stack data is reused
1017 // all obsolete stack sections must be erased from the list
1019 m_in_datalen +=
1021 }
1026 }
1027 }
1032 }
1033 }
1036 // all obsolete stack sections must be erased from the list
1039 }
1040 // new stack table is created
1042 // create MIC buffer
1043 COIRESULT res;
1046 // create buffer with large pages if data length exceeds
1047 // large page threshold
1050 }
1052 COI_BUFFER_NORMAL,
1053 buffer_flags,
1061 }
1064 }
1067 }
1068 // make buffer valid on the device.
1071 COI_BUFFER_VALID,
1072 COI_BUFFER_NO_MOVE,
1077 }
1080 }
1083 }
1085 COI_PROCESS_SOURCE,
1086 COI_BUFFER_INVALID,
1087 COI_BUFFER_NO_MOVE,
1092 }
1095 }
1098 }
1099 // persistence algorithm requires target stack initialy to be nullified
1103 }
1115 }
1117 // Search through persistent stack buffers
1118 // for the top-of-stack buffer for this thread
1122 bool thread_specific_function_locals
1123 )
1124 {
1136 {
1140 {
1141 // For OpenMP shared function locals matching is done without
1142 // regard to thread id. But, we return the last match, which
1143 // corresponds to the outer stack.
1145 {
1148 }
1149 // For non-OpenMP shared function-local variables
1150 // the thread-id must match
1152 {
1155 }
1156 }
1157 }
1160 {
1163 }
1168 }
1170 // Search through persistent stack buffers
1171 // for the top-of-stack MIC buffer for this thread
1175 bool thread_specific_function_locals
1176 )
1177 {
1189 {
1193 {
1194 // For OpenMP shared function locals matching is done without
1195 // regard to thread id. But, we return the last match, which
1196 // corresponds to the outer stack.
1198 {
1201 }
1202 // For non-OpenMP shared function-local variables
1203 // the thread-id must match
1205 {
1208 }
1209 }
1210 }
1213 {
1216 }
1221 }
1224 {
1231 }
1234 }
1241 // Create a entry in targetptr table using device_ptr
1242 // as lookup for later recover the host pointer
1246 // Actually the base is a host pointer and cpu_addr is
1247 // device pointer. This is special case where the 2
1248 // address usage is reversed to enable using existing
1249 // PtrData structure instead of adding new fields.
1254 // Replace host pointer with device pointer
1257 }
1260 }
1261 }
1263 // For use_device_ptr and out find associated host ptr
1264 // and assign to host ptr
1269 }
1272 }
1276 }
1279 }
1280 }
1288 )
1289 {
1290 COIRESULT res;
1291 // To enable caching the CPU stack base address for stack variables
1293 // To properly deal with non-OpenMP threading and function-local variables
1294 // For OpenMP threading we support all function-locals in shared mode only
1298 // make a copy of variable descriptors
1308 }
1310 // dependencies
1320 }
1321 // copyin/copyout data length
1325 // First pass over variable descriptors
1326 // - Calculate size of the input and output non-pointer data
1327 // - Allocate buffers for input and output pointers
1340 }
1341 }
1343 // instead of m_vars[i].type.src we will use m_vars_extra[i].type_src
1349 }
1352 }
1353 // instead of m_vars[i].type.dst we will use m_vars_extra[i].type_dst
1360 }
1363 }
1364 }
1367 }
1375 }
1377 " type_src=%d, type_dstn=%d, direction=%d, "
1378 "alloc_if=%d, free_if=%d, align=%d, mic_offset=%d, flags=0x%x, "
1393 // If any varDesc flags bits set, show them
1396 }
1398 // preallocated implies targetptr
1400 // targetptr preallocated alloc_if(1) may not be used with
1401 // an in clause
1405 }
1407 }
1409 // array descriptor
1413 // debug dump
1419 }
1429 // flag is_arr_ptr_el is 1 only for var_descs generated
1430 // for c_data_ptr_array type
1433 }
1440 }
1444 {
1461 }
1466 }
1471 }
1476 }
1486 }
1497 }
1507 }
1517 }
1527 }
1537 }
1538 }
1541 }
1548 // In all uses later
1549 // VarDesc.size will have the length of the data to be
1550 // transferred
1551 // VarDesc.disp will have an offset from base
1561 }
1563 // array descriptor
1567 // debug dump
1570 // offset and length are derived from the array descriptor
1577 }
1578 // all necessary information about length and offset is
1579 // transferred in var descriptor. There is no need to send
1580 // array descriptor to the target side.
1582 }
1586 }
1589 // make sure that transfer size > 0
1593 }
1597 // find data associated with variable
1604 }
1607 // offset to base from the beginning of the buffer
1608 // memory
1612 }
1617 }
1618 }
1620 }
1627 // for non-static target destination defined as CEAN
1628 // expression we pass to target its size and dist
1632 }
1634 }
1640 }
1641 }
1646 // Static data is transferred either by omp target
1647 // update construct which passes zeros for
1648 // alloc_if and free_if or by always modifier.
1649 // Implicit openmp reference is transfered also
1650 // if its reference count is equal to 1
1655 }
1661 }
1662 }
1667 }
1668 }
1675 }
1677 // TODO: what should be done if var is not in
1678 // the table?
1681 }
1683 // For automatic variables data is transferred:
1684 // - if always modifier is used OR
1685 // - if alloc_if == 0 && free_if == 0 OR
1686 // - if reference count is 1
1692 }
1694 // save data for later use
1696 }
1697 }
1704 }
1710 // debug dump
1713 // send dope vector contents excluding base
1716 }
1728 }
1729 /* fallthru */
1736 // this var_desc is for stack buffer
1744 }
1753 new_el);
1754 }
1761 thread_specific_function_locals);
1762 }
1763 }
1767 // need to add or remove references for stack buffer at target
1770 }
1773 }
1774 /* fallthru */
1786 }
1789 // array descriptor
1793 // debug dump
1796 // offset and length are derived from the array descriptor
1804 }
1805 // all necessary information about length and offset is
1806 // transferred in var descriptor. There is no need to send
1807 // array descriptor to the target side.
1809 }
1811 // need to send DV to the device unless it is 'nocopy'
1817 // debug dump
1820 // for use_device_ptr don't need to change
1821 // OUT direction to IN direction
1824 }
1825 }
1827 // no displacement
1829 }
1831 // For "use_device_ptr" if direction is "in" then need to
1832 // find the associated device pointer and replace the host
1833 // pointer with device pointer. Also save the host pointer
1834 // to restore when "out" is encountered.
1835 // For "out" find the host pointer associated with the
1836 // device pointer and restore the host pointer
1840 }
1842 // c_data_ptr or c_string_ptr
1845 }
1852 // check that buffer length > 0
1858 }
1860 // base address
1863 // allocate buffer if we have no INTO and don't need
1864 // allocation for the ptr at target
1867 // for stack persistent objects ptr data is created
1868 // by var_desc with number 0.
1869 // Its ptr_data is stored at m_stack_ptr_data
1871 }
1877 }
1878 // add new entry
1880 ptr_data,
1886 alloc_disp,
1893 }
1897 }
1900 }
1903 // add buffer to the list of buffers that
1904 // are passed to dispatch call
1907 }
1910 // will send buffer address to device
1913 }
1917 // need to add reference for buffer
1919 }
1920 }
1924 // For omp target update variable is ignored
1925 // if it does not exist.
1930 }
1931 }
1933 // use existing association from pointer table
1935 base,
1939 error_if_not_found)) {
1941 }
1944 // make var nocopy if it does not exist
1947 c_parameter_nocopy;
1948 }
1949 }
1954 }
1955 }
1962 }
1965 // get buffers's address on the sink
1968 }
1971 }
1975 // need to decrement buffer reference on target
1977 }
1979 // offset to base from the beginning of the buffer
1980 // memory
1984 // copy other pointer properties to var descriptor
1987 }
1988 }
1991 base,
1996 }
2001 }
2002 }
2007 }
2008 // for TO transfer of stack buffer's variable
2017 }
2021 }
2022 // save data for later use
2025 // For automatic variables
2026 // data is transferred:
2027 // - if always modifier is used OR
2028 // - if alloc_if == 0 && free_if == 0 OR
2029 // - if reference count is 1
2036 c_parameter_nocopy;
2037 }
2038 }
2039 // for FROM transfer of global pointer variable
2040 // FROM transfer of stack buffer's variable
2041 // is treated at INTO branch
2044 // data is transferred only if
2045 // alloc_if == 0 && free_if == 0
2046 // or reference count is 1
2050 ptr_data &&
2052 {
2054 c_parameter_nocopy;
2055 }
2056 }
2057 }
2058 // save pointer data
2060 }
2067 }
2070 }
2084 }
2092 }
2097 }
2099 // if allocatable dope vector isn't allocated don't
2100 // transfer its data
2105 }
2114 // debug dump
2116 }
2121 }
2123 // size and displacement
2125 // offset and length are derived from the
2126 // array descriptor
2135 }
2139 }
2140 }
2141 }
2146 }
2149 }
2151 }
2153 // check that length >= 0
2158 }
2160 // base address
2164 // allocate buffer if we have no INTO and don't need
2165 // allocation for the ptr at target
2168 // add new entry
2170 ptr_data,
2176 alloc_disp,
2183 }
2187 // add buffer to the list of buffers
2188 // that are passed to dispatch call
2191 }
2193 // will send buffer address to device
2195 }
2198 // need to add reference for buffer
2200 }
2201 }
2205 // For omp target update variable is ignored
2206 // if it does not exist.
2211 }
2212 }
2214 // use existing association from pointer table
2216 base,
2220 error_if_not_found)) {
2222 }
2225 // make var nocopy if it does not exist
2228 c_parameter_nocopy;
2229 }
2230 }
2233 // need to update base in dope vector on device
2235 }
2236 }
2240 // data is transferred if
2241 // - if always modifier is used OR
2242 // - if alloc_if == 0 && free_if == 0 OR
2243 // - if reference count is 1
2249 c_parameter_nocopy;
2250 }
2251 }
2256 }
2259 // get buffers's address on the sink
2262 }
2265 }
2268 // need to decrement buffer reference on target
2270 }
2272 // offset to base from the beginning of the buffer
2273 // memory
2278 // copy other pointer properties to var descriptor
2281 }
2282 }
2285 base,
2290 }
2294 }
2296 // save pointer data
2298 }
2303 LIBOFFLOAD_ABORT;
2304 }
2307 }
2311 this_threads_cpu_stack_addr =
2314 }
2317 this_threads_cpu_stack_addr;
2318 }
2319 // if source is used at CPU save its offset and disp
2323 }
2325 // If "into" is define we need to do the similar work for it
2328 }
2349 }
2351 // array descriptor
2355 // debug dump
2358 // offset and length are derived from the array descriptor
2370 }
2371 }
2373 }
2381 size;
2382 // It's supposed that "into" size must be not less
2383 // than src size
2388 }
2394 // find data associated with variable
2398 }
2400 // offset to base from the beginning of the buffer
2401 // memory
2402 into_offset =
2405 }
2408 }
2410 }
2411 }
2417 // for non-static target destination defined as CEAN
2418 // expression we pass to target its size and dist
2421 }
2423 }
2427 // Static data is transferred either by omp target
2428 // update construct which passes zeros for
2429 // alloc_if and free_if or by always modifier.
2433 }
2434 }
2441 }
2443 // TODO: what should be done if var is not in
2444 // the table?
2447 }
2449 // For automatic variables data is transferred:
2450 // - if always modifier is used OR
2451 // - if alloc_if == 0 && free_if == 0 OR
2452 // - if reference count is 1
2458 }
2459 // save data for later use
2461 }
2462 }
2464 }
2472 // debug dump
2475 // send dope vector contents excluding base
2478 }
2492 // array descriptor
2496 // debug dump
2499 // offset and length are derived from the array descriptor
2510 }
2511 }
2513 }
2515 // need to send DV to the device unless it is 'nocopy'
2521 // debug dump
2525 }
2526 }
2534 size;
2535 // It's supposed that "into" size must be not less than
2536 // src size
2541 }
2546 // base address
2550 // allocate buffer
2552 // for stack persistent objects ptr data is created
2553 // by var_desc with number 0.
2554 // Its ptr_data is stored at m_stack_ptr_data
2556 }
2562 }
2563 // add new entry
2565 ptr_data,
2571 alloc_disp,
2578 }
2582 }
2585 }
2588 // add buffer to the list of buffers that
2589 // are passed to dispatch call
2592 }
2594 // will send buffer address to device
2596 }
2599 // need to add reference for buffer
2601 }
2602 }
2604 // use existing association from pointer table
2608 }
2610 }
2615 }
2618 // get buffers's address on the sink
2621 }
2624 }
2627 // need to decrement buffer reference on target
2629 }
2631 // copy other pointer properties to var descriptor
2634 }
2637 base,
2638 into_disp,
2642 }
2643 }
2649 }
2652 // for FROM transfer of stack buffer's variable
2659 size);
2661 }
2665 }
2666 // save data for later use
2668 // For automatic variables
2669 // data is transferred:
2670 // - if always modifier is used OR
2671 // - if alloc_if == 0 && free_if == 0 OR
2672 // - if reference count is 1
2679 c_parameter_nocopy;
2680 }
2681 }
2682 }
2683 // save pointer data
2685 }
2687 }
2709 // debug dump
2715 }
2720 }
2725 }
2726 // size and displacement
2728 // offset and length are derived from the array
2729 // descriptor
2736 }
2744 }
2745 }
2746 }
2747 }
2751 }
2754 }
2756 }
2761 src_is_for_mic) ?
2767 size;
2768 // It's supposed that "into" size must be not less
2769 // than src size
2774 }
2776 // base address
2779 // allocate buffer
2782 // add new entry
2784 ptr_data,
2790 alloc_disp,
2797 }
2800 // add buffer to the list of buffers
2801 // that are passed to dispatch call
2804 }
2806 // will send buffer address to device
2808 }
2811 // need to add reference for buffer
2813 }
2814 }
2816 // use existing association from pointer table
2820 }
2822 // need to update base in dope vector on device
2824 }
2829 }
2832 // get buffers's address on the sink
2835 }
2837 }
2840 // need to decrement buffer reference on target
2842 }
2844 // offset to base from the beginning of the buffer
2845 // memory
2846 into_offset =
2849 // copy other pointer properties to var descriptor
2852 }
2855 base,
2856 into_disp,
2857 size,
2860 }
2864 }
2866 // save pointer data
2868 }
2873 LIBOFFLOAD_ABORT;
2874 }
2875 // if into is used at CPU save its offset and disp
2879 }
2883 this_threads_cpu_stack_addr =
2886 thread_specific_function_locals);
2887 }
2890 this_threads_cpu_stack_addr;
2891 }
2894 }
2895 }
2898 }
2901 {
2904 // we can skip run functon call together with wait if offloaded
2905 // region is empty and there is no user defined non-pointer IN/OUT data
2907 // variable descriptors are sent as input data
2910 // timer data is sent as a part of the output data
2913 // max from input data and output data length
2915 m_out_datalen;
2917 // Misc data has the following layout
2918 // <Function Descriptor>
2919 // <Function Name>
2920 // <In/Out Data> (optional)
2921 //
2922 // We can transfer copyin/copyout data in misc/return data which can
2923 // be passed to run function call if its size does not exceed
2924 // COI_PIPELINE_MAX_IN_MISC_DATA_LEN. Otherwise we have to allocate
2925 // buffer for it.
2936 // use misc/return data for copyin/copyout
2939 }
2942 c_offload_host_alloc_data_buffer);
2944 // send/receive data using buffer
2946 COI_BUFFER_OPENCL,
2954 }
2956 }
2960 }
2961 }
2963 // initialize function descriptor
2965 misc_data_size);
2979 // append entry name
2981 }
2984 }
2987 {
3006 }
3008 target_is_static) {
3010 }
3026 }
3029 }
3033 }
3036 }
3037 }
3041 }
3044 }
3047 m_omp_async_last_event_type);
3048 }
3052 COIEVENT e,
3055 )
3056 {
3058 }
3060 // Callback function for asynchronous offloads
3062 COIEVENT e,
3065 )
3066 {
3076 }
3078 #ifndef TARGET_WINNT
3083 // Waiting for the last event
3087 }
3089 // Callback could be called when execution at host is completed.
3090 // Do nothing as engine data is destructed
3094 }
3101 // Completed offload has a signal
3106 }
3108 // Asynchronous offload can have both signal and stream. Need to
3109 // clean stream if any.
3115 }
3116 }
3122 }
3124 }
3125 // Asynchronous by stream
3130 // the stream was not created or was destroyed
3134 LIBOFFLOAD_ABORT;
3135 }
3139 }
3142 }
3143 // if the offload has both signal and stream we will complete
3144 // it as it has the signal. So we don't need to mark signal
3145 // as completed.
3150 }
3152 }
3153 }
3154 }
3155 }
3160 {
3162 }
3168 {
3174 func,
3178 }
3179 }
3184 _Offload_stream handle
3185 )
3186 {
3193 // Prepare in dependencies for stream
3196 }
3198 // wait for streams
3201 // some specific stream of the device
3206 // the stream was not created or was destroyed
3209 LIBOFFLOAD_ABORT;
3210 }
3213 // offload was completed by previous offload_wait pragma
3214 // or wait clause
3218 }
3224 }
3225 }
3231 }
3234 }
3235 // all streams of the device or over all devices
3244 }
3247 // get associated async task
3249 // offload was completed by offload_wait pragma or wait clause
3253 }
3257 task) {
3259 }
3260 }
3266 }
3269 }
3270 // no uncompleted streams
3272 }
3273 }
3276 // If offload is asynchronous we will not really wait for signals.
3277 // We will collect all waited events into m_p_in_dependencies vector
3278 // to be used in future calls to COI::Copy... API.
3290 // check previous offload with the stream_handle
3291 // to be noncompleted
3294 }
3295 }
3301 LIBOFFLOAD_ABORT;
3302 }
3305 }
3309 }
3312 }
3321 }
3334 }
3335 }
3337 m_num_in_dependencies;
3339 }
3340 // wait and do offload_finish for serial offload
3343 _Offload_stream stream_handle;
3351 LIBOFFLOAD_ABORT;
3352 }
3356 }
3357 // Need to mark signal as completed to prevent run condition
3358 // with the call to "offload_complete_task" for the same
3359 // signal.
3362 // Asynchronous offload can have both signal and stream.
3363 // Need to clean stream if any.
3370 }
3371 }
3377 }
3381 }
3382 }
3383 }
3385 }
3398 OffloadFlags offload_flags
3399 )
3400 {
3404 // define kind of wait if any;
3405 // there can be one of the following kind:
3406 // 1. c_offload_wait_signal for "offload_wait wait(signal)"
3407 // 2. c_offload_wait_stream for "offload_wait stream(stream)"
3408 // 3. c_offload_wait_all_streams for "offload_wait stream(0)"
3411 c_offload_wait_all_streams :
3412 c_offload_wait_stream;
3413 }
3419 }
3422 }
3426 }
3431 c_offload_init_func,
3432 "Offload function %s, is_empty=%d, #varDescs=%d, "
3436 // Breaks the norm of using OFFLOAD_DEBUG_TRACE to print the waits
3437 // since the number of waits is not fixed.
3445 }
3446 }
3449 }
3452 }
3454 }
3457 }
3458 // stream in wait is reported further in OFFLOAD_REPORT for waits
3461 c_offload_stream,
3463 }
3465 c_offload_signal,
3467 }
3471 c_offload_init_func,
3472 "Offload function %s, is_empty=%d, #varDescs=%d, "
3476 // Breaks the norm of using OFFLOAD_DEBUG_TRACE to print the waits
3477 // since the number of waits is not fixed.
3485 }
3487 }
3490 }
3493 }
3495 }
3498 }
3499 // stream in wait is reported further in OFFLOAD_REPORT for waits
3502 c_offload_stream,
3504 }
3506 c_offload_signal,
3508 }
3511 }
3517 waits :
3523 }
3527 // wait for dependencies to finish or set
3528 // m_num_in_dependencies and m_p_in_dependencies for asynchronous offload
3532 }
3534 // setup buffers
3538 }
3542 }
3544 // initiate send for pointers. Want to do it as early as possible.
3546 signal)) {
3549 }
3551 // setup misc data for run function
3555 }
3557 // gather copyin data into buffer
3561 }
3563 // Start the computation
3567 }
3569 // initiate receive for pointers
3574 }
3578 }
3580 // if there is a signal or stream save descriptor for the later use.
3581 // num_waits == -1 is for offload_wait and there is nothing to save
3586 }
3592 }
3595 LIBOFFLOAD_ABORT;
3596 }
3597 }
3598 // Register callback function "offload_complete_task" for all out
3599 // events or for all in events if there are no out transfers
3608 }
3611 }
3612 }
3614 // wait for the offload to finish.
3618 }
3622 }
3635 OffloadFlags offload_flags
3636 )
3637 {
3648 }
3649 }
3651 }
3654 bool is_traceback
3655 )
3656 {
3657 COIRESULT res;
3659 // wait for compute dependencies to become signaled
3665 // keep CPU busy
3668 }
3670 }
3673 }
3683 }
3685 }
3691 }
3693 }
3694 }
3696 // need to do scatter copyout data received from target after
3697 // completing in dependencies to get preallocated buffers.
3698 // If there are no preallocated buffers we will scatter_copyout_data
3699 // after completing out dependencies. In this case we dont need wait
3700 // in dependencies as they are already in DAG.
3704 }
3708 }
3709 }
3711 // wait for receive dependencies to become signaled
3716 // keep CPU busy
3719 }
3721 }
3724 }
3734 }
3736 }
3742 }
3744 }
3745 }
3749 }
3750 // destroy buffers
3751 {
3761 }
3763 }
3764 }
3765 }
3768 }
3771 {
3772 // release device in orsl
3777 // report stuff
3779 }
3782 {
3784 COIRESULT res;
3786 // check compute and receive dependencies
3790 }
3794 }
3797 }
3803 int64_t size
3804 )
3805 {
3818 }
3820 // Send pointer data if source or destination or both of them are
3821 // noncontiguous. There is guarantee that length of destination enough for
3822 // transferred data.
3830 COIEVENT *in_deps
3831 )
3832 {
3839 COIRESULT res;
3843 // If BufferWriteMultiD is defined we can set values of required arguments
3844 // and transfer noncontiguous data via call to the COI routine.
3882 arr_desc_src :
3910 }
3912 }
3914 }
3920 }
3922 // Set length_src and length_dst
3930 }
3932 // if event is defined we must multiplate it for all contiguous ranges
3933 // that will be Copied/Write.
3934 // Take in account that we already have 1 event.
3942 m_in_deps =
3944 m_in_deps_total--;
3945 }
3947 // consequently get contiguous ranges,
3948 // define corresponded destination offset and send data
3951 // ranges are over
3954 }
3957 noncont_num++;
3958 }
3964 // source ranges are over - nothing to send
3966 }
3967 }
3970 }
3973 }
3975 }
3977 // if source is contiguous or its contiguous range is greater
3978 // than destination one
3980 }
3989 // destination ranges are over
3992 }
3993 }
3994 // into is contiguous.
3997 }
3999 }
4000 // same as source
4004 }
4005 }
4007 // if destination is contiguous or its contiguous range is greater
4008 // than source one
4010 }
4013 }
4016 }
4024 send_size,
4025 COI_COPY_UNSPECIFIED,
4026 m_num_in_dependencies,
4027 m_p_in_dependencies,
4028 event);
4033 }
4035 }
4036 }
4046 send_size,
4047 COI_COPY_UNSPECIFIED,
4048 m_num_in_dependencies,
4049 m_p_in_dependencies,
4050 event);
4055 }
4057 }
4058 }
4060 }
4063 }
4066 {
4071 COIRESULT res;
4075 // For offload_transfer and offload with empty body without signal:
4076 // - if there is only one buffer copy - send data synchronously
4077 // - if there are multiple buffer copy and
4078 // __offload_parallel_copy is false - send data synchronously
4079 // - if there are multiple buffer copy and
4080 // __offload_parallel_copy is true - send data asynchronously
4081 // It concerns only big size data - greater than __offload_use_async_buffer_write.
4082 // Data of size less than __offload_use_async_buffer_write are sent synchronously.
4083 // Synchronous transfer results in better performance in COI.
4084 // __offload_parallel_copy is false by default but can be changed
4085 // via environment variable OFFLOAD_PARALLEL_COPY
4097 big_size_count++;
4098 }
4111 big_size_count++;
4115 }
4116 }
4117 }
4120 }
4121 }
4123 // Initiate send for pointer data
4131 }
4143 is_async ||
4163 }
4164 }
4174 COI_COPY_UNSPECIFIED,
4175 m_num_in_dependencies,
4176 m_p_in_dependencies,
4177 event);
4182 }
4184 }
4185 }
4195 COI_COPY_UNSPECIFIED,
4196 m_num_in_dependencies,
4197 m_p_in_dependencies,
4198 event);
4203 }
4205 }
4206 }
4208 }
4212 // If use_device_ptr no data needs to be sent
4215 }
4225 is_async ||
4244 }
4254 COI_COPY_UNSPECIFIED,
4255 m_num_in_dependencies,
4256 m_p_in_dependencies,
4257 event);
4262 }
4264 }
4265 }
4275 COI_COPY_UNSPECIFIED,
4276 m_num_in_dependencies,
4277 m_p_in_dependencies,
4278 event);
4283 }
4285 }
4286 }
4289 }
4303 is_async ||
4314 }
4324 COI_COPY_UNSPECIFIED,
4325 m_num_in_dependencies,
4326 m_p_in_dependencies,
4327 event);
4332 }
4334 }
4335 }
4345 COI_COPY_UNSPECIFIED,
4346 m_num_in_dependencies,
4347 m_p_in_dependencies,
4348 event);
4353 }
4355 }
4356 }
4358 }
4377 is_async ||
4387 }
4398 COI_COPY_UNSPECIFIED,
4399 m_num_in_dependencies,
4400 m_p_in_dependencies,
4401 event);
4406 }
4408 }
4409 }
4419 COI_COPY_UNSPECIFIED,
4420 m_num_in_dependencies,
4421 m_p_in_dependencies,
4422 event);
4427 }
4429 }
4430 }
4433 }
4438 }
4441 }
4442 // alloc field isn't used at target.
4443 // We can reuse it for offset of array pointers.
4446 }
4447 }
4448 // list of out events created while send_pointer_data now became input
4449 // dependencies for runfunction (or Read transfers from target if
4450 // runfunction is absent)
4452 m_num_in_dependencies;
4457 }
4461 c_offload_sent_pointer_data,
4463 ptr_sent);
4466 }
4469 {
4473 COIMAPINSTANCE map_inst;
4476 // init marshaller
4479 c_offload_host_map_in_data_buffer);
4482 COI_MAP_WRITE_ENTIRE_BUFFER,
4489 }
4491 }
4492 }
4495 }
4497 // send variable descriptors
4501 // init marshaller
4504 // Gather copy data into buffer
4514 }
4521 }
4522 // send sink address to the target
4526 }
4541 // offset and length are derived from the array
4542 // descriptor
4549 }
4553 }
4560 // send dope vector excluding base
4564 }
4568 // send to target addresses of obsolete
4569 // stacks to be released
4580 }
4581 }
4587 }
4592 }
4593 }
4597 }
4601 c_offload_host_unmap_in_data_buffer);
4607 }
4609 }
4610 }
4611 }
4620 }
4623 {
4641 }
4642 }
4644 // dispatch task
4645 COIRESULT res;
4646 COIEVENT event;
4649 m_compute_buffers,
4652 m_num_in_dependencies,
4653 m_p_in_dependencies,
4660 }
4662 }
4666 }
4671 }
4674 }
4676 // receive pointer data if source or destination or both of them are
4677 // noncontiguous. There is guarantee that length of destination enough for
4678 // transferred data.
4681 COIBUFFER dst_buf,
4685 COIEVENT *in_deps
4686 )
4687 {
4694 COIRESULT res;
4705 // If BufferReadMultiD is defined we can set values of required arguments
4706 // and transfer noncontiguous data via call to the COI routine.
4736 el_size);
4759 }
4761 }
4763 }
4767 }
4769 // Set length_src and length_dst
4776 }
4778 // if event is defined we must multiplate for all contiguous intervals
4779 // that will be Copied/Read.
4780 // Take in account that we already have 1 event.
4788 m_out_deps =
4790 m_out_deps_total--;
4791 }
4793 // consequently get contiguous ranges,
4794 // define corresponded destination offset and receive data
4797 // ranges are over
4800 }
4803 noncont_num++;
4804 }
4810 // source ranges are over - nothing to send
4812 }
4813 }
4816 }
4819 }
4821 }
4823 // if source is contiguous or its contiguous range is greater
4824 // than destination one
4826 }
4830 // get destination offset
4836 // destination ranges are over
4839 }
4840 }
4841 // destination is contiguous.
4844 }
4846 }
4847 // same as source
4851 }
4852 }
4854 // if destination is contiguous or its contiguous range is greater
4855 // than source one
4857 }
4860 }
4863 }
4866 dst_buf,
4871 receive_size,
4872 COI_COPY_UNSPECIFIED,
4873 m_num_in_dependencies,
4874 m_p_in_dependencies,
4875 event);
4880 }
4882 }
4883 }
4890 receive_size,
4891 COI_COPY_UNSPECIFIED,
4892 m_num_in_dependencies,
4893 m_p_in_dependencies,
4894 event);
4899 }
4901 }
4902 }
4904 }
4907 }
4911 {
4916 COIRESULT res;
4918 // For offload_transfer and offload with empty body without signal:
4919 // - if there is only one buffer copy - get data synchronously
4920 // - if there are multiple buffer copy and
4921 // __offload_parallel_copy is false - get data synchronously
4922 // - if there are multiple buffer copy
4923 // and __offload_parallel_copy is true - get data asynchronously
4924 // It concerns only data with size greater than __offload_use_async_buffer_read.
4925 // Data of size less than __offload_use_async_buffer_read are received synchronously.
4926 // Synchronous transfer results in better performance in COI.
4927 // __offload_parallel_copy is false by default but can be changed
4928 // via environment variable OFFLOAD_PARALLEL_COPY
4935 // preallocated OUT only at second run
4938 }
4945 big_size_count++;
4946 }
4959 big_size_count++;
4963 }
4964 }
4965 }
4968 }
4969 }
4976 // Nothing to receive if use_device_ptr
4983 }
4984 // At first run don't receive by preallocated target pointer as the
4985 //pointer value will be ready later after call to scatter_copyout_data
4988 // need one more call to OffloadDescriptor::receive_pointer_data
4991 }
4993 }
5005 is_async ||
5018 }
5024 }
5025 }
5033 }
5041 }
5044 dst_buf,
5050 COI_COPY_UNSPECIFIED,
5051 m_num_in_dependencies,
5052 m_p_in_dependencies,
5053 event);
5058 }
5060 }
5061 }
5069 COI_COPY_UNSPECIFIED,
5070 m_num_in_dependencies,
5071 m_p_in_dependencies,
5072 event);
5077 }
5079 }
5080 }
5082 }
5100 is_async ||
5118 }
5121 }
5127 }
5134 }
5137 }
5151 }
5154 }
5162 }
5165 dst_buf,
5167 dst_offset,
5171 COI_COPY_UNSPECIFIED,
5172 m_num_in_dependencies,
5173 m_p_in_dependencies,
5174 event);
5179 }
5181 }
5182 }
5190 COI_COPY_UNSPECIFIED,
5191 m_num_in_dependencies,
5192 m_p_in_dependencies,
5193 event);
5198 }
5200 }
5201 }
5203 }
5205 }
5209 }
5213 }
5214 // destroy buffers for obsolete stacks
5222 }
5224 }
5226 // remove association for automatic variables
5232 }
5235 }
5237 }
5244 }
5246 }
5247 }
5248 }
5250 // destroy buffers
5256 }
5260 // destroy buffers
5263 }
5266 }
5270 // remove association from map
5273 }
5276 }
5277 }
5278 }
5285 // destroy buffers
5288 }
5291 }
5295 // remove association from map
5298 }
5301 }
5302 }
5303 }
5304 }
5305 }
5309 }
5312 m_num_in_dependencies;
5317 c_offload_received_pointer_data,
5319 ptr_received);
5322 }
5325 {
5330 // total size that need to be transferred from target to host
5331 COIMAPINSTANCE map_inst;
5332 COIRESULT res;
5335 // output data buffer
5338 c_offload_host_map_out_data_buffer);
5348 }
5350 }
5351 }
5354 }
5356 // get timing data
5360 // initialize output marshaller
5378 // array descriptor
5385 }
5387 // get pointer to target memory
5390 // add new entry
5392 ptr_data,
5393 ptr_value,
5398 alloc_disp,
5404 }
5410 }
5413 }
5416 }
5434 }
5440 }
5441 }
5448 }
5453 }
5454 }
5458 }
5462 c_offload_host_unmap_out_data_buffer);
5469 }
5471 }
5472 }
5473 }
5480 }
5488 CeanReadRanges* &ptr_ranges
5489 )
5490 {
5495 }
5501 }
5502 }
5505 {
5530 // "pointers_number" for total number of transferred pointers.
5531 // For each of them we create new var_desc and put it at the bottom
5532 // of the var_desc's array
5537 // 2. prepare memory for new var_descs
5542 m_vars_extra =
5546 m_in_deps =
5550 m_out_deps =
5555 // 3. Prepare for reading new var_desc's fields
5556 // EXTENT START
5567 }
5568 }
5571 }
5574 }
5576 // EXTENT ELEMENTS NUMBER
5588 }
5589 }
5592 }
5595 }
5597 // ALLOC_IF
5608 }
5609 }
5612 }
5614 // FREE_IF
5625 }
5626 }
5629 }
5631 // ALIGN
5643 }
5644 }
5647 }
5649 // 3.1 INTO
5660 }
5661 }
5663 // 3.2 INTO_START
5675 }
5676 }
5679 }
5682 }
5684 // 3.3 INTO_ELEMENTS
5696 }
5697 }
5700 }
5703 }
5705 // alloc_start
5718 }
5719 }
5722 }
5725 }
5727 // alloc_elem
5739 }
5740 }
5743 }
5746 }
5751 // Get new values
5752 // type_src, type_dst
5759 // Get ptr val
5762 }
5765 }
5767 // !!! If we got error at phase of reading - it's an internal
5768 // !!! error, as we must detect mismatch before
5770 // Get into val
5774 LIBOFFLOAD_ABORT;
5775 }
5778 }
5779 }
5781 // Get other components of the clause
5784 LIBOFFLOAD_ABORT;
5785 }
5789 LIBOFFLOAD_ABORT;
5790 }
5793 LIBOFFLOAD_ABORT;
5794 }
5797 LIBOFFLOAD_ABORT;
5798 }
5801 LIBOFFLOAD_ABORT;
5802 }
5805 LIBOFFLOAD_ABORT;
5806 }
5809 LIBOFFLOAD_ABORT;
5810 }
5813 LIBOFFLOAD_ABORT;
5814 }
5818 LIBOFFLOAD_ABORT;
5819 }
5839 }
5840 }
5852 type_src;
5855 }
5856 }
5867 type_src;
5868 }
5871 }
5880 }
5883 }
5894 }
5895 // count and alloc fields are useless at target. They can be reused
5896 // for pointer arrays.
5900 }
5902 // Gets in dependencies of the previous offload via the stream "m_stream".
5903 // Out argument in_deps_amount - address of amount of the dependencies
5904 // Out argument in_deps - address of array of dependencies.
5905 // Description of the dependencies scheme for streams :
5906 // ----------------------------------------------------
5907 // Every offload forms DAG consisted of 3 nodes:
5908 // for in-transfers, runfunction and out-transfers.
5909 // Every node has in-dependencies and out-dependencies
5910 // Out-dependencies of previous node forms in-dependencies of current node.
5911 // In-dependencies of 1-st node (of in-transfers) without streams is equal
5912 // to NULL. For streams in-dependencies of 1-st node is equal to list of out
5913 // dependencies of last node of previous offload via this stream.
5914 // So we can say that DAGs of 2 consequent offloads via the same stream are
5915 // connected by the way described above.
5918 COIEVENT* &in_deps
5919 )
5920 {
5926 LIBOFFLOAD_ABORT;
5927 }
5930 // if it's the first offload in the stream
5933 }
5934 // if last offload has out-tranfers
5938 }
5939 // last offload only sends pointer data or run function or both of them
5940 // and has no out-transfers
5944 }
5945 }
5946 }
5949 {
5958 }
5963 }
5968 }
5970 // destroy thread key
5972 }
5974 // unload COI library
5977 }
5980 }
5988 deviceLcpuList &device_cpu_list
5989 )
5990 {
5991 // Value is composed of comma separated physical device index
5992 // optionally qualified by logical CPU subset, e.g. 0[60,70-80]
6004 // CPU subset specified
6013 // A single CPU specified
6016 // Convert cpu string to an int
6026 }
6030 }
6033 // A range of CPUs specified
6035 // Convert cpu string to an int
6044 range_start);
6051 // Convert cpu string to an int
6061 range_end);
6069 {
6072 i);
6074 }
6078 }
6079 }
6080 }
6086 }
6087 }
6089 // Error: expected , or - or ]
6094 }
6096 // Point to next device specification
6101 // Skip the comma after a device specification
6102 str++;
6103 }
6105 // CPU subset not specified
6106 // Point to next device specification
6109 // No more device specifications
6111 }
6113 // Convert device string to an int
6120 }
6126 }
6128 // Save the specified physical device and cpu mask
6130 num_devices_specified++;
6134 }
6139 }
6142 {
6143 COIRESULT res;
6145 deviceLcpuList device_cpu_list;
6148 // initialize trace
6154 }
6155 }
6169 offload_report_level);
6170 }
6173 offload_report_envname);
6174 }
6175 }
6178 offload_report_envname);
6179 }
6180 }
6187 timer_enabled);
6188 }
6189 }
6191 // initialize COI
6194 }
6196 // Process OFFLOAD_NODES, specification of physical MICs available
6200 // Pass env var on to COI
6210 // value is composed of comma separated physical device indexes
6218 // count this MIC
6219 num_mics++;
6220 }
6223 }
6226 }
6228 // get number of devices installed in the system
6232 }
6236 }
6238 // Determine devices & cpus that can be used for offloading
6243 mic_engines_total =
6248 mic_engines_total);
6249 }
6253 }
6254 }
6256 // No need to continue since no offload devices
6258 }
6259 }
6262 }
6264 // Fallback to using all available devices and all CPUs on each
6269 COIENGINE engine;
6274 mic_engines_total++;
6275 }
6276 }
6277 }
6279 // no need to continue if there are no devices to offload to
6282 }
6284 // Initialize indexes for available devices
6290 deviceIterator++)
6291 {
6307 }
6310 }
6311 l_idx++;
6312 }
6314 // Get DMA channel count to pass it to COI
6322 mic_dma_channel_count);
6323 }
6327 }
6328 }
6331 }
6333 // Set COI_HOST_THREAD_AFFINITY if OFFLOAD_HOST_THREAD_AFFINITY is set.
6334 // Use putenv instead of setenv as Windows has no setenv.
6335 // Note: putenv requires its argument can't be freed or modified.
6336 // So no free after call to putenv or elsewhere.
6349 }
6352 }
6354 // library search path for KNC device binaries
6362 }
6365 }
6367 // library search path for KNL device binaries
6375 }
6378 }
6380 // memory size reserved for COI buffers
6389 mic_buffer_size);
6390 }
6393 }
6394 }
6397 }
6399 // memory size reserved for 4K pages for COI buffers
6408 mic_4k_buffer_size);
6409 }
6412 }
6413 }
6416 }
6418 // memory size reserved for 2M pages for COI buffers
6427 mic_2m_buffer_size);
6428 }
6432 }
6433 }
6436 }
6438 // determine stacksize for the pipeline on the device
6447 mic_stack_size);
6448 }
6451 }
6452 }
6455 }
6457 // proxy I/O
6465 mic_proxy_io);
6466 }
6469 }
6470 }
6473 }
6483 mic_proxy_fs_root);
6484 }
6487 }
6489 // Prepare environment for the target process using the following
6490 // rules
6491 // - If MIC_ENV_PREFIX is set then any environment variable on the
6492 // host which has that prefix are copied to the device without
6493 // the prefix.
6494 // All other host environment variables are ignored.
6495 // - If MIC_ENV_PREFIX is not set or if MIC_ENV_PREFIX="" then host
6496 // environment is duplicated.
6508 }
6509 }
6510 }
6513 }
6515 // create key for thread data
6519 }
6521 // cpu frequency
6533 __offload_use_2mb_buffers);
6534 }
6537 mic_use_2mb_buffers_envname);
6538 }
6539 }
6542 }
6553 __offload_use_async_buffer_write);
6554 }
6555 }
6558 mic_use_async_buffer_write_envname);
6559 }
6570 __offload_use_async_buffer_read);
6571 }
6572 }
6575 mic_use_async_buffer_read_envname);
6576 }
6578 // mic initialization type
6586 "A MIC device will be initialized "
6588 }
6592 "All MIC devices will be initialized "
6594 }
6598 "All MIC devices will be initialized "
6600 }
6603 }
6604 }
6607 }
6609 // active wait
6619 __offload_active_wait);
6620 }
6623 offload_active_wait_envname);
6624 }
6625 }
6628 }
6630 // always wait
6640 __offload_active_wait);
6641 }
6644 offload_always_wait_envname);
6645 }
6646 }
6649 }
6651 // omp device num
6660 __omp_device_num);
6661 }
6664 omp_device_num_envname);
6665 }
6666 }
6669 }
6671 // parallel copy of offload_transfer
6681 __offload_parallel_copy);
6682 }
6685 parallel_copy_envname);
6686 }
6687 }
6690 }
6692 // use COI interface for noncontiguous arrays transfer
6702 __offload_use_coi_noncontiguous_transfer);
6703 }
6706 use_coi_noncontiguous_transfer_envname);
6707 }
6708 }
6711 use_coi_noncontiguous_transfer_envname);
6712 }
6716 // init ORSL
6718 }
6721 {
6722 // do one time intialization
6726 // offload is available if COI is available and the number of devices > 0
6729 // register pending libraries if there are any
6735 // Register library in COI
6739 // add lib to all engines
6742 }
6743 }
6747 }
6750 }
6753 {
6756 // decode image
6760 // determine image type
6763 }
6766 {
6772 // decode image
6781 // The origin name is the name of the file on the host
6782 // this is used by Vtune, since it is a fat binary we
6783 // use the host file name of the fat binary.
6784 // Driver prepends the host file name ending with "?"
6785 // to the image->data name so need to extract the string
6786 // name format: <mic_name>?<origin>
6788 // Get <mic_name>
6792 fat_name++;
6793 i++;
6794 }
6796 // Remove the host file name by inserting end of string marker
6799 // Get <host_name>
6801 // The string following "?" is the name of the host file name.
6802 fat_name++;
6806 fat_name++;
6807 i++;
6808 }
6810 }
6812 // Windows current does not have host name
6815 }
6817 // our actions depend on the image type
6822 // Each offload application is supposed to have only one target
6823 // image representing target executable.
6824 // No thread synchronization is required here as the initialization
6825 // code is always executed in a single thread.
6829 }
6832 // Registration code for execs is always called from the context
6833 // of main and thus we can safely call any function here,
6834 // including LoadLibrary API on windows. This is the place where
6835 // we do the offload library initialization.
6837 // initialize engine if init_type is on_start
6841 }
6842 }
6843 }
6847 {
6850 // We add the library to a list of pending libraries
6856 // If __target_exe is set, then main has started running
6857 // If not main, then we can't do anything useful here
6858 // because this registration code is called from DllMain
6859 // context (on windows).
6861 // There is no need to delay loading the library
6863 // Couldn't validate library as a fat offload library
6866 }
6867 }
6869 }
6872 // something is definitely wrong, issue an error and exit
6875 }
6876 }
6878 // When dlopen is used dlclose may happen after the COI process
6879 // is destroyed. In which case images cannot be unloaded and should
6880 // be skipped. So track if coi has been unloaded.
6884 {
6885 // Target image is packed as follows:
6886 // 8 bytes - size of the target binary
6887 // null-terminated string - binary name
6888 // <size> bytes - binary contents
6894 // decode image
6898 // our actions depend on the image type
6901 // We are executing exec's desctructors.
6902 // It is time to do a library cleanup.
6905 }
6909 // Do not unload the MYO library if it loaded in dll.
6911 {
6912 #ifdef MYO_SUPPORT
6917 }
6918 }
6922 }
6924 }
6925 }
6928 {
6930 }
6932 // Runtime trace interface for user programs
6935 {
6937 }
6939 // User-visible offload API
6942 {
6945 }
6948 {
6950 }
6953 {
6955 }
6958 {
6961 // check index value
6964 LIBOFFLOAD_ABORT;
6965 }
6969 // find associated async task
6974 LIBOFFLOAD_ABORT;
6975 }
6976 // if signal is removed by wait completing
6979 }
6981 }
6984 {
6988 }
6989 }
6999 )
7000 {
7003 // check target value
7006 LIBOFFLOAD_ABORT;
7007 }
7010 // find existing association in pointer table
7015 }
7026 }
7027 }
7034 }
7039 )
7040 {
7043 // check target value
7046 LIBOFFLOAD_ABORT;
7047 }
7050 // Create new stream and get its handle
7055 }
7057 // create pipeline associated with the new stream
7061 }
7065 _Offload_stream handle // stream to destroy
7066 )
7067 {
7070 LIBOFFLOAD_ABORT;
7071 }
7072 // check target value
7075 LIBOFFLOAD_ABORT;
7076 }
7082 }
7085 _Offload_stream handle // stream to destroy
7086 )
7087 {
7093 LIBOFFLOAD_ABORT;
7094 }
7097 // the stream was not created or was destroyed
7100 LIBOFFLOAD_ABORT;
7101 }
7108 }
7111 {
7114 LIBOFFLOAD_ABORT;
7115 }
7116 // check device index value
7119 LIBOFFLOAD_ABORT;
7120 }
7123 }
7124 // get stream
7130 // the stream was not created or was destroyed
7133 LIBOFFLOAD_ABORT;
7134 }
7139 LIBOFFLOAD_ABORT;
7140 }
7141 // find associated async task
7144 // offload was completed by offload_wait pragma or wait clause
7147 }
7149 }
7150 // zero handler is for all streams at the device
7158 }
7159 // find associated async task
7162 // offload was completed by offload_wait pragma or wait clause
7165 }
7166 // if even one stream is not completed result is false
7169 }
7170 }
7171 // no uncompleted streams
7173 }
7174 }
7177 {
7182 LIBOFFLOAD_ABORT;
7183 }
7187 // the stream was not created or was destroyed
7190 LIBOFFLOAD_ABORT;
7191 }
7193 }
7196 }
7197 // Use 0 for device index as _Offload_stream_completed
7198 // ignores this value while defining streams completion
7200 }
7203 {
7206 LIBOFFLOAD_ABORT;
7207 }
7208 // check index value
7211 LIBOFFLOAD_ABORT;
7212 }
7215 }
7220 {
7225 }
7226 // find associated async task
7229 // offload was completed by offload_wait pragma or wait clause
7232 }
7233 // if even one stream is not completed result is false
7236 }
7237 }
7238 // no uncompleted streams
7240 }
7242 // IDB support
7251 {
7252 }
7254 {
7255 }