| blob | e52019dfb28e7733cba65fc506d160ccb27e4fba |
1 /*
2 Copyright (c) 2014-2015 Intel Corporation. All Rights Reserved.
4 Redistribution and use in source and binary forms, with or without
5 modification, are permitted provided that the following conditions
6 are met:
8 * Redistributions of source code must retain the above copyright
9 notice, this list of conditions and the following disclaimer.
10 * Redistributions in binary form must reproduce the above copyright
11 notice, this list of conditions and the following disclaimer in the
12 documentation and/or other materials provided with the distribution.
13 * Neither the name of Intel Corporation nor the names of its
14 contributors may be used to endorse or promote products derived
15 from this software without specific prior written permission.
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 */
31 // Forward declaration as the following 2 functions are declared as friend
32 // in offload_engine.h.
33 // CLANG does not like static to been after friend declaration.
38 #ifdef MYO_SUPPORT
40 #endif
42 #include <malloc.h>
43 #ifndef TARGET_WINNT
44 #include <alloca.h>
45 #include <elf.h>
47 #include <errno.h>
48 #include <fcntl.h>
49 #include <stdlib.h>
50 #include <string.h>
51 #include <sys/stat.h>
52 #include <sys/types.h>
53 #include <sys/stat.h>
55 #include <algorithm>
56 #include <bitset>
58 #if defined(HOST_WINNT)
60 #else
62 #endif
64 #define GET_OFFLOAD_NUMBER(timer_data) \
65 timer_data? timer_data->offload_number : 0
70 #ifdef TARGET_WINNT
71 // Windows does not support imports from libraries without actually
72 // including them as dependence. We don't want to include in the
73 // dependence since is it used only for Fortran when traceback is enabled.
74 // Chose to implement it with GetProcAddress.
75 #define FORTRAN_TRACE_BACK win_for__continue_traceback
77 {
78 HINSTANCE hDLL;
87 }
92 }
93 }
97 }
99 }
103 #define FORTRAN_TRACE_BACK for__continue_traceback
105 // for__continue_traceback is provided as a dummy to resolve link time symbols
106 // for C/C++ programs. For Fortran the actual fortran library function in
107 // libifcore.so is used.
108 #pragma weak for__continue_traceback
110 {
114 }
118 #ifdef TARGET_WINNT
119 // Small subset of ELF declarations for Windows which is needed to compile
120 // this file. ELF header is used to understand what binary type is contained
121 // in the target image - shared library or executable.
128 #define EI_NIDENT 16
130 #define ET_EXEC 2
131 #define ET_DYN 3
134 {
136 Elf64_Half e_type;
137 Elf64_Half e_machine;
138 Elf64_Word e_version;
139 Elf64_Addr e_entry;
140 Elf64_Off e_phoff;
141 Elf64_Off e_shoff;
142 Elf64_Word e_flags;
143 Elf64_Half e_ehsize;
144 Elf64_Half e_phentsize;
145 Elf64_Half e_phnum;
146 Elf64_Half e_shentsize;
147 Elf64_Half e_shnum;
148 Elf64_Half e_shstrndx;
152 // Host console and file logging
161 // location of offload_main executable
162 // To be used if the main application has no offload and is not built
163 // with -offload but dynamic library linked in has offload pragma
166 // DMA channel count used by COI and set via
167 // OFFLOAD_DMA_CHANNEL_COUNT environment variable
170 // Trace information
175 "INOUT"
176 };
195 "c_string_ptr_array"
196 };
200 pthread_key_t mic_thread_key;
201 MicEnvVar mic_env_vars;
204 // MIC_STACKSIZE
207 // MIC_BUFFERSIZE
210 // Preallocated 4K page memory size for buffers on MIC
213 // Preallocated 2M page memory size for buffers on MIC
217 // MIC_LD_LIBRARY_PATH
220 // MIC_PROXY_IO
223 // MIC_PROXY_FS_ROOT
226 // Threshold for creating buffers with large pages. Buffer is created
227 // with large pages hint if its size exceeds the threshold value.
228 // By default large pages are disabled right now (by setting default
229 // value for threshold to MAX) due to HSD 4114629.
242 // device initialization type
246 // active wait
250 // OMP_DEFAULT_DEVICE
254 //OFFLOAD_PARALLEL_COPY
258 //Use COI interface for noncontiguous transfer if it exists.
263 // The list of pending target libraries
269 // Target executable
272 // Print readable offload flags
275 OffloadFlags offload_flags
276 )
277 {
278 // Sized big enough for all flag names
286 }
290 }
294 }
295 }
297 // Print readable varDesc flags
300 varDescFlags offload_flags
301 )
302 {
303 // SIzed big enough for all flag names
311 }
316 }
321 }
326 }
331 }
336 }
341 }
346 }
351 }
356 }
361 }
366 }
371 }
375 }
376 }
379 {
383 }
386 }
394 }
399 }
401 }
404 }
406 }
409 {
410 // special case for the 'process died' error
413 }
419 }
420 /* fallthru */
441 }
442 }
445 }
448 {
461 }
462 }
464 // is_targetptr == 0 && is_prealloc == 0 - allocation of pointer data;
465 // is_targetptr == 1 && is_prealloc == 0 - allocation of target memory:
466 // allocate memory at target; use its value as base in target table.
467 // is_targetptr == 1 && is_prealloc == 1 - use preallocated target memory:
468 // base - is address at target of preallocated memory; use its value as
469 // base in target table.
480 bool pin
481 )
482 {
483 // total length of base
486 COIBUFFER targptr_buf;
487 COIRESULT res;
491 // create buffer with large pages if data length exceeds
492 // large page threshold
495 }
496 // Allocate memory at target for targetptr without preallocated as we need
497 // its address as base argument in call to m_device.insert_ptr_data
501 length,
502 COI_BUFFER_NORMAL,
503 buffer_flags,
511 }
514 }
516 }
523 }
526 }
528 }
529 }
535 // add new entry
540 // if ptr_data is found just need to check it for overlapping
544 }
546 // If association is not found we must create it.
551 }
559 // align should be a power of 2
562 // offset within mic_buffer. Can do offset optimization
563 // only when source address alignment satisfies requested
564 // alignment on the target (cq172736).
568 }
569 }
571 // buffer size and flags
574 // For targetptr there is no CPU buffer
576 // create CPU buffer
579 c_offload_create_buf_host,
580 "Creating buffer from source memory %p, "
583 // result is not checked because we can continue without cpu
584 // buffer. In this case we will use COIBufferRead/Write
585 // instead of COIBufferCopy.
588 COI_BUFFER_NORMAL,
590 base,
594 }
596 // create MIC buffer
600 c_offload_create_buf_mic,
601 "Creating buffer from sink memory: size %lld, offset %d, "
605 COI_BUFFER_NORMAL,
606 COI_SINK_MEMORY,
607 base,
614 }
617 }
620 }
621 }
624 }
628 c_offload_create_buf_mic,
629 "Creating buffer for sink: size %lld, offset %d, "
633 COI_BUFFER_NORMAL,
634 buffer_flags,
642 }
645 }
648 }
649 }
652 // make buffer valid on the device.
655 COI_BUFFER_VALID,
656 COI_BUFFER_NO_MOVE,
661 }
664 }
667 }
670 COI_PROCESS_SOURCE,
671 COI_BUFFER_INVALID,
672 COI_BUFFER_NO_MOVE,
677 }
680 }
683 }
684 }
685 }
688 }
697 // This is not a new entry. Make sure that provided address range fits
698 // into existing one.
705 }
707 // if the entry is associated with static data it may not have buffers
708 // created because they are created on demand.
711 }
712 }
715 }
723 bool report_error
724 )
725 {
726 // total length of base
733 // find existing association in pointer table
741 }
744 }
750 // make sure that provided address range fits into existing one
758 }
763 }
765 // if the entry is associated with static data it may not have buffers
766 // created because they are created on demand.
769 }
772 }
775 {
784 COI_BUFFER_NORMAL,
794 }
796 }
797 }
805 COI_BUFFER_NORMAL,
806 COI_SINK_MEMORY,
815 }
817 }
818 }
821 }
824 {
831 }
834 }
836 }
837 }
839 }
842 COIBUFFER targ_buf,
843 uint64_t size
844 )
845 {
849 COIRESULT res;
853 targ_buf,
855 ptr,
856 size,
857 COI_COPY_UNSPECIFIED,
864 }
866 }
868 }
876 {
892 // this stack data must be destroyed
896 erase++;
897 }
898 }
901 // this stack data must be destroyed
904 erase++;
906 }
908 // stack data is reused
911 // all obsolete stack sections must be erased from the list
914 m_in_datalen +=
916 }
920 }
921 }
925 }
926 }
929 // all obsolete stack sections must be erased from the list
932 }
933 // new stack table is created
935 // create MIC buffer
936 COIRESULT res;
939 // create buffer with large pages if data length exceeds
940 // large page threshold
943 }
945 COI_BUFFER_NORMAL,
946 buffer_flags,
954 }
957 }
959 }
960 // make buffer valid on the device.
963 COI_BUFFER_VALID,
964 COI_BUFFER_NO_MOVE,
969 }
972 }
974 }
976 COI_PROCESS_SOURCE,
977 COI_BUFFER_INVALID,
978 COI_BUFFER_NO_MOVE,
983 }
986 }
988 }
989 // persistence algorithm requires target stack initialy to be nullified
992 }
1002 }
1010 )
1011 {
1012 COIRESULT res;
1016 // make a copy of variable descriptors
1026 }
1028 // dependencies
1038 }
1040 // copyin/copyout data length
1044 // First pass over variable descriptors
1045 // - Calculate size of the input and output non-pointer data
1046 // - Allocate buffers for input and output pointers
1058 }
1059 }
1067 }
1069 " type_src=%d, type_dstn=%d, direction=%d, "
1070 "alloc_if=%d, free_if=%d, align=%d, mic_offset=%d, flags=0x%x, "
1085 // If any varDesc flags bits set, show them
1088 }
1090 // preallocated implies targetptr
1092 // targetptr preallocated alloc_if(1) may not be used with
1093 // an in clause
1097 }
1099 }
1101 // array descriptor
1105 // debug dump
1111 }
1120 // flag is_arr_ptr_el is 1 only for var_descs generated
1121 // for c_data_ptr_array type
1124 }
1128 {
1145 }
1150 }
1155 }
1160 }
1170 }
1181 }
1191 }
1201 }
1211 }
1221 }
1222 }
1225 }
1231 // In all uses later
1232 // VarDesc.size will have the length of the data to be
1233 // transferred
1234 // VarDesc.disp will have an offset from base
1236 // array descriptor
1240 // debug dump
1243 // offset and length are derived from the array descriptor
1250 }
1251 // all necessary information about length and offset is
1252 // transferred in var descriptor. There is no need to send
1253 // array descriptor to the target side.
1255 }
1259 }
1262 // make sure that transfer size > 0
1266 }
1271 // find data associated with variable
1278 }
1281 // offset to base from the beginning of the buffer
1282 // memory
1286 }
1291 }
1292 }
1294 }
1298 // Static data is transferred either by omp target
1299 // update construct which passes zeros for
1300 // alloc_if and free_if or by always modifier.
1304 }
1305 }
1312 }
1314 // TODO: what should be done if var is not in
1315 // the table?
1318 }
1320 // For automatic variables data is transferred:
1321 // - if always modifier is used OR
1322 // - if alloc_if == 0 && free_if == 0 OR
1323 // - if reference count is 1
1329 }
1331 // save data for later use
1333 }
1334 }
1340 // for non-static target destination defined as CEAN
1341 // expression we pass to target its size and dist
1345 }
1347 }
1352 }
1353 }
1362 // debug dump
1365 // send dope vector contents excluding base
1368 }
1379 }
1380 /* fallthru */
1386 // this var_desc is for stack buffer
1393 }
1399 }
1403 }
1407 // need to add or remove references for stack buffer at target
1410 }
1413 }
1414 /* fallthru */
1419 // array descriptor
1423 // debug dump
1426 // offset and length are derived from the array descriptor
1434 }
1435 // all necessary information about length and offset is
1436 // transferred in var descriptor. There is no need to send
1437 // array descriptor to the target side.
1439 }
1441 // need to send DV to the device unless it is 'nocopy'
1447 // debug dump
1451 }
1453 // no displacement
1455 }
1457 // c_data_ptr or c_string_ptr
1460 }
1467 // check that buffer length > 0
1473 }
1475 // base address
1478 // allocate buffer if we have no INTO and don't need
1479 // allocation for the ptr at target
1482 // for stack persistent objects ptr data is created
1483 // by var_desc with number 0.
1484 // Its ptr_data is stored at m_stack_ptr_data
1487 }
1493 }
1494 // add new entry
1496 ptr_data,
1502 alloc_disp,
1509 }
1513 }
1516 }
1519 // add buffer to the list of buffers that
1520 // are passed to dispatch call
1523 }
1526 // will send buffer address to device
1528 }
1532 // need to add reference for buffer
1534 }
1535 }
1539 // For omp target update variable is ignored
1540 // if it does not exist.
1545 }
1546 }
1548 // use existing association from pointer table
1550 base,
1554 error_if_not_found)) {
1556 }
1559 // make var nocopy if it does not exist
1562 c_parameter_nocopy;
1563 }
1564 }
1568 }
1569 }
1573 // data is transferred only if
1574 // alloc_if == 0 && free_if == 0
1575 // or reference count is 1
1581 c_parameter_nocopy;
1582 }
1583 }
1588 }
1591 // get buffers's address on the sink
1594 }
1597 }
1601 // need to decrement buffer reference on target
1603 }
1605 // offset to base from the beginning of the buffer
1606 // memory
1610 // copy other pointer properties to var descriptor
1613 }
1614 }
1617 base,
1622 }
1627 }
1628 }
1630 // save pointer data
1632 }
1638 }
1641 }
1657 }
1662 }
1664 // if allocatable dope vector isn't allocated don't
1665 // transfer its data
1670 }
1679 // debug dump
1681 }
1686 }
1688 // size and displacement
1690 // offset and length are derived from the
1691 // array descriptor
1700 }
1704 }
1705 }
1706 }
1711 }
1714 }
1716 }
1718 // check that length >= 0
1723 }
1725 // base address
1729 // allocate buffer if we have no INTO and don't need
1730 // allocation for the ptr at target
1733 // add new entry
1735 ptr_data,
1741 alloc_disp,
1748 }
1752 // add buffer to the list of buffers
1753 // that are passed to dispatch call
1756 }
1758 // will send buffer address to device
1760 }
1763 // need to add reference for buffer
1765 }
1766 }
1770 // For omp target update variable is ignored
1771 // if it does not exist.
1776 }
1777 }
1779 // use existing association from pointer table
1781 base,
1785 error_if_not_found)) {
1787 }
1790 // make var nocopy if it does not exist
1793 c_parameter_nocopy;
1794 }
1795 }
1798 // need to update base in dope vector on device
1800 }
1801 }
1805 // data is transferred if
1806 // - if always modifier is used OR
1807 // - if alloc_if == 0 && free_if == 0 OR
1808 // - if reference count is 1
1814 c_parameter_nocopy;
1815 }
1816 }
1821 }
1824 // get buffers's address on the sink
1827 }
1830 }
1833 // need to decrement buffer reference on target
1835 }
1837 // offset to base from the beginning of the buffer
1838 // memory
1843 // copy other pointer properties to var descriptor
1846 }
1847 }
1850 base,
1855 }
1859 }
1861 // save pointer data
1863 }
1868 LIBOFFLOAD_ABORT;
1869 }
1872 }
1877 }
1878 // if source is used at CPU save its offset and disp
1882 }
1884 // If "into" is define we need to do the similar work for it
1887 }
1900 // array descriptor
1904 // debug dump
1907 // offset and length are derived from the array descriptor
1919 }
1920 }
1922 }
1929 size;
1930 // It's supposed that "into" size must be not less
1931 // than src size
1936 }
1942 // find data associated with variable
1946 }
1948 // offset to base from the beginning of the buffer
1949 // memory
1950 into_offset =
1953 }
1956 }
1958 }
1959 }
1965 // for non-static target destination defined as CEAN
1966 // expression we pass to target its size and dist
1969 }
1971 }
1973 }
1981 // debug dump
1984 // send dope vector contents excluding base
1987 }
1997 // array descriptor
2001 // debug dump
2004 // offset and length are derived from the array descriptor
2015 }
2016 }
2018 }
2020 // need to send DV to the device unless it is 'nocopy'
2026 // debug dump
2030 }
2031 }
2038 size;
2039 // It's supposed that "into" size must be not less than
2040 // src size
2045 }
2050 // base address
2054 // allocate buffer
2056 // for stack persistent objects ptr data is created
2057 // by var_desc with number 0.
2058 // Its ptr_data is stored at m_stack_ptr_data
2061 }
2067 }
2068 // add new entry
2070 ptr_data,
2076 alloc_disp,
2083 }
2087 }
2090 }
2093 // add buffer to the list of buffers that
2094 // are passed to dispatch call
2097 }
2099 // will send buffer address to device
2101 }
2104 // need to add reference for buffer
2106 }
2107 }
2109 // use existing association from pointer table
2113 }
2115 }
2120 }
2123 // get buffers's address on the sink
2126 }
2129 }
2132 // need to decrement buffer reference on target
2134 }
2136 // copy other pointer properties to var descriptor
2139 }
2142 base,
2143 into_disp,
2147 }
2148 }
2154 }
2155 // save pointer data
2157 }
2159 }
2180 // debug dump
2186 }
2191 }
2196 }
2197 // size and displacement
2199 // offset and length are derived from the array
2200 // descriptor
2207 }
2215 }
2216 }
2217 }
2218 }
2222 }
2225 }
2227 }
2236 size;
2237 // It's supposed that "into" size must be not less
2238 // than src size
2243 }
2245 // base address
2248 // allocate buffer
2251 // add new entry
2253 ptr_data,
2259 alloc_disp,
2266 }
2269 // add buffer to the list of buffers
2270 // that are passed to dispatch call
2273 }
2275 // will send buffer address to device
2277 }
2280 // need to add reference for buffer
2282 }
2283 }
2285 // use existing association from pointer table
2289 }
2291 // need to update base in dope vector on device
2293 }
2298 }
2301 // get buffers's address on the sink
2304 }
2306 }
2309 // need to decrement buffer reference on target
2311 }
2313 // offset to base from the beginning of the buffer
2314 // memory
2315 into_offset =
2318 // copy other pointer properties to var descriptor
2321 }
2324 base,
2325 into_disp,
2326 size,
2329 }
2333 }
2335 // save pointer data
2337 }
2342 LIBOFFLOAD_ABORT;
2343 }
2344 // if into is used at CPU save its offset and disp
2348 }
2353 }
2356 }
2357 }
2360 }
2363 {
2366 // we can skip run functon call together with wait if offloaded
2367 // region is empty and there is no user defined non-pointer IN/OUT data
2369 // variable descriptors are sent as input data
2372 // timer data is sent as a part of the output data
2375 // max from input data and output data length
2377 m_out_datalen;
2379 // Misc data has the following layout
2380 // <Function Descriptor>
2381 // <Function Name>
2382 // <In/Out Data> (optional)
2383 //
2384 // We can transfer copyin/copyout data in misc/return data which can
2385 // be passed to run function call if its size does not exceed
2386 // COI_PIPELINE_MAX_IN_MISC_DATA_LEN. Otherwise we have to allocate
2387 // buffer for it.
2398 // use misc/return data for copyin/copyout
2401 }
2404 c_offload_host_alloc_data_buffer);
2406 // send/receive data using buffer
2408 COI_BUFFER_NORMAL,
2416 }
2418 }
2422 }
2423 }
2425 // initialize function descriptor
2441 // append entry name
2443 }
2446 }
2449 {
2464 }
2468 }
2481 }
2484 }
2488 }
2491 }
2492 }
2496 }
2499 }
2502 m_omp_async_last_event_type);
2503 }
2507 COIEVENT e,
2510 )
2511 {
2513 }
2514 }
2519 {
2529 }
2530 }
2535 _Offload_stream handle
2536 )
2537 {
2543 }
2545 // wait for streams
2548 // some specific stream of the device
2552 // the stream was not created or was destroyed
2555 LIBOFFLOAD_ABORT;
2556 }
2559 // offload was completed by previous offload_wait pragma
2560 // or wait clause
2563 }
2566 }
2570 }
2571 // all streams of the device or over all devices
2581 }
2582 // get associated async task
2585 // offload was completed by offload_wait pragma or wait clause
2588 }
2591 }
2595 }
2596 // no uncompleted streams
2598 }
2599 }
2601 // if handle is equal to no_stream it's wait for signals
2603 _Offload_stream stream_handle;
2609 LIBOFFLOAD_ABORT;
2610 }
2613 }
2616 }
2618 // if the offload both has signal and is last offload of its
2619 // stream, we must wipe out the "last_offload" reference as
2620 // the offload already is finished.
2626 }
2627 }
2629 }
2630 }
2632 }
2645 OffloadFlags offload_flags
2646 )
2647 {
2651 // define kind of wait if any;
2652 // there can be one off the following kind:
2653 // 1. c_offload_wait_signal for "offload_wait wait(signal)"
2654 // 2. c_offload_wait_stream for "offload_wait stream(stream)"
2655 // 3. c_offload_wait_all_streams for "offload_wait stream(0)"
2658 c_offload_wait_all_streams :
2659 c_offload_wait_stream;
2660 }
2666 }
2669 }
2673 }
2678 c_offload_init_func,
2679 "Offload function %s, is_empty=%d, #varDescs=%d, "
2683 // Breaks the norm of using OFFLOAD_DEBUG_TRACE to print the waits
2684 // since the number of waits is not fixed.
2692 }
2693 }
2696 }
2699 }
2701 }
2704 }
2705 // stream in wait is reported further in OFFLOAD_REPORT for waits
2708 c_offload_stream,
2710 }
2712 c_offload_signal,
2714 }
2718 c_offload_init_func,
2719 "Offload function %s, is_empty=%d, #varDescs=%d, "
2723 // Breaks the norm of using OFFLOAD_DEBUG_TRACE to print the waits
2724 // since the number of waits is not fixed.
2732 }
2734 }
2737 }
2740 }
2742 }
2745 }
2746 // stream in wait is reported further in OFFLOAD_REPORT for waits
2749 c_offload_stream,
2751 }
2753 c_offload_signal,
2755 }
2758 }
2764 waits :
2770 }
2774 // wait for dependencies to finish
2778 }
2780 // setup buffers
2784 }
2788 }
2790 // initiate send for pointers. Want to do it as early as possible.
2792 signal)) {
2795 }
2797 // setup misc data for run function
2801 }
2803 // gather copyin data into buffer
2807 }
2809 // Start the computation
2813 }
2815 // initiate receive for pointers
2820 }
2823 }
2824 // if there is a signal or stream save descriptor for the later use.
2825 // num_waits == -1 is for offload_wait and there is nothing to save
2829 }
2835 }
2838 LIBOFFLOAD_ABORT;
2839 }
2840 }
2841 // if there is a clause with alloc_if(1) and preallocated need to call
2842 // offload_finish after runfunction
2845 }
2846 }
2848 // wait for the offload to finish.
2852 }
2856 }
2869 OffloadFlags offload_flags
2870 )
2871 {
2882 }
2883 }
2885 }
2888 bool is_traceback
2889 )
2890 {
2891 COIRESULT res;
2893 // wait for compute dependencies to become signaled
2898 // keep CPU busy
2901 }
2903 }
2906 }
2916 }
2918 }
2925 }
2928 }
2929 }
2931 // scatter copyout data received from target
2934 }
2940 }
2942 // wait for receive dependencies to become signaled
2947 // keep CPU busy
2950 }
2952 }
2955 }
2961 }
2963 }
2964 }
2966 // destroy buffers
2967 {
2977 }
2979 }
2980 }
2981 }
2984 }
2987 {
2988 // release device in orsl
2993 // report stuff
2995 }
2998 {
3000 COIRESULT res;
3002 // check compute and receive dependencies
3006 }
3010 }
3013 }
3019 int64_t size
3020 )
3021 {
3034 }
3036 // Send pointer data if source or destination or both of them are
3037 // noncontiguous. There is guarantee that length of destination enough for
3038 // transferred data.
3046 COIEVENT *in_deps
3047 )
3048 {
3053 COIRESULT res;
3059 // Set length_src and length_dst
3067 // If BufferWriteMultiD is defined we can set values of required arguments
3068 // and transfer noncontiguous data via call to the COI routine.
3105 arr_desc_src :
3133 }
3135 }
3137 }
3139 // if event is defined we must multiplate it for all contiguous intervals
3140 // that will be Copied/Write.
3141 // Take in account that we already have 1 event.
3146 m_in_deps =
3148 m_in_deps_total--;
3149 }
3151 // consequently get contiguous ranges,
3152 // define corresponded destination offset and send data
3158 // source ranges are over - nothing to send
3160 }
3161 }
3164 }
3167 }
3169 }
3171 // if source is contiguous or its contiguous range is greater
3172 // than destination one
3174 }
3183 // destination ranges are over
3186 }
3187 }
3188 // into is contiguous.
3191 }
3193 }
3194 // same as source
3198 }
3199 }
3201 // if destination is contiguous or its contiguous range is greater
3202 // than source one
3204 }
3210 }
3218 send_size,
3219 COI_COPY_UNSPECIFIED,
3221 event);
3226 }
3228 }
3229 }
3239 send_size,
3240 COI_COPY_UNSPECIFIED,
3242 event);
3247 }
3249 }
3250 }
3252 }
3255 }
3258 {
3263 COIRESULT res;
3267 // For offload_transfer and offload with empty body without signal:
3268 // - if there is only one buffer copy - send data synchronously
3269 // - if there are multiple buffer copy and
3270 // __offload_parallel_copy is false - send data synchronously
3271 // - if there are multiple buffer copy and
3272 // __offload_parallel_copy is true - send data asynchronously
3273 // It concerns only big size data - greater than __offload_use_async_buffer_write.
3274 // Data of size less than __offload_use_async_buffer_write are sent synchronously.
3275 // Synchronous transfer results in better performance in COI.
3276 // __offload_parallel_copy is false by default but can be changed
3277 // via environment variable OFFLOAD_PARALLEL_COPY
3288 big_size_count++;
3289 }
3299 big_size_count++;
3303 }
3304 }
3305 }
3308 }
3309 }
3313 }
3315 // Initiate send for pointer data
3330 }
3331 }
3360 }
3361 }
3371 COI_COPY_UNSPECIFIED,
3373 event);
3378 }
3380 }
3381 }
3391 COI_COPY_UNSPECIFIED,
3393 event);
3398 }
3400 }
3401 }
3403 }
3430 }
3440 COI_COPY_UNSPECIFIED,
3442 event);
3447 }
3449 }
3450 }
3460 COI_COPY_UNSPECIFIED,
3462 event);
3467 }
3469 }
3470 }
3473 }
3496 }
3506 COI_COPY_UNSPECIFIED,
3508 event);
3513 }
3515 }
3516 }
3526 COI_COPY_UNSPECIFIED,
3528 event);
3533 }
3535 }
3536 }
3538 }
3565 }
3576 COI_COPY_UNSPECIFIED,
3578 event);
3583 }
3585 }
3586 }
3596 COI_COPY_UNSPECIFIED,
3598 event);
3603 }
3605 }
3606 }
3609 }
3614 }
3619 }
3620 // alloc field isn't used at target.
3621 // We can reuse it for offset of array pointers.
3624 }
3625 }
3629 }
3633 c_offload_sent_pointer_data,
3635 ptr_sent);
3638 }
3641 {
3645 COIMAPINSTANCE map_inst;
3648 // init marshaller
3651 c_offload_host_map_in_data_buffer);
3654 COI_MAP_WRITE_ENTIRE_BUFFER,
3661 }
3663 }
3664 }
3667 }
3669 // send variable descriptors
3673 // init marshaller
3676 // Gather copy data into buffer
3686 }
3688 // send sink address to the target
3692 }
3706 // offset and length are derived from the array
3707 // descriptor
3714 }
3718 }
3725 // send dope vector excluding base
3729 }
3733 // send to target addresses of obsolete
3734 // stacks to be released
3745 }
3746 }
3751 }
3756 }
3757 }
3761 }
3765 c_offload_host_unmap_in_data_buffer);
3771 }
3773 }
3774 }
3775 }
3784 }
3787 {
3805 }
3806 }
3808 // dispatch task
3809 COIRESULT res;
3810 COIEVENT event;
3816 }
3819 m_compute_buffers,
3822 in_deps_amount,
3823 in_deps,
3830 }
3832 }
3836 }
3840 }
3843 }
3845 // receive pointer data if source or destination or both of them are
3846 // noncontiguous. There is guarantee that length of destination enough for
3847 // transferred data.
3850 COIBUFFER dst_buf,
3854 COIEVENT *in_deps
3855 )
3856 {
3861 COIRESULT res;
3872 // Set length_src and length_dst
3880 // If BufferReadMultiD is defined we can set values of required arguments
3881 // and transfer noncontiguous data via call to the COI routine.
3911 el_size);
3934 }
3936 }
3938 }
3939 // if event is defined we must multiplate for all contiguous intervals
3940 // that will be Copied/Read.
3941 // Take in account that we already have 1 event.
3946 m_out_deps =
3948 m_out_deps_total--;
3949 }
3951 // consequently get contiguous ranges,
3952 // define corresponded destination offset and receive data
3954 // get sorce offset
3959 // source ranges are over - nothing to send
3961 }
3962 }
3965 }
3968 }
3970 }
3972 // if source is contiguous or its contiguous range is greater
3973 // than destination one
3975 }
3979 // get destination offset
3985 // destination ranges are over
3988 }
3989 }
3990 // destination is contiguous.
3993 }
3995 }
3996 // same as source
4000 }
4001 }
4003 // if destination is contiguous or its contiguous range is greater
4004 // than source one
4006 }
4011 }
4014 dst_buf,
4019 receive_size,
4020 COI_COPY_UNSPECIFIED,
4021 in_deps_amount,
4022 in_deps,
4023 event);
4028 }
4030 }
4031 }
4038 receive_size,
4039 COI_COPY_UNSPECIFIED,
4040 in_deps_amount,
4041 in_deps,
4042 event);
4047 }
4049 }
4050 }
4052 }
4055 }
4059 {
4064 COIRESULT res;
4066 // For offload_transfer and offload with empty body without signal:
4067 // - if there is only one buffer copy - get data synchronously
4068 // - if there are multiple buffer copy and
4069 // __offload_parallel_copy is false - get data synchronously
4070 // - if there are multiple buffer copy
4071 // and __offload_parallel_copy is true - get data asynchronously
4072 // It concerns only data with size greater than __offload_use_async_buffer_read.
4073 // Data of size less than __offload_use_async_buffer_read are received synchronously.
4074 // Synchronous transfer results in better performance in COI.
4075 // __offload_parallel_copy is false by default but can be changed
4076 // via environment variable OFFLOAD_PARALLEL_COPY
4083 // preallocated OUT only at second run
4086 }
4092 big_size_count++;
4093 }
4103 big_size_count++;
4107 }
4108 }
4109 }
4112 }
4113 }
4121 }
4140 m_out_deps,
4142 }
4143 }
4144 // At first run don't receive by preallocated target pointer as the
4145 //pointer value will be ready later after call to scatter_copyout_data
4148 // need one more call to OffloadDescriptor::receive_pointer_data
4151 }
4153 }
4176 }
4182 }
4183 }
4191 }
4198 }
4201 dst_buf,
4207 COI_COPY_UNSPECIFIED,
4208 in_deps_amount,
4209 in_deps,
4210 event);
4215 }
4217 }
4218 }
4226 COI_COPY_UNSPECIFIED,
4227 in_deps_amount,
4228 in_deps,
4229 event);
4234 }
4236 }
4237 }
4239 }
4271 }
4274 }
4280 }
4287 }
4290 }
4304 }
4307 }
4313 in_deps_amount,
4314 in_deps);
4315 }
4318 dst_buf,
4320 dst_offset,
4324 COI_COPY_UNSPECIFIED,
4325 in_deps_amount,
4326 in_deps,
4327 event);
4332 }
4334 }
4335 }
4343 COI_COPY_UNSPECIFIED,
4344 in_deps_amount,
4345 in_deps,
4346 event);
4351 }
4353 }
4354 }
4356 }
4358 }
4362 }
4368 }
4369 // destroy buffers for obsolete stacks
4377 }
4379 }
4381 // remove association for automatic variables
4390 }
4393 }
4394 }
4395 }
4397 // destroy buffers
4403 }
4407 // destroy buffers
4410 }
4413 }
4417 // remove association from map
4420 }
4423 }
4424 }
4425 }
4431 // destroy buffers
4434 }
4437 }
4441 // remove association from map
4444 }
4447 }
4448 }
4449 }
4450 }
4451 }
4455 }
4459 c_offload_received_pointer_data,
4461 ptr_received);
4464 }
4467 {
4472 // total size that need to be transferred from target to host
4473 COIMAPINSTANCE map_inst;
4474 COIRESULT res;
4477 // output data buffer
4480 c_offload_host_map_out_data_buffer);
4490 }
4492 }
4493 }
4496 }
4498 // get timing data
4502 // initialize output marshaller
4520 // array descriptor
4527 }
4529 // get pointer to target memory
4532 // add new entry
4534 ptr_data,
4535 ptr_value,
4540 alloc_disp,
4546 }
4552 }
4555 }
4558 }
4575 }
4581 }
4582 }
4588 }
4593 }
4594 }
4598 }
4602 c_offload_host_unmap_out_data_buffer);
4609 }
4611 }
4612 }
4613 }
4620 }
4628 CeanReadRanges* &ptr_ranges
4629 )
4630 {
4635 }
4641 }
4642 }
4645 {
4670 // "pointers_number" for total number of transferred pointers.
4671 // For each of them we create new var_desc and put it at the bottom
4672 // of the var_desc's array
4679 // 2. prepare memory for new var_descs
4684 m_vars_extra =
4688 m_in_deps =
4692 m_out_deps =
4697 // 3. Prepare for reading new var_desc's fields
4698 // EXTENT START
4709 }
4710 }
4713 }
4716 }
4718 // EXTENT ELEMENTS NUMBER
4730 }
4731 }
4734 }
4737 }
4739 // ALLOC_IF
4750 }
4751 }
4754 }
4756 // FREE_IF
4767 }
4768 }
4771 }
4773 // ALIGN
4785 }
4786 }
4789 }
4791 // 3.1 INTO
4802 }
4803 }
4805 // 3.2 INTO_START
4817 }
4818 }
4821 }
4824 }
4826 // 3.3 INTO_ELEMENTS
4838 }
4839 }
4842 }
4845 }
4847 // alloc_start
4860 }
4861 }
4864 }
4867 }
4869 // alloc_elem
4881 }
4882 }
4885 }
4888 }
4893 // Get new values
4894 // type_src, type_dst
4901 // Get ptr val
4904 }
4907 }
4909 // !!! If we got error at phase of reading - it's an internal
4910 // !!! error, as we must detect mismatch before
4912 // Get into val
4916 LIBOFFLOAD_ABORT;
4917 }
4920 }
4921 }
4923 // Get other components of the clause
4926 LIBOFFLOAD_ABORT;
4927 }
4931 LIBOFFLOAD_ABORT;
4932 }
4935 LIBOFFLOAD_ABORT;
4936 }
4939 LIBOFFLOAD_ABORT;
4940 }
4943 LIBOFFLOAD_ABORT;
4944 }
4947 LIBOFFLOAD_ABORT;
4948 }
4951 LIBOFFLOAD_ABORT;
4952 }
4955 LIBOFFLOAD_ABORT;
4956 }
4960 LIBOFFLOAD_ABORT;
4961 }
4980 }
4981 }
4993 type_src;
4996 }
4997 }
5008 type_src;
5009 }
5012 }
5021 }
5024 }
5033 }
5034 // count and alloc fields are useless at target. They can be reused
5035 // for pointer arrays.
5039 }
5041 // Gets in dependencies of the previous offload via the stream "m_stream".
5042 // Out argument in_deps_amount - address of amount of the dependencies
5043 // Out argument in_deps - array of dependencies.
5044 // Description of the dependencies scheme for streams :
5045 // ----------------------------------------------------
5046 // Every offload forms DAG consisted of 3 nodes:
5047 // for in-transfers, runfunction and out-transfers.
5048 // Every node has in-dependencies and out-dependencies
5049 // Out-dependencies of previous node forms in-dependencies of current node.
5050 // In-dependencies of 1-st node (of in-transfers) without streams is equal
5051 // to NULL. For streams in-dependencies of 1-st node is equal to list of out
5052 // dependencies of last node of previous offload via this stream.
5053 // So we can say that DAGs of 2 consequent offloads via the same stream are
5054 // connected by the way described above.
5057 COIEVENT* &in_deps
5058 )
5059 {
5065 LIBOFFLOAD_ABORT;
5066 }
5069 // if it's the first offload in the stream
5072 }
5073 // if last offload has out-tranfers
5077 }
5078 // last offload only sends pointer data or run function or both of them
5079 // and has no out-transfers
5083 }
5084 }
5085 }
5088 {
5097 }
5102 }
5104 // destroy thread key
5106 }
5108 // unload COI library
5111 }
5114 }
5117 {
5118 COIRESULT res;
5123 // initialize trace
5129 }
5130 }
5140 }
5143 offload_report_envname);
5144 }
5145 }
5148 offload_report_envname);
5149 }
5150 }
5155 }
5156 }
5158 // initialize COI
5161 }
5163 // get number of devices installed in the system
5167 }
5171 }
5173 // fill in the list of devices that can be used for offloading
5177 // value is composed of comma separated physical device indexes
5184 // convert string to an int
5189 // fallback to using all installed devices
5193 }
5195 }
5199 }
5201 }
5203 }
5204 }
5206 // use all available devices
5208 COIENGINE engine;
5212 }
5213 }
5214 }
5218 // no need to continue if there are no devices to offload to
5221 }
5223 // initialize indexes for available devices
5228 l_idx++;
5229 }
5230 }
5232 // Get DMA channel count to pass it to COI
5238 }
5242 }
5243 }
5245 // Set COI_HOST_THREAD_AFFINITY if OFFLOAD_HOST_THREAD_AFFINITY is set.
5246 // Use putenv instead of setenv as Windows has no setenv.
5247 // Note: putenv requires its argument can't be freed or modified.
5248 // So no free after call to putenv or elsewhere.
5258 }
5260 // library search path for device binaries
5266 }
5269 // find target executable to be used if main application is not an
5270 // offload build application.
5282 // compose a full path
5285 // check if such file exists
5292 }
5293 }
5296 }
5298 // memory size reserved for COI buffers
5304 }
5307 }
5308 }
5310 // memory size reserved for 4K pages for COI buffers
5316 }
5319 }
5320 }
5322 // memory size reserved for 2M pages for COI buffers
5328 }
5331 }
5332 }
5334 // determine stacksize for the pipeline on the device
5341 }
5344 }
5345 }
5347 // proxy I/O
5353 }
5356 }
5357 }
5363 }
5365 // Prepare environment for the target process using the following
5366 // rules
5367 // - If MIC_ENV_PREFIX is set then any environment variable on the
5368 // host which has that prefix are copied to the device without
5369 // the prefix.
5370 // All other host environment variables are ignored.
5371 // - If MIC_ENV_PREFIX is not set or if MIC_ENV_PREFIX="" then host
5372 // environment is duplicated.
5383 }
5384 }
5385 }
5387 // create key for thread data
5391 }
5393 // cpu frequency
5401 }
5404 mic_use_2mb_buffers_envname);
5405 }
5406 }
5413 }
5414 }
5421 }
5422 }
5424 // mic initialization type
5429 }
5432 }
5435 }
5438 }
5439 }
5441 // active wait
5447 }
5450 offload_active_wait_envname);
5451 }
5452 }
5454 // omp device num
5460 }
5463 omp_device_num_envname);
5464 }
5465 }
5467 // parallel copy of offload_transfer
5473 }
5476 parallel_copy_envname);
5477 }
5478 }
5480 // use COI interface for noncontiguous arrays transfer
5486 }
5489 use_coi_noncontiguous_transfer_envname);
5490 }
5491 }
5493 // init ORSL
5495 }
5498 {
5499 // do one time intialization
5503 // offload is available if COI is available and the number of devices > 0
5506 // register pending libraries if there are any
5512 // Register library in COI
5516 // add lib to all engines
5519 }
5520 }
5524 }
5527 }
5530 {
5533 // decode image
5537 // determine image type
5540 }
5543 {
5546 // decode image
5558 // The origin name is the name of the file on the host
5559 // this is used by Vtune, since it is a fat binary we
5560 // use the host file name of the fat binary.
5561 // Driver prepends the host file name ending with "?"
5562 // to the image->data name so need to extract the string
5566 host_name++;
5567 i++;
5568 }
5570 // Implies the host name does not exist which really should
5571 // not occur. Allow this since only consumer is Vtune.
5575 }
5577 // our actions depend on the image type
5581 // Each offload application is supposed to have only one target
5582 // image representing target executable.
5583 // No thread synchronization is required here as the initialization
5584 // code is always executed in a single thread.
5588 }
5591 // Registration code for execs is always called from the context
5592 // of main and thus we can safely call any function here,
5593 // including LoadLibrary API on windows. This is the place where
5594 // we do the offload library initialization.
5596 // initialize engine if init_type is on_start
5600 }
5601 }
5602 }
5606 {
5608 // We add the library to a list of pending libraries
5614 // If __target_exe is set, then main has started running
5615 // If not main, then we can't do anything useful here
5616 // because this registration code is called from DllMain
5617 // context (on windows).
5619 // There is no need to delay loading the library
5621 // Couldn't validate library as a fat offload library
5624 }
5625 }
5627 }
5630 // something is definitely wrong, issue an error and exit
5633 }
5634 }
5637 {
5638 // Target image is packed as follows:
5639 // 8 bytes - size of the target binary
5640 // null-terminated string - binary name
5641 // <size> bytes - binary contents
5647 // decode image
5651 // our actions depend on the image type
5654 // We are executing exec's desctructors.
5655 // It is time to do a library cleanup.
5658 }
5660 #ifdef MYO_SUPPORT
5665 }
5669 }
5671 }
5672 }
5675 {
5677 }
5679 // Runtime trace interface for user programs
5682 {
5684 }
5686 // User-visible offload API
5689 {
5692 }
5695 {
5697 }
5700 {
5702 }
5705 {
5708 // check index value
5711 LIBOFFLOAD_ABORT;
5712 }
5716 // find associated async task
5721 LIBOFFLOAD_ABORT;
5722 }
5723 // if signal is removed by wait completing
5726 }
5728 }
5731 {
5735 }
5736 }
5746 )
5747 {
5750 // check target value
5753 LIBOFFLOAD_ABORT;
5754 }
5757 // find existing association in pointer table
5762 }
5773 }
5774 }
5781 }
5786 )
5787 {
5790 // check target value
5793 LIBOFFLOAD_ABORT;
5794 }
5797 // Create new stream and get its handle
5802 }
5804 // create pipeline associated with the new stream
5808 }
5812 _Offload_stream handle // stream to destroy
5813 )
5814 {
5817 // check target value
5820 LIBOFFLOAD_ABORT;
5821 }
5827 }
5830 {
5833 // check index value
5836 LIBOFFLOAD_ABORT;
5837 }
5841 // get stream
5847 // the stream was not created or was destroyed
5850 LIBOFFLOAD_ABORT;
5851 }
5853 // find associated async task
5856 // offload was completed by offload_wait pragma or wait clause
5859 }
5861 }
5862 // zero handler is for all streams at the device
5868 // find associated async task
5871 // offload was completed by offload_wait pragma or wait clause
5874 }
5875 // if even one stream is not completed result is false
5878 }
5879 }
5880 // no uncompleted streams
5882 }
5883 }
5885 // IDB support
5894 {
5895 }
5897 {
5898 }