liboffloadmic/runtime/offload_myo_host.cpp

   1 /*
   2     Copyright (c) 2014 Intel Corporation.  All Rights Reserved.
   3
   4     Redistribution and use in source and binary forms, with or without
   5     modification, are permitted provided that the following conditions
   6     are met:
   7
   8       * Redistributions of source code must retain the above copyright
   9         notice, this list of conditions and the following disclaimer.
  10       * Redistributions in binary form must reproduce the above copyright
  11         notice, this list of conditions and the following disclaimer in the
  12         documentation and/or other materials provided with the distribution.
  13       * Neither the name of Intel Corporation nor the names of its
  14         contributors may be used to endorse or promote products derived
  15         from this software without specific prior written permission.
  16
  17     THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18     "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19     LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20     A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21     HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22     SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23     LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24     DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25     THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26     (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27     OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 */
  29
  30
  31 #include "offload_myo_host.h"
  32 #include <errno.h>
  33 #include <malloc.h>
  34 #include "offload_host.h"
  35
  36 #if defined(LINUX) || defined(FREEBSD)
  37 #include <mm_malloc.h>
  38 #endif
  39
  40 #define MYO_VERSION1    "MYO_1.0"
  41
  42 extern "C" void __cilkrts_cilk_for_32(void*, void*, uint32_t, int32_t);
  43 extern "C" void __cilkrts_cilk_for_64(void*, void*, uint64_t, int32_t);
  44
  45 #ifndef TARGET_WINNT
  46 #pragma weak __cilkrts_cilk_for_32
  47 #pragma weak __cilkrts_cilk_for_64
  48 #endif // TARGET_WINNT
  49
  50 #ifdef TARGET_WINNT
  51 #define MYO_TABLE_END_MARKER() reinterpret_cast<const char*>(-1)
  52 #else // TARGET_WINNT
  53 #define MYO_TABLE_END_MARKER() reinterpret_cast<const char*>(0)
  54 #endif // TARGET_WINNT
  55
  56 class MyoWrapper {
  57 public:
  58     MyoWrapper() : m_lib_handle(0), m_is_available(false)
  59     {}
  60
  61     bool is_available() const {
  62         return m_is_available;
  63     }
  64
  65     bool LoadLibrary(void);
  66
  67     // unloads the library
  68     void UnloadLibrary(void) {
  69 //        if (m_lib_handle != 0) {
  70 //            DL_close(m_lib_handle);
  71 //            m_lib_handle = 0;
  72 //        }
  73     }
  74
  75     // Wrappers for MYO client functions
  76     void LibInit(void *arg, void *func) const {
  77         OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myoinit,
  78                                  "%s(%p, %p)\n", __func__, arg, func);
  79         CheckResult(__func__, m_lib_init(arg, func));
  80     }
  81
  82     void LibFini(void) const {
  83         OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myofini, "%s()\n", __func__);
  84         m_lib_fini();
  85     }
  86
  87     void* SharedMalloc(size_t size) const {
  88         OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myosharedmalloc,
  89                                  "%s(%lld)\n", __func__, size);
  90         return m_shared_malloc(size);
  91     }
  92
  93     void SharedFree(void *ptr) const {
  94         OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myosharedfree,
  95                                  "%s(%p)\n", __func__, ptr);
  96         m_shared_free(ptr);
  97     }
  98
  99     void* SharedAlignedMalloc(size_t size, size_t align) const {
 100         OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myosharedalignedmalloc,
 101                                  "%s(%lld, %lld)\n", __func__, size, align);
 102         return m_shared_aligned_malloc(size, align);
 103     }
 104
 105     void SharedAlignedFree(void *ptr) const {
 106         OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myosharedalignedfree,
 107                               "%s(%p)\n", __func__, ptr);
 108         m_shared_aligned_free(ptr);
 109     }
 110
 111     void Acquire(void) const {
 112         OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myoacquire,
 113                               "%s()\n", __func__);
 114         CheckResult(__func__, m_acquire());
 115     }
 116
 117     void Release(void) const {
 118         OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myorelease,
 119                             "%s()\n", __func__);
 120         CheckResult(__func__, m_release());
 121     }
 122
 123     void HostVarTablePropagate(void *table, int num_entries) const {
 124         OFFLOAD_DEBUG_TRACE(4, "%s(%p, %d)\n", __func__, table, num_entries);
 125         CheckResult(__func__, m_host_var_table_propagate(table, num_entries));
 126     }
 127
 128     void HostFptrTableRegister(void *table, int num_entries,
 129                                int ordered) const {
 130         OFFLOAD_DEBUG_TRACE_1(4, 0, c_offload_myoregister,
 131                             "%s(%p, %d, %d)\n", __func__, table,
 132                             num_entries, ordered);
 133         CheckResult(__func__,
 134                     m_host_fptr_table_register(table, num_entries, ordered));
 135     }
 136
 137     void RemoteThunkCall(void *thunk, void *args, int device) {
 138         OFFLOAD_DEBUG_TRACE(4, "%s(%p, %p, %d)\n", __func__, thunk, args,
 139                             device);
 140         CheckResult(__func__, m_remote_thunk_call(thunk, args, device));
 141     }
 142
 143     MyoiRFuncCallHandle RemoteCall(char *func, void *args, int device) const {
 144         OFFLOAD_DEBUG_TRACE(4, "%s(%s, %p, %d)\n", __func__, func, args,
 145                             device);
 146         return m_remote_call(func, args, device);
 147     }
 148
 149     void GetResult(MyoiRFuncCallHandle handle) const {
 150         OFFLOAD_DEBUG_TRACE(4, "%s(%p)\n", __func__, handle);
 151         CheckResult(__func__, m_get_result(handle));
 152     }
 153
 154 private:
 155     void CheckResult(const char *func, MyoError error) const {
 156         if (error != MYO_SUCCESS) {
 157              LIBOFFLOAD_ERROR(c_myowrapper_checkresult, func, error);
 158             exit(1);
 159         }
 160     }
 161
 162 private:
 163     void* m_lib_handle;
 164     bool  m_is_available;
 165
 166     // pointers to functions from myo library
 167     MyoError (*m_lib_init)(void*, void*);
 168     void     (*m_lib_fini)(void);
 169     void*    (*m_shared_malloc)(size_t);
 170     void     (*m_shared_free)(void*);
 171     void*    (*m_shared_aligned_malloc)(size_t, size_t);
 172     void     (*m_shared_aligned_free)(void*);
 173     MyoError (*m_acquire)(void);
 174     MyoError (*m_release)(void);
 175     MyoError (*m_host_var_table_propagate)(void*, int);
 176     MyoError (*m_host_fptr_table_register)(void*, int, int);
 177     MyoError (*m_remote_thunk_call)(void*, void*, int);
 178     MyoiRFuncCallHandle (*m_remote_call)(char*, void*, int);
 179     MyoError (*m_get_result)(MyoiRFuncCallHandle);
 180 };
 181
 182 bool MyoWrapper::LoadLibrary(void)
 183 {
 184 #ifndef TARGET_WINNT
 185     const char *lib_name = "libmyo-client.so";
 186 #else // TARGET_WINNT
 187     const char *lib_name = "myo-client.dll";
 188 #endif // TARGET_WINNT
 189
 190     OFFLOAD_DEBUG_TRACE(2, "Loading MYO library %s ...\n", lib_name);
 191
 192     m_lib_handle = DL_open(lib_name);
 193     if (m_lib_handle == 0) {
 194         OFFLOAD_DEBUG_TRACE(2, "Failed to load the library. errno = %d\n",
 195                             errno);
 196         return false;
 197     }
 198
 199     m_lib_init = (MyoError (*)(void*, void*))
 200         DL_sym(m_lib_handle, "myoiLibInit", MYO_VERSION1);
 201     if (m_lib_init == 0) {
 202         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 203                             "myoiLibInit");
 204         UnloadLibrary();
 205         return false;
 206     }
 207
 208     m_lib_fini = (void (*)(void))
 209         DL_sym(m_lib_handle, "myoiLibFini", MYO_VERSION1);
 210     if (m_lib_fini == 0) {
 211         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 212                             "myoiLibFini");
 213         UnloadLibrary();
 214         return false;
 215     }
 216
 217     m_shared_malloc = (void* (*)(size_t))
 218         DL_sym(m_lib_handle, "myoSharedMalloc", MYO_VERSION1);
 219     if (m_shared_malloc == 0) {
 220         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 221                             "myoSharedMalloc");
 222         UnloadLibrary();
 223         return false;
 224     }
 225
 226     m_shared_free = (void (*)(void*))
 227         DL_sym(m_lib_handle, "myoSharedFree", MYO_VERSION1);
 228     if (m_shared_free == 0) {
 229         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 230                             "myoSharedFree");
 231         UnloadLibrary();
 232         return false;
 233     }
 234
 235     m_shared_aligned_malloc = (void* (*)(size_t, size_t))
 236         DL_sym(m_lib_handle, "myoSharedAlignedMalloc", MYO_VERSION1);
 237     if (m_shared_aligned_malloc == 0) {
 238         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 239                             "myoSharedAlignedMalloc");
 240         UnloadLibrary();
 241         return false;
 242     }
 243
 244     m_shared_aligned_free = (void (*)(void*))
 245         DL_sym(m_lib_handle, "myoSharedAlignedFree", MYO_VERSION1);
 246     if (m_shared_aligned_free == 0) {
 247         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 248                             "myoSharedAlignedFree");
 249         UnloadLibrary();
 250         return false;
 251     }
 252
 253     m_acquire = (MyoError (*)(void))
 254         DL_sym(m_lib_handle, "myoAcquire", MYO_VERSION1);
 255     if (m_acquire == 0) {
 256         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 257                             "myoAcquire");
 258         UnloadLibrary();
 259         return false;
 260     }
 261
 262     m_release = (MyoError (*)(void))
 263         DL_sym(m_lib_handle, "myoRelease", MYO_VERSION1);
 264     if (m_release == 0) {
 265         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 266                             "myoRelease");
 267         UnloadLibrary();
 268         return false;
 269     }
 270
 271     m_host_var_table_propagate = (MyoError (*)(void*, int))
 272         DL_sym(m_lib_handle, "myoiHostVarTablePropagate", MYO_VERSION1);
 273     if (m_host_var_table_propagate == 0) {
 274         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 275                             "myoiHostVarTablePropagate");
 276         UnloadLibrary();
 277         return false;
 278     }
 279
 280     m_host_fptr_table_register = (MyoError (*)(void*, int, int))
 281         DL_sym(m_lib_handle, "myoiHostFptrTableRegister", MYO_VERSION1);
 282     if (m_host_fptr_table_register == 0) {
 283         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 284                             "myoiHostFptrTableRegister");
 285         UnloadLibrary();
 286         return false;
 287     }
 288
 289     m_remote_thunk_call = (MyoError (*)(void*, void*, int))
 290         DL_sym(m_lib_handle, "myoiRemoteThunkCall", MYO_VERSION1);
 291     if (m_remote_thunk_call == 0) {
 292         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 293                             "myoiRemoteThunkCall");
 294         UnloadLibrary();
 295         return false;
 296     }
 297
 298     m_remote_call = (MyoiRFuncCallHandle (*)(char*, void*, int))
 299         DL_sym(m_lib_handle, "myoiRemoteCall", MYO_VERSION1);
 300     if (m_remote_call == 0) {
 301         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 302                             "myoiRemoteCall");
 303         UnloadLibrary();
 304         return false;
 305     }
 306
 307     m_get_result = (MyoError (*)(MyoiRFuncCallHandle))
 308         DL_sym(m_lib_handle, "myoiGetResult", MYO_VERSION1);
 309     if (m_get_result == 0) {
 310         OFFLOAD_DEBUG_TRACE(2, "Failed to find %s in MYO library\n",
 311                             "myoiGetResult");
 312         UnloadLibrary();
 313         return false;
 314     }
 315
 316     OFFLOAD_DEBUG_TRACE(2, "The library was successfully loaded\n");
 317
 318     m_is_available = true;
 319
 320     return true;
 321 }
 322
 323 static bool myo_is_available;
 324 static MyoWrapper myo_wrapper;
 325
 326 struct MyoTable
 327 {
 328     MyoTable(SharedTableEntry *tab, int len) : var_tab(tab), var_tab_len(len)
 329     {}
 330
 331     SharedTableEntry*   var_tab;
 332     int                 var_tab_len;
 333 };
 334
 335 typedef std::list<MyoTable> MyoTableList;
 336 static MyoTableList __myo_table_list;
 337 static mutex_t      __myo_table_lock;
 338 static bool         __myo_tables = false;
 339
 340 static void __offload_myo_shared_table_register(SharedTableEntry *entry);
 341 static void __offload_myo_shared_init_table_register(InitTableEntry* entry);
 342 static void __offload_myo_fptr_table_register(FptrTableEntry *entry);
 343
 344 static void __offload_myoLoadLibrary_once(void)
 345 {
 346     if (__offload_init_library()) {
 347         myo_wrapper.LoadLibrary();
 348     }
 349 }
 350
 351 static bool __offload_myoLoadLibrary(void)
 352 {
 353     static OffloadOnceControl ctrl = OFFLOAD_ONCE_CONTROL_INIT;
 354     __offload_run_once(&ctrl, __offload_myoLoadLibrary_once);
 355
 356     return myo_wrapper.is_available();
 357 }
 358
 359 static void __offload_myoInit_once(void)
 360 {
 361     if (!__offload_myoLoadLibrary()) {
 362         return;
 363     }
 364
 365     // initialize all devices
 366     for (int i = 0; i < mic_engines_total; i++) {
 367         mic_engines[i].init();
 368     }
 369
 370     // load and initialize MYO library
 371     OFFLOAD_DEBUG_TRACE(2, "Initializing MYO library ...\n");
 372
 373     COIEVENT events[MIC_ENGINES_MAX];
 374     MyoiUserParams params[MIC_ENGINES_MAX+1];
 375
 376     // load target library to all devices
 377     for (int i = 0; i < mic_engines_total; i++) {
 378         mic_engines[i].init_myo(&events[i]);
 379
 380         params[i].type = MYOI_USERPARAMS_DEVID;
 381         params[i].nodeid = mic_engines[i].get_physical_index() + 1;
 382     }
 383
 384     params[mic_engines_total].type = MYOI_USERPARAMS_LAST_MSG;
 385
 386     // initialize myo runtime on host
 387     myo_wrapper.LibInit(params, 0);
 388
 389     // wait for the target init calls to finish
 390     COIRESULT res;
 391     res = COI::EventWait(mic_engines_total, events, -1, 1, 0, 0);
 392     if (res != COI_SUCCESS) {
 393         LIBOFFLOAD_ERROR(c_event_wait, res);
 394         exit(1);
 395     }
 396
 397     myo_is_available = true;
 398
 399     OFFLOAD_DEBUG_TRACE(2, "Initializing MYO library ... done\n");
 400 }
 401
 402 static bool __offload_myoInit(void)
 403 {
 404     static OffloadOnceControl ctrl = OFFLOAD_ONCE_CONTROL_INIT;
 405     __offload_run_once(&ctrl, __offload_myoInit_once);
 406
 407     // register pending shared var tables
 408     if (myo_is_available && __myo_tables) {
 409         mutex_locker_t locker(__myo_table_lock);
 410
 411         if (__myo_tables) {
 412             //  Register tables with MYO so it can propagate to target.
 413             for(MyoTableList::const_iterator it = __myo_table_list.begin();
 414                 it != __myo_table_list.end(); ++it) {
 415 #ifdef TARGET_WINNT
 416                 for (SharedTableEntry *entry = it->var_tab;
 417                      entry->varName != MYO_TABLE_END_MARKER(); entry++) {
 418                     if (entry->varName == 0) {
 419                         continue;
 420                     }
 421                     myo_wrapper.HostVarTablePropagate(entry, 1);
 422                 }
 423 #else // TARGET_WINNT
 424                 myo_wrapper.HostVarTablePropagate(it->var_tab,
 425                                                   it->var_tab_len);
 426 #endif // TARGET_WINNT
 427             }
 428
 429             __myo_table_list.clear();
 430             __myo_tables = false;
 431         }
 432     }
 433
 434     return myo_is_available;
 435 }
 436
 437 static bool shared_table_entries(
 438     SharedTableEntry *entry
 439 )
 440 {
 441     OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
 442
 443     for (; entry->varName != MYO_TABLE_END_MARKER(); entry++) {
 444 #ifdef TARGET_WINNT
 445         if (entry->varName == 0) {
 446             continue;
 447         }
 448 #endif // TARGET_WINNT
 449
 450         return true;
 451     }
 452
 453     return false;
 454 }
 455
 456 static bool fptr_table_entries(
 457     FptrTableEntry *entry
 458 )
 459 {
 460     OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
 461
 462     for (; entry->funcName != MYO_TABLE_END_MARKER(); entry++) {
 463 #ifdef TARGET_WINNT
 464         if (entry->funcName == 0) {
 465             continue;
 466         }
 467 #endif // TARGET_WINNT
 468
 469         return true;
 470     }
 471
 472     return false;
 473 }
 474
 475 extern "C" void __offload_myoRegisterTables(
 476     InitTableEntry* init_table,
 477     SharedTableEntry *shared_table,
 478     FptrTableEntry *fptr_table
 479 )
 480 {
 481     // check whether we need to initialize MYO library. It is
 482     // initialized only if at least one myo table is not empty
 483     if (shared_table_entries(shared_table) || fptr_table_entries(fptr_table)) {
 484         // make sure myo library is loaded
 485         __offload_myoLoadLibrary();
 486
 487         // register tables
 488         __offload_myo_shared_table_register(shared_table);
 489         __offload_myo_fptr_table_register(fptr_table);
 490         __offload_myo_shared_init_table_register(init_table);
 491     }
 492 }
 493
 494 void __offload_myoFini(void)
 495 {
 496     if (myo_is_available) {
 497         OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
 498
 499         COIEVENT events[MIC_ENGINES_MAX];
 500
 501         // kick off myoiLibFini calls on all devices
 502         for (int i = 0; i < mic_engines_total; i++) {
 503             mic_engines[i].fini_myo(&events[i]);
 504         }
 505
 506         // cleanup myo runtime on host
 507         myo_wrapper.LibFini();
 508
 509         // wait for the target fini calls to finish
 510         COIRESULT res;
 511         res = COI::EventWait(mic_engines_total, events, -1, 1, 0, 0);
 512         if (res != COI_SUCCESS) {
 513             LIBOFFLOAD_ERROR(c_event_wait, res);
 514             exit(1);
 515         }
 516     }
 517 }
 518
 519 static void __offload_myo_shared_table_register(
 520     SharedTableEntry *entry
 521 )
 522 {
 523     OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
 524
 525     SharedTableEntry *start = entry;
 526     int entries = 0;
 527
 528     // allocate shared memory for vars
 529     for (; entry->varName != MYO_TABLE_END_MARKER(); entry++) {
 530 #ifdef TARGET_WINNT
 531         if (entry->varName == 0) {
 532             OFFLOAD_DEBUG_TRACE(4, "skip registering a NULL MyoSharedTable entry\n");
 533             continue;
 534         }
 535 #endif // TARGET_WINNT
 536
 537         OFFLOAD_DEBUG_TRACE(4, "registering MyoSharedTable entry for %s @%p\n",
 538                             entry->varName, entry);
 539
 540         // Invoke the function to create shared memory
 541         reinterpret_cast<void(*)(void)>(entry->sharedAddr)();
 542         entries++;
 543     }
 544
 545     // and table to the list if it is not empty
 546     if (entries > 0) {
 547         mutex_locker_t locker(__myo_table_lock);
 548         __myo_table_list.push_back(MyoTable(start, entries));
 549         __myo_tables = true;
 550     }
 551 }
 552
 553 static void __offload_myo_shared_init_table_register(InitTableEntry* entry)
 554 {
 555     OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
 556
 557 #ifdef TARGET_WINNT
 558     for (; entry->funcName != MYO_TABLE_END_MARKER(); entry++) {
 559         if (entry->funcName == 0) {
 560             OFFLOAD_DEBUG_TRACE(4, "skip registering a NULL MyoSharedInit entry\n");
 561             continue;
 562         }
 563
 564         //  Invoke the function to init the shared memory
 565         entry->func();
 566     }
 567 #else // TARGET_WINNT
 568     for (; entry->func != 0; entry++) {
 569         // Invoke the function to init the shared memory
 570         entry->func();
 571     }
 572 #endif // TARGET_WINNT
 573 }
 574
 575 static void __offload_myo_fptr_table_register(
 576     FptrTableEntry *entry
 577 )
 578 {
 579     OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, entry);
 580
 581     FptrTableEntry *start = entry;
 582     int entries = 0;
 583
 584     for (; entry->funcName != MYO_TABLE_END_MARKER(); entry++) {
 585 #ifdef TARGET_WINNT
 586         if (entry->funcName == 0) {
 587             OFFLOAD_DEBUG_TRACE(4, "skip registering a NULL MyoFptrTable entry\n");
 588             continue;
 589         }
 590 #endif // TARGET_WINNT
 591
 592         if (!myo_wrapper.is_available()) {
 593             *(static_cast<void**>(entry->localThunkAddr)) = entry->funcAddr;
 594         }
 595
 596         OFFLOAD_DEBUG_TRACE(4, "registering MyoFptrTable entry for %s @%p\n",
 597                             entry->funcName, entry);
 598
 599 #ifdef TARGET_WINNT
 600         if (myo_wrapper.is_available()) {
 601             myo_wrapper.HostFptrTableRegister(entry, 1, false);
 602         }
 603 #endif // TARGET_WINNT
 604
 605         entries++;
 606     }
 607
 608 #ifndef TARGET_WINNT
 609     if (myo_wrapper.is_available() && entries > 0) {
 610         myo_wrapper.HostFptrTableRegister(start, entries, false);
 611     }
 612 #endif // TARGET_WINNT
 613 }
 614
 615 extern "C" int __offload_myoIsAvailable(int target_number)
 616 {
 617     OFFLOAD_DEBUG_TRACE(3, "%s(%d)\n", __func__, target_number);
 618
 619     if (target_number >= -2) {
 620         bool is_default_number = (target_number == -2);
 621
 622         if (__offload_myoInit()) {
 623             if (target_number >= 0) {
 624                 // User provided the device number
 625                 int num = target_number % mic_engines_total;
 626
 627                 // reserve device in ORSL
 628                 target_number = ORSL::reserve(num) ? num : -1;
 629             }
 630             else {
 631                 // try to use device 0
 632                 target_number = ORSL::reserve(0) ? 0 : -1;
 633             }
 634
 635             // make sure device is initialized
 636             if (target_number >= 0) {
 637                 mic_engines[target_number].init();
 638             }
 639         }
 640         else {
 641             // fallback to CPU
 642             target_number = -1;
 643         }
 644
 645         if (target_number < 0 && !is_default_number) {
 646             LIBOFFLOAD_ERROR(c_device_is_not_available);
 647             exit(1);
 648         }
 649     }
 650     else {
 651         LIBOFFLOAD_ERROR(c_invalid_device_number);
 652         exit(1);
 653     }
 654
 655     return target_number;
 656 }
 657
 658 extern "C" void __offload_myoiRemoteIThunkCall(
 659     void *thunk,
 660     void *arg,
 661     int target_number
 662 )
 663 {
 664     OFFLOAD_DEBUG_TRACE(3, "%s(%p, %p, %d)\n", __func__, thunk, arg,
 665                         target_number);
 666
 667     myo_wrapper.Release();
 668     myo_wrapper.RemoteThunkCall(thunk, arg, target_number);
 669     myo_wrapper.Acquire();
 670
 671     ORSL::release(target_number);
 672 }
 673
 674 extern "C" void* _Offload_shared_malloc(size_t size)
 675 {
 676     OFFLOAD_DEBUG_TRACE(3, "%s(%lld)\n", __func__, size);
 677
 678     if (__offload_myoLoadLibrary()) {
 679         return myo_wrapper.SharedMalloc(size);
 680     }
 681     else {
 682         return malloc(size);
 683     }
 684 }
 685
 686 extern "C" void _Offload_shared_free(void *ptr)
 687 {
 688     OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, ptr);
 689
 690     if (__offload_myoLoadLibrary()) {
 691         myo_wrapper.SharedFree(ptr);
 692     }
 693     else {
 694         free(ptr);
 695     }
 696 }
 697
 698 extern "C" void* _Offload_shared_aligned_malloc(size_t size, size_t align)
 699 {
 700     OFFLOAD_DEBUG_TRACE(3, "%s(%lld, %lld)\n", __func__, size, align);
 701
 702     if (__offload_myoLoadLibrary()) {
 703         return myo_wrapper.SharedAlignedMalloc(size, align);
 704     }
 705     else {
 706         if (align < sizeof(void*)) {
 707             align = sizeof(void*);
 708         }
 709         return _mm_malloc(size, align);
 710     }
 711 }
 712
 713 extern "C" void _Offload_shared_aligned_free(void *ptr)
 714 {
 715     OFFLOAD_DEBUG_TRACE(3, "%s(%p)\n", __func__, ptr);
 716
 717     if (__offload_myoLoadLibrary()) {
 718         myo_wrapper.SharedAlignedFree(ptr);
 719     }
 720     else {
 721         _mm_free(ptr);
 722     }
 723 }
 724
 725 extern "C" void __intel_cilk_for_32_offload(
 726     int size,
 727     void (*copy_constructor)(void*, void*),
 728     int target_number,
 729     void *raddr,
 730     void *closure_object,
 731     unsigned int iters,
 732     unsigned int grain_size)
 733 {
 734     OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
 735
 736     target_number = __offload_myoIsAvailable(target_number);
 737     if (target_number >= 0) {
 738         struct S {
 739             void *M1;
 740             unsigned int M2;
 741             unsigned int M3;
 742             char closure[];
 743         } *args;
 744
 745         args = (struct S*) _Offload_shared_malloc(sizeof(struct S) + size);
 746         if (args == NULL)
 747           LIBOFFLOAD_ERROR(c_malloc);
 748         args->M1 = raddr;
 749         args->M2 = iters;
 750         args->M3 = grain_size;
 751
 752         if (copy_constructor == 0) {
 753             memcpy(args->closure, closure_object, size);
 754         }
 755         else {
 756             copy_constructor(args->closure, closure_object);
 757         }
 758
 759         myo_wrapper.Release();
 760         myo_wrapper.GetResult(
 761             myo_wrapper.RemoteCall("__intel_cilk_for_32_offload",
 762                                    args, target_number)
 763         );
 764         myo_wrapper.Acquire();
 765
 766         _Offload_shared_free(args);
 767
 768         ORSL::release(target_number);
 769     }
 770     else {
 771         __cilkrts_cilk_for_32(raddr,
 772                               closure_object,
 773                               iters,
 774                               grain_size);
 775     }
 776 }
 777
 778 extern "C" void __intel_cilk_for_64_offload(
 779     int size,
 780     void (*copy_constructor)(void*, void*),
 781     int target_number,
 782     void *raddr,
 783     void *closure_object,
 784     uint64_t iters,
 785     uint64_t grain_size)
 786 {
 787     OFFLOAD_DEBUG_TRACE(3, "%s\n", __func__);
 788
 789     target_number = __offload_myoIsAvailable(target_number);
 790     if (target_number >= 0) {
 791         struct S {
 792             void *M1;
 793             uint64_t M2;
 794             uint64_t M3;
 795             char closure[];
 796         } *args;
 797
 798         args = (struct S*) _Offload_shared_malloc(sizeof(struct S) + size);
 799         if (args == NULL)
 800           LIBOFFLOAD_ERROR(c_malloc);
 801         args->M1 = raddr;
 802         args->M2 = iters;
 803         args->M3 = grain_size;
 804
 805         if (copy_constructor == 0) {
 806             memcpy(args->closure, closure_object, size);
 807         }
 808         else {
 809             copy_constructor(args->closure, closure_object);
 810         }
 811
 812         myo_wrapper.Release();
 813         myo_wrapper.GetResult(
 814             myo_wrapper.RemoteCall("__intel_cilk_for_64_offload", args,
 815                                    target_number)
 816         );
 817         myo_wrapper.Acquire();
 818
 819         _Offload_shared_free(args);
 820
 821         ORSL::release(target_number);
 822     }
 823     else {
 824         __cilkrts_cilk_for_64(raddr,
 825                               closure_object,
 826                               iters,
 827                               grain_size);
 828     }
 829 }