src/conv-core/cpuaffinity.c

   1
   2 /*
   3  This scheme relies on using IP address to identify nodes and assigning
   4 cpu affinity.
   5
   6  when CMK_NO_SOCKETS, which is typically on cray xt3 and bluegene/L.
   7  There is no hostname for the compute nodes.
   8  *
   9  * last updated 3/20/2010   Gengbin Zheng
  10  * new options +pemap +commmap takes complex pattern of a list of cores
  11 */
  12
  13 #define _GNU_SOURCE
  14
  15 #include "converse.h"
  16 #include "sockRoutines.h"
  17
  18 #define DEBUGP(x)    /* CmiPrintf x;  */
  19 CpvDeclare(int, myCPUAffToCore);
  20 #if CMK_OS_IS_LINUX
  21 /*
  22  * /proc/<PID>/[task/<TID>]/stat file descriptor
  23  * Used to retrieve the info about which physical
  24  * coer this process or thread is on.
  25  **/
  26 CpvDeclare(void *, myProcStatFP);
  27 #endif
  28
  29 #if CMK_HAS_SETAFFINITY || defined (_WIN32) || CMK_HAS_BINDPROCESSOR
  30
  31 #include <stdlib.h>
  32 #include <stdio.h>
  33 #include <unistd.h>
  34
  35 #ifdef _WIN32
  36 #include <windows.h>
  37 #include <winbase.h>
  38 #else
  39 #define _GNU_SOURCE
  40 #include <sched.h>
  41 //long sched_setaffinity(pid_t pid, unsigned int len, unsigned long *user_mask_ptr);
  42 //long sched_getaffinity(pid_t pid, unsigned int len, unsigned long *user_mask_ptr);
  43 #endif
  44
  45 #if CMK_OS_IS_LINUX
  46 #include <sys/syscall.h>
  47 #endif
  48
  49 #if defined(__APPLE__)
  50 #include <Carbon/Carbon.h> /* Carbon APIs for Multiprocessing */
  51 #endif
  52
  53
  54 #define MAX_EXCLUDE      64
  55 static int excludecore[MAX_EXCLUDE] = {-1};
  56 static int excludecount = 0;
  57
  58 static int affinity_doneflag = 0;
  59
  60 static int in_exclude(int core)
  61 {
  62   int i;
  63   for (i=0; i<excludecount; i++) if (core == excludecore[i]) return 1;
  64   return 0;
  65 }
  66
  67 static void add_exclude(int core)
  68 {
  69   if (in_exclude(core)) return;
  70   CmiAssert(excludecount < MAX_EXCLUDE);
  71   excludecore[excludecount++] = core;
  72 }
  73
  74 #if CMK_HAS_BINDPROCESSOR
  75 #include <sys/processor.h>
  76 #endif
  77
  78 #define SET_MASK(cpuid)    \
  79   /* set the affinity mask if possible */      \
  80   if ((cpuid / 8) > len) {      \
  81     printf("Mask size too small to handle requested CPU ID\n");   \
  82     return -1;      \
  83   } else {    \
  84     mask = 1 << cpuid;   /* set the affinity mask exclusively to one CPU */ \
  85   }
  86
  87
  88 /* This implementation assumes the default x86 CPU mask size used by Linux */
  89 /* For a large SMP machine, this code should be changed to use a variable sized   */
  90 /* CPU affinity mask buffer instead, as the present code will fail beyond 32 CPUs */
  91 int set_cpu_affinity(unsigned int cpuid) {
  92   unsigned long mask = 0xffffffff;
  93   unsigned int len = sizeof(mask);
  94   int retValue = 0;
  95   int pid;
  96
  97  #ifdef _WIN32
  98    HANDLE hProcess;
  99  #endif
 100
 101 #ifdef _WIN32
 102   SET_MASK(cpuid)
 103   hProcess = GetCurrentProcess();
 104   if (SetProcessAffinityMask(hProcess, mask) == 0) {
 105     return -1;
 106   }
 107 #elif CMK_HAS_BINDPROCESSOR
 108   pid = getpid();
 109   if (bindprocessor(BINDPROCESS, pid, cpuid) == -1) return -1;
 110 #else
 111 #ifdef CPU_ALLOC
 112  if ( cpuid >= CPU_SETSIZE ) {
 113   cpu_set_t *cpusetp;
 114   size_t size;
 115   int num_cpus;
 116   num_cpus = cpuid + 1;
 117   cpusetp = CPU_ALLOC(num_cpus);
 118   if (cpusetp == NULL) {
 119     perror("set_cpu_affinity CPU_ALLOC");
 120     return -1;
 121   }
 122   size = CPU_ALLOC_SIZE(num_cpus);
 123   CPU_ZERO_S(size, cpusetp);
 124   CPU_SET_S(cpuid, size, cpusetp);
 125   if (sched_setaffinity(0, size, cpusetp) < 0) {
 126     perror("sched_setaffinity dynamically allocated");
 127     CPU_FREE(cpusetp);
 128     return -1;
 129   }
 130   CPU_FREE(cpusetp);
 131  } else
 132 #endif
 133  {
 134   cpu_set_t cpuset;
 135   CPU_ZERO(&cpuset);
 136   CPU_SET(cpuid, &cpuset);
 137   /*SET_MASK(cpuid)*/
 138
 139   /* PID 0 refers to the current process */
 140   /*if (sched_setaffinity(0, len, &mask) < 0) {*/
 141   if (sched_setaffinity(0, sizeof(cpuset), &cpuset) < 0) {
 142     perror("sched_setaffinity");
 143     return -1;
 144   }
 145  }
 146 #endif
 147
 148   return 0;
 149 }
 150
 151 #if CMK_SMP
 152 int set_thread_affinity(int cpuid) {
 153   unsigned long mask = 0xffffffff;
 154   unsigned int len = sizeof(mask);
 155
 156 #ifdef _WIN32
 157   HANDLE hThread;
 158 #endif
 159
 160 #ifdef _WIN32
 161   SET_MASK(cpuid)
 162   hThread = GetCurrentThread();
 163   if (SetThreadAffinityMask(hThread, mask) == 0) {
 164     return -1;
 165   }
 166 #elif  CMK_HAS_PTHREAD_SETAFFINITY
 167 #ifdef CPU_ALLOC
 168  if ( cpuid >= CPU_SETSIZE ) {
 169   cpu_set_t *cpusetp;
 170   pthread_t thread;
 171   size_t size;
 172   int num_cpus;
 173   num_cpus = cpuid + 1;
 174   cpusetp = CPU_ALLOC(num_cpus);
 175   if (cpusetp == NULL) {
 176     perror("set_thread_affinity CPU_ALLOC");
 177     return -1;
 178   }
 179   size = CPU_ALLOC_SIZE(num_cpus);
 180   thread = pthread_self();
 181   CPU_ZERO_S(size, cpusetp);
 182   CPU_SET_S(cpuid, size, cpusetp);
 183   if (pthread_setaffinity_np(thread, size, cpusetp)) {
 184     perror("pthread_setaffinity dynamically allocated");
 185     CPU_FREE(cpusetp);
 186     return -1;
 187   }
 188   CPU_FREE(cpusetp);
 189  } else
 190 #endif
 191  {
 192   int s, j;
 193   cpu_set_t cpuset;
 194   pthread_t thread;
 195
 196   thread = pthread_self();
 197
 198   CPU_ZERO(&cpuset);
 199   CPU_SET(cpuid, &cpuset);
 200
 201   s = pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
 202   if (s != 0) {
 203     perror("pthread_setaffinity");
 204     return -1;
 205   }
 206  }
 207 #elif CMK_HAS_BINDPROCESSOR
 208   if (bindprocessor(BINDTHREAD, thread_self(), cpuid) != 0)
 209     return -1;
 210 #else
 211   return set_cpu_affinity(cpuid);
 212 #endif
 213
 214   return 0;
 215 }
 216 #endif
 217
 218
 219 int CmiSetCPUAffinity(int mycore)
 220 {
 221   int core = mycore;
 222   if (core < 0) {
 223     core = CmiNumCores() + core;
 224   }
 225   if (core < 0) {
 226     CmiError("Error: Invalid cpu affinity core number: %d\n", mycore);
 227     CmiAbort("CmiSetCPUAffinity failed");
 228   }
 229
 230   CpvAccess(myCPUAffToCore) = core;
 231
 232   /* set cpu affinity */
 233 #if CMK_SMP
 234   return set_thread_affinity(core);
 235 #else
 236   return set_cpu_affinity(core);
 237   /* print_cpu_affinity(); */
 238 #endif
 239 }
 240
 241 /* This implementation assumes the default x86 CPU mask size used by Linux */
 242 /* For a large SMP machine, this code should be changed to use a variable sized   */
 243 /* CPU affinity mask buffer instead, as the present code will fail beyond 32 CPUs */
 244 int print_cpu_affinity() {
 245 #ifdef _WIN32
 246   unsigned long pMask, sMask;
 247   HANDLE hProcess = GetCurrentProcess();
 248   if(GetProcessAffinityMask(hProcess, &pMask, &sMask)){
 249         perror("On Windows: GetProcessAffinityMask");
 250     return -1;
 251   }
 252
 253  CmiPrintf("[%d] CPU affinity mask is: 0x%08lx\n", CmiMyPe(), pMask);
 254
 255 #elif CMK_HAS_BINDPROCESSOR
 256   printf("[%d] CPU affinity mask is unknown for AIX. \n", CmiMyPe());
 257 #else
 258   /*unsigned long mask;
 259   unsigned int len = sizeof(mask);*/
 260   cpu_set_t cpuset;
 261   char str[256], pe[16];
 262   int i;
 263   CPU_ZERO(&cpuset);
 264
 265   /* PID 0 refers to the current process */
 266   /*if (sched_getaffinity(0, len, &mask) < 0) {*/
 267   if (sched_getaffinity(0, sizeof(cpuset), &cpuset) < 0) {
 268     perror("sched_getaffinity");
 269     return -1;
 270   }
 271
 272   sprintf(str, "[%d] CPU affinity mask is: ", CmiMyPe());
 273   for (i = 0; i < CPU_SETSIZE; i++)
 274         if (CPU_ISSET(i, &cpuset)) {
 275             sprintf(pe, " %d ", i);
 276             strcat(str, pe);
 277         }
 278   CmiPrintf("%s\n", str);
 279 #endif
 280   return 0;
 281 }
 282
 283 #if CMK_SMP
 284 int print_thread_affinity() {
 285   unsigned long mask;
 286   size_t len = sizeof(mask);
 287
 288 #if  CMK_HAS_PTHREAD_SETAFFINITY
 289   int s, j;
 290   cpu_set_t cpuset;
 291   pthread_t thread;
 292   char str[256], pe[16];
 293
 294   thread = pthread_self();
 295   s = pthread_getaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
 296   if (s != 0) {
 297     perror("pthread_getaffinity");
 298     return -1;
 299   }
 300
 301   sprintf(str, "[%d] %s affinity is: ", CmiMyPe(), CmiMyPe()>=CmiNumPes()?"communication pthread":"pthread");
 302   for (j = 0; j < CPU_SETSIZE; j++)
 303         if (CPU_ISSET(j, &cpuset)) {
 304             sprintf(pe, " %d ", j);
 305             strcat(str, pe);
 306         }
 307   CmiPrintf("%s\n", str);
 308 #endif
 309   return 0;
 310 }
 311 #endif
 312
 313 int CmiPrintCPUAffinity()
 314 {
 315 #if CMK_SMP
 316   return print_thread_affinity();
 317 #else
 318   return print_cpu_affinity();
 319 #endif
 320 }
 321
 322 int CmiOnCore() {
 323 #if CMK_OS_IS_LINUX
 324   /*
 325    * The info (task_cpu) is read from the Linux /proc virtual file system.
 326    * The /proc/<PID>/[task/<TID>]/stat is explained in the Linux
 327    * kernel documentation. The online one could be found in:
 328    * http://www.mjmwired.net/kernel/Documentation/filesystems/proc.txt
 329    * Based on the documentation, task_cpu is found at the 39th field in
 330    * the stat file.
 331    **/
 332 #define TASK_CPU_POS (39)
 333   int n;
 334   char str[128];
 335   FILE *fp = (FILE *)CpvAccess(myProcStatFP);
 336   if (fp == NULL){
 337     printf("WARNING: CmiOnCore IS NOT SUPPORTED ON THIS PLATFORM\n");
 338     return -1;
 339   }
 340   fseek(fp, 0, SEEK_SET);
 341   for (n=0; n<TASK_CPU_POS; n++)  {
 342     fscanf(fp, "%s", str);
 343   }
 344   return atoi(str);
 345 #else
 346   printf("WARNING: CmiOnCore IS NOT SUPPORTED ON THIS PLATFORM\n");
 347   return -1;
 348 #endif
 349 }
 350
 351
 352 static int cpuAffinityHandlerIdx;
 353 static int cpuAffinityRecvHandlerIdx;
 354
 355 typedef struct _hostnameMsg {
 356   char core[CmiMsgHeaderSizeBytes];
 357   int pe;
 358   skt_ip_t ip;
 359   int ncores;
 360   int rank;
 361   int seq;
 362 } hostnameMsg;
 363
 364 typedef struct _rankMsg {
 365   char core[CmiMsgHeaderSizeBytes];
 366   int *ranks;                  /* PE => core rank mapping */
 367   int *nodes;                  /* PE => node number mapping */
 368 } rankMsg;
 369
 370 static rankMsg *rankmsg = NULL;
 371 static CmmTable hostTable;
 372 static CmiNodeLock affLock = 0;
 373
 374 /* called on PE 0 */
 375 static void cpuAffinityHandler(void *m)
 376 {
 377   static int count = 0;
 378   static int nodecount = 0;
 379   hostnameMsg *rec;
 380   hostnameMsg *msg = (hostnameMsg *)m;
 381   hostnameMsg *tmpm;
 382   int tag, tag1, pe, myrank;
 383   int npes = CmiNumPes();
 384
 385 /*   for debug
 386   char str[128];
 387   skt_print_ip(str, msg->ip);
 388   printf("hostname: %d %s\n", msg->pe, str);
 389 */
 390   CmiAssert(CmiMyPe()==0 && rankmsg != NULL);
 391   tag = *(int*)&msg->ip;
 392   pe = msg->pe;
 393   if ((rec = (hostnameMsg *)CmmProbe(hostTable, 1, &tag, &tag1)) != NULL) {
 394     CmiFree(msg);
 395   }
 396   else {
 397     rec = msg;
 398     rec->seq = nodecount;
 399     nodecount++;                          /* a new node record */
 400     CmmPut(hostTable, 1, &tag, msg);
 401   }
 402   myrank = rec->rank%rec->ncores;
 403   while (in_exclude(myrank)) {             /* skip excluded core */
 404     myrank = (myrank+1)%rec->ncores;
 405     rec->rank ++;
 406   }
 407   rankmsg->ranks[pe] = myrank;             /* core rank */
 408   rankmsg->nodes[pe] = rec->seq;           /* on which node */
 409   rec->rank ++;
 410   count ++;
 411   if (count == CmiNumPes()) {
 412     /* CmiPrintf("Cpuaffinity> %d unique compute nodes detected! \n", CmmEntries(hostTable)); */
 413     tag = CmmWildCard;
 414     while (tmpm = CmmGet(hostTable, 1, &tag, &tag1)) CmiFree(tmpm);
 415     CmmFree(hostTable);
 416 #if 1
 417     /* bubble sort ranks on each node according to the PE number */
 418     {
 419     int i,j;
 420     for (i=0; i<npes-1; i++)
 421       for(j=i+1; j<npes; j++) {
 422         if (rankmsg->nodes[i] == rankmsg->nodes[j] &&
 423               rankmsg->ranks[i] > rankmsg->ranks[j])
 424         {
 425           int tmp = rankmsg->ranks[i];
 426           rankmsg->ranks[i] = rankmsg->ranks[j];
 427           rankmsg->ranks[j] = tmp;
 428         }
 429       }
 430     }
 431 #endif
 432     CmiSyncBroadcastAllAndFree(sizeof(rankMsg)+CmiNumPes()*sizeof(int)*2, (void *)rankmsg);
 433   }
 434 }
 435
 436 /* called on each processor */
 437 static void cpuAffinityRecvHandler(void *msg)
 438 {
 439   int myrank, mynode;
 440   rankMsg *m = (rankMsg *)msg;
 441   m->ranks = (int *)((char*)m + sizeof(rankMsg));
 442   m->nodes = (int *)((char*)m + sizeof(rankMsg) + CmiNumPes()*sizeof(int));
 443   myrank = m->ranks[CmiMyPe()];
 444   mynode = m->nodes[CmiMyPe()];
 445
 446   /*CmiPrintf("[%d %d] set to core #: %d\n", CmiMyNode(), CmiMyPe(), myrank);*/
 447
 448   if (-1 != CmiSetCPUAffinity(myrank)) {
 449     DEBUGP(("Processor %d is bound to core #%d on node #%d\n", CmiMyPe(), myrank, mynode));
 450   }
 451   else{
 452     CmiPrintf("Processor %d set affinity failed!\n", CmiMyPe());
 453     CmiAbort("set cpu affinity abort!\n");
 454   }
 455   CmiFree(m);
 456 }
 457
 458 #if defined(_WIN32) && ! defined(__CYGWIN__)
 459   /* strtok is thread safe in VC++ */
 460 #define strtok_r(x,y,z) strtok(x,y)
 461 #endif
 462
 463 static int search_pemap(char *pecoremap, int pe)
 464 {
 465   int *map = (int *)malloc(CmiNumPesGlobal()*sizeof(int));
 466   char *ptr = NULL;
 467   int i, j, k, count;
 468   char *str;
 469
 470   char *mapstr = (char*)malloc(strlen(pecoremap)+1);
 471   strcpy(mapstr, pecoremap);
 472
 473   str = strtok_r(mapstr, ",", &ptr);
 474   count = 0;
 475   while (str && count < CmiNumPesGlobal())
 476   {
 477       int hasdash=0, hascolon=0, hasdot=0, hasstar1=0, hasstar2 = 0;
 478       int start, end, stride=1, block=1;
 479       int iter=1;
 480       for (i=0; i<strlen(str); i++) {
 481           if (str[i] == '-' && i!=0) hasdash=1;
 482           else if (str[i] == ':') hascolon=1;
 483           else if (str[i] == '.') hasdot=1;
 484           else if (str[i] == 'x') hasstar1=1;
 485           else if (str[i] == 'X') hasstar2=1;
 486       }
 487       if (hasstar1 || hasstar2) {
 488           if (hasstar1) sscanf(str, "%dx", &iter);
 489           if (hasstar2) sscanf(str, "%dX", &iter);
 490           while (*str!='x' && *str!='X') str++;
 491           str++;
 492       }
 493       if (hasdash) {
 494           if (hascolon) {
 495             if (hasdot) {
 496               if (sscanf(str, "%d-%d:%d.%d", &start, &end, &stride, &block) != 4)
 497                  printf("Warning: Check the format of \"%s\".\n", str);
 498             }
 499             else {
 500               if (sscanf(str, "%d-%d:%d", &start, &end, &stride) != 3)
 501                  printf("Warning: Check the format of \"%s\".\n", str);
 502             }
 503           }
 504           else {
 505             if (sscanf(str, "%d-%d", &start, &end) != 2)
 506                  printf("Warning: Check the format of \"%s\".\n", str);
 507           }
 508       }
 509       else {
 510           sscanf(str, "%d", &start);
 511           end = start;
 512       }
 513       if (block > stride) {
 514         printf("Warning: invalid block size in \"%s\" ignored.\n", str);
 515         block=1;
 516       }
 517       //if (CmiMyPe() == 0) printf("iter: %d start: %d end: %d stride: %d, block: %d.\n", iter, start, end, stride, block);
 518       for (k = 0; k<iter; k++) {
 519         for (i = start; i<=end; i+=stride) {
 520           for (j=0; j<block; j++) {
 521             if (i+j>end) break;
 522             map[count++] = i+j;
 523             if (count == CmiNumPesGlobal()) break;
 524           }
 525           if (count == CmiNumPesGlobal()) break;
 526         }
 527         if (count == CmiNumPesGlobal()) break;
 528       }
 529       str = strtok_r(NULL, ",", &ptr);
 530   }
 531   i = map[pe % count];
 532
 533   free(map);
 534   free(mapstr);
 535   return i;
 536 }
 537
 538 #if CMK_CRAYXT || CMK_CRAYXE || CMK_CRAYXC
 539 extern int getXTNodeID(int mpirank, int nummpiranks);
 540 #endif
 541
 542
 543 void CmiInitCPUAffinity(char **argv)
 544 {
 545   static skt_ip_t myip;
 546   int ret, i, exclude;
 547   hostnameMsg  *msg;
 548   char *pemap = NULL;
 549   char *commap = NULL;
 550   char *pemapfile = NULL;
 551
 552   int show_affinity_flag;
 553   int affinity_flag = CmiGetArgFlagDesc(argv,"+setcpuaffinity",
 554                                                 "set cpu affinity");
 555
 556   while (CmiGetArgIntDesc(argv,"+excludecore", &exclude, "avoid core when setting cpuaffinity"))  {
 557     if (CmiMyRank() == 0) add_exclude(exclude);
 558     affinity_flag = 1;
 559   }
 560
 561   if (CmiGetArgStringDesc(argv, "+pemapfile", &pemapfile, "define pe to core mapping file")) {
 562     FILE *fp;
 563     char buf[128];
 564     pemap = (char*)malloc(1024);
 565     fp = fopen(pemapfile, "r");
 566     if (fp == NULL) CmiAbort("pemapfile does not exist");
 567     while (!feof(fp)) {
 568       if (fgets(buf, 128, fp)) {
 569         if (buf[strlen(buf)-1] == '\n') buf[strlen(buf)-1] = 0;
 570         strcat(pemap, buf);
 571       }
 572     }
 573     fclose(fp);
 574     if (CmiMyPe()==0) CmiPrintf("Charm++> read from pemap file '%s': %s\n", pemapfile, pemap);
 575   }
 576
 577   CmiGetArgStringDesc(argv, "+pemap", &pemap, "define pe to core mapping");
 578   if (pemap!=NULL && excludecount>0)
 579     CmiAbort("Charm++> +pemap can not be used with +excludecore.\n");
 580
 581   CmiGetArgStringDesc(argv, "+commap", &commap, "define comm threads to core mapping");
 582
 583   if (pemap!=NULL || commap!=NULL) affinity_flag = 1;
 584
 585   show_affinity_flag = CmiGetArgFlagDesc(argv,"+showcpuaffinity",
 586                                                 "print cpu affinity");
 587
 588   cpuAffinityHandlerIdx =
 589        CmiRegisterHandler((CmiHandler)cpuAffinityHandler);
 590   cpuAffinityRecvHandlerIdx =
 591        CmiRegisterHandler((CmiHandler)cpuAffinityRecvHandler);
 592
 593   if (CmiMyRank() ==0) {
 594      affLock = CmiCreateLock();
 595   }
 596
 597 #if CMK_BLUEGENEP | CMK_BLUEGENEQ
 598   if(affinity_flag){
 599       affinity_flag = 0;
 600       if(CmiMyPe()==0) CmiPrintf("Charm++> cpu affinity setting is not needed on BG/P, thus ignored.\n");
 601   }
 602   if(show_affinity_flag){
 603       show_affinity_flag = 0;
 604       if(CmiMyPe()==0) CmiPrintf("Charm++> printing cpu affinity is not supported on BG/P.\n");
 605   }
 606 #endif
 607
 608   if (!affinity_flag) {
 609     if (show_affinity_flag) CmiPrintCPUAffinity();
 610     return;
 611   }
 612
 613   if (CmiMyPe() == 0) {
 614      CmiPrintf("Charm++> cpu affinity enabled. \n");
 615      if (excludecount > 0) {
 616        CmiPrintf("Charm++> cpuaffinity excludes core: %d", excludecore[0]);
 617        for (i=1; i<excludecount; i++) CmiPrintf(" %d", excludecore[i]);
 618        CmiPrintf(".\n");
 619      }
 620      if (pemap!=NULL)
 621        CmiPrintf("Charm++> cpuaffinity PE-core map : %s\n", pemap);
 622   }
 623
 624   if (CmiMyPe() >= CmiNumPes()) {         /* this is comm thread */
 625       /* comm thread either can float around, or pin down to the last rank.
 626          however it seems to be reportedly slower if it is floating */
 627     CmiNodeAllBarrier();
 628     if (commap != NULL) {
 629       int mycore = search_pemap(commap, CmiMyPeGlobal()-CmiNumPesGlobal());
 630       if(CmiMyPe()-CmiNumPes()==0) printf("Charm++> set comm %d on node %d to core #%d\n", CmiMyPe()-CmiNumPes(), CmiMyNode(), mycore);
 631       if (-1 == CmiSetCPUAffinity(mycore))
 632         CmiAbort("set_cpu_affinity abort!");
 633       CmiNodeAllBarrier();
 634       if (show_affinity_flag) CmiPrintCPUAffinity();
 635       return;    /* comm thread return */
 636     }
 637     else {
 638     /* if (CmiSetCPUAffinity(CmiNumCores()-1) == -1) CmiAbort("set_cpu_affinity abort!"); */
 639 #if !CMK_CRAYXT && !CMK_CRAYXE && !CMK_CRAYXC && !CMK_BLUEGENEQ
 640       if (pemap == NULL) {
 641 #if CMK_MACHINE_PROGRESS_DEFINED
 642         while (affinity_doneflag < CmiMyNodeSize())  CmiNetworkProgress();
 643 #else
 644 #if CMK_SMP
 645        #error "Machine progress call needs to be implemented for cpu affinity!"
 646 #endif
 647 #endif
 648       }
 649 #endif
 650 #if CMK_CRAYXT || CMK_CRAYXE || CMK_CRAYXC
 651       /* if both pemap and commmap are NULL, will compute one */
 652       if (pemap != NULL)
 653 #endif
 654       {
 655       CmiNodeAllBarrier();
 656       if (show_affinity_flag) CmiPrintCPUAffinity();
 657       return;    /* comm thread return */
 658       }
 659     }
 660   }
 661
 662   if (pemap != NULL && CmiMyPe()<CmiNumPes()) {    /* work thread */
 663     int mycore = search_pemap(pemap, CmiMyPeGlobal());
 664     if(show_affinity_flag) CmiPrintf("Charm++> set PE %d on node %d to core #%d\n", CmiMyPe(), CmiMyNode(), mycore);
 665     if (mycore >= CmiNumCores()) {
 666       CmiPrintf("Error> Invalid core number %d, only have %d cores (0-%d) on the node. \n", mycore, CmiNumCores(), CmiNumCores()-1);
 667       CmiAbort("Invalid core number");
 668     }
 669     if (CmiSetCPUAffinity(mycore) == -1) CmiAbort("set_cpu_affinity abort!");
 670     CmiNodeAllBarrier();
 671     CmiNodeAllBarrier();
 672     /* if (show_affinity_flag) CmiPrintCPUAffinity(); */
 673     return;
 674   }
 675
 676 #if CMK_CRAYXT || CMK_CRAYXE || CMK_CRAYXC
 677   {
 678     int numCores = CmiNumCores();
 679
 680     int myid = getXTNodeID(CmiMyNodeGlobal(), CmiNumNodesGlobal());
 681     int myrank;
 682     int pe, mype = CmiMyPeGlobal();
 683     int node = CmiMyNodeGlobal();
 684     int nnodes = 0;
 685 #if CMK_SMP
 686     if (CmiMyPe() >= CmiNumPes()) {         /* this is comm thread */
 687       int node = CmiMyPe() - CmiNumPes();
 688       mype = CmiGetPeGlobal(CmiNodeFirst(node) + CmiMyNodeSize() - 1, CmiMyPartition()); /* last pe on SMP node */
 689       node = CmiGetNodeGlobal(node, CmiMyPartition());
 690     }
 691 #endif
 692     pe = mype - 1;
 693     while (pe >= 0) {
 694       int n = CmiNodeOf(pe);
 695       if (n != node) { nnodes++; node = n; }
 696       if (getXTNodeID(n, CmiNumNodesGlobal()) != myid) break;
 697       pe --;
 698     }
 699     CmiAssert(numCores > 0);
 700     myrank = (mype - pe - 1 + nnodes)%numCores;
 701 #if CMK_SMP
 702     if (CmiMyPe() >= CmiNumPes())
 703         myrank = (myrank + 1)%numCores;
 704 #endif
 705
 706     if (-1 != CmiSetCPUAffinity(myrank)) {
 707       DEBUGP(("Processor %d is bound to core #%d on node #%d\n", CmiMyPe(), myrank, mynode));
 708     }
 709     else{
 710       CmiPrintf("Processor %d set affinity failed!\n", CmiMyPe());
 711       CmiAbort("set cpu affinity abort!\n");
 712     }
 713   }
 714   if (CmiMyPe() < CmiNumPes())
 715   CmiNodeAllBarrier();
 716   CmiNodeAllBarrier();
 717 #else
 718     /* get my ip address */
 719   if (CmiMyRank() == 0)
 720   {
 721 #if CMK_HAS_GETHOSTNAME
 722     myip = skt_my_ip();        /* not thread safe, so only calls on rank 0 */
 723 #else
 724     CmiAbort("Can not get unique name for the compute nodes. \n");
 725 #endif
 726   }
 727   CmiNodeAllBarrier();
 728
 729     /* prepare a msg to send */
 730   msg = (hostnameMsg *)CmiAlloc(sizeof(hostnameMsg));
 731   CmiSetHandler((char *)msg, cpuAffinityHandlerIdx);
 732   msg->pe = CmiMyPe();
 733   msg->ip = myip;
 734   msg->ncores = CmiNumCores();
 735   DEBUGP(("PE %d's node has %d number of cores. \n", CmiMyPe(), msg->ncores));
 736   msg->rank = 0;
 737   CmiSyncSendAndFree(0, sizeof(hostnameMsg), (void *)msg);
 738
 739   if (CmiMyPe() == 0) {
 740     int i;
 741     hostTable = CmmNew();
 742     rankmsg = (rankMsg *)CmiAlloc(sizeof(rankMsg)+CmiNumPes()*sizeof(int)*2);
 743     CmiSetHandler((char *)rankmsg, cpuAffinityRecvHandlerIdx);
 744     rankmsg->ranks = (int *)((char*)rankmsg + sizeof(rankMsg));
 745     rankmsg->nodes = (int *)((char*)rankmsg + sizeof(rankMsg) + CmiNumPes()*sizeof(int));
 746     for (i=0; i<CmiNumPes(); i++) {
 747       rankmsg->ranks[i] = 0;
 748       rankmsg->nodes[i] = -1;
 749     }
 750
 751     for (i=0; i<CmiNumPes(); i++) CmiDeliverSpecificMsg(cpuAffinityHandlerIdx);
 752   }
 753
 754     /* receive broadcast from PE 0 */
 755   CmiDeliverSpecificMsg(cpuAffinityRecvHandlerIdx);
 756   CmiLock(affLock);
 757   affinity_doneflag++;
 758   CmiUnlock(affLock);
 759   CmiNodeAllBarrier();
 760 #endif
 761
 762   if (show_affinity_flag) CmiPrintCPUAffinity();
 763 }
 764
 765 /* called in ConverseCommonInit to initialize basic variables */
 766 void CmiInitCPUAffinityUtil(){
 767     char fname[64];
 768     CpvInitialize(int, myCPUAffToCore);
 769     CpvAccess(myCPUAffToCore) = -1;
 770 #if CMK_OS_IS_LINUX
 771     CpvInitialize(void *, myProcStatFP);
 772     CmiLock(_smp_mutex);
 773 #if CMK_SMP
 774     sprintf(fname, "/proc/%d/task/%d/stat", getpid(), syscall(SYS_gettid));
 775 #else
 776     sprintf(fname, "/proc/%d/stat", getpid());
 777 #endif
 778     CpvAccess(myProcStatFP) = (void *)fopen(fname, "r");
 779     CmiUnlock(_smp_mutex);
 780 /*
 781     if(CmiMyPe()==0 && CpvAccess(myProcStatFP) == NULL){
 782         CmiPrintf("WARNING: ERROR IN OPENING FILE %s on PROC %d, CmiOnCore() SHOULDN'T BE CALLED\n", fname, CmiMyPe());
 783     }
 784 */
 785 #endif
 786 }
 787
 788 #else           /* not supporting affinity */
 789
 790
 791 int CmiPrintCPUAffinity()
 792 {
 793   CmiPrintf("Warning: CmiPrintCPUAffinity not supported.\n");
 794 }
 795
 796 void CmiInitCPUAffinity(char **argv)
 797 {
 798   char *pemap = NULL;
 799   char *pemapfile = NULL;
 800   char *commap = NULL;
 801   int excludecore = -1;
 802   int affinity_flag = CmiGetArgFlagDesc(argv,"+setcpuaffinity",
 803                                                 "set cpu affinity");
 804   while (CmiGetArgIntDesc(argv,"+excludecore",&excludecore, "avoid core when setting cpuaffinity"));
 805   CmiGetArgStringDesc(argv, "+pemap", &pemap, "define pe to core mapping");
 806   CmiGetArgStringDesc(argv, "+pemapfile", &pemapfile, "define pe to core mapping file");
 807   CmiGetArgStringDesc(argv, "+commap", &commap, "define comm threads to core mapping");
 808   if (affinity_flag && CmiMyPe()==0)
 809     CmiPrintf("sched_setaffinity() is not supported, +setcpuaffinity disabled.\n");
 810 }
 811
 812 /* called in ConverseCommonInit to initialize basic variables */
 813 void CmiInitCPUAffinityUtil(){
 814     CpvInitialize(int, myCPUAffToCore);
 815     CpvAccess(myCPUAffToCore) = -1;
 816 #if CMK_OS_IS_LINUX
 817     CpvInitialize(void *, myProcStatFP);
 818     CpvAccess(myProcStatFP) = NULL;
 819  #endif
 820 }
 821
 822 int CmiOnCore(){
 823   printf("WARNING: CmiOnCore IS NOT SUPPORTED ON THIS PLATFORM\n");
 824   return -1;
 825 }
 826 #endif