tests/converse/commbench/pingpong-cachemiss.c

   1 #include <stdlib.h>
   2 #include <converse.h>
   3 #include "commbench.h"
   4
   5 #define pva CpvAccess
   6 #define pvd CpvStaticDeclare
   7 #define pvi CpvInitialize
   8
   9 static struct testdata {
  10   int size;
  11   int numiter;
  12 } sizes[] = {
  13     {16,      4000},
  14     {32,      4000},
  15     {128,      4000},
  16     {256,     1000},
  17     {512,     1000},
  18     {1024,    1000},
  19     {2048,    1000},
  20     {4096,    1000},
  21     {8192,    1000},
  22     {16384,   1000},
  23     {32768,   100},
  24     {65536,   100},
  25     {131072,   100},
  26     {524288,   100},
  27     {1048576, 40},
  28     {4194304, 10},
  29     {-1,      -1},
  30 };
  31
  32 typedef struct message_{
  33   char core[CmiMsgHeaderSizeBytes];
  34   int srcpe;
  35   int idx;
  36   int iter;
  37   int data[1];
  38 } Message;
  39
  40 char     *buffer_msgs;
  41
  42 static void fillMessage(Message *msg)
  43 {
  44   int i, size, items;
  45   size = sizes[msg->idx].size + sizeof(double);
  46   items = size/sizeof(double);
  47   for(i=0;i<items;i++)
  48     msg->data[i] = i+0x1234;
  49 }
  50
  51 static void checkMessage(Message *msg)
  52 {
  53   int i, size, items;
  54   size = sizes[msg->idx].size + sizeof(double);
  55   items = size/sizeof(double);
  56   for(i=0;i<items;i++) {
  57     if(msg->data[i] != (i+0x1234))
  58       CmiAbort("[pingpong] Data corrupted. Run megacon !!\n");
  59   }
  60 }
  61
  62 typedef struct Time_{
  63   char core[CmiMsgHeaderSizeBytes];
  64   int srcNode;
  65   double data[1];
  66 } TimeMessage;
  67
  68 pvd(int, nextIter);
  69 pvd(int, nextSize);
  70 pvd(int, nextNbr);
  71
  72 pvd(double, starttime);
  73 pvd(double, endtime);
  74 pvd(double **, times);/* stores times for all nbrs and sizes */
  75 pvd(int *, nodeList); /* stores nums of pes rank 0 on all nodes*/
  76 pvd(double *, gavg);
  77 pvd(double *, gmax);
  78 pvd(double *, gmin);
  79 pvd(int *, gmaxSrc);
  80 pvd(int *, gmaxDest);
  81 pvd(int *, gminSrc);
  82 pvd(int *, gminDest);
  83
  84 pvd(int, numSizes);    /* how many test sizes exist */
  85 pvd(int, numRecv);
  86
  87 pvd(int, timeHandler);
  88 pvd(int, nodeHandler);
  89 pvd(int, nbrHandler);
  90 pvd(int, sizeHandler);
  91 pvd(int, iterHandler);
  92 pvd(int, bounceHandler);
  93 pvd(int, setupHandler);
  94 pvd(int, startHandler);
  95
  96 static void recvTime(TimeMessage *msg)
  97 {
  98   EmptyMsg m;
  99   int i, j;
 100   double time;
 101
 102   pva(numRecv)++;
 103   for(i=0;i<CmiNumNodes();i++) {
 104     if(i==msg->srcNode)
 105       continue;
 106     for(j=0;j<pva(numSizes);j++) {
 107       time = *(msg->data+i*pva(numSizes)+j);
 108       pva(gavg)[j] += time;
 109       if(time > pva(gmax)[j]) {
 110         pva(gmax)[j] = time;
 111         pva(gmaxSrc)[j] = msg->srcNode;
 112         pva(gmaxDest)[j] = i;
 113       }
 114       if(time < pva(gmin)[j]) {
 115         pva(gmin)[j] = time;
 116         pva(gminSrc)[j] = msg->srcNode;
 117         pva(gminDest)[j] = i;
 118       }
 119     }
 120   }
 121   if(pva(numRecv)==CmiNumNodes()){
 122     for(j=0;j<pva(numSizes);j++)
 123       pva(gavg)[j] /= (CmiNumNodes()*(CmiNumNodes()-1));
 124     for(j=0;j<pva(numSizes);j++) {
 125       CmiPrintf("%d\t\t%le\n",
 126                             sizes[j].size, pva(gavg)[j]);
 127       //CmiPrintf("[pingpong] size=%d\tmaxTime=%le seconds\t[%d->%d]\n",
 128       //            sizes[j].size, pva(gmax)[j],pva(gmaxSrc)[j],pva(gmaxDest)[j]);
 129       //CmiPrintf("[pingpong] size=%d\tminTime=%le seconds\t[%d->%d]\n",
 130       //            sizes[j].size, pva(gmin)[j],pva(gminSrc)[j],pva(gminDest)[j]);
 131     }
 132     CmiSetHandler(&m, pva(ack_handler));
 133     CmiSyncSend(0, sizeof(EmptyMsg), &m);
 134   }
 135   CmiFree(msg);
 136 }
 137
 138 static void startNextNode(EmptyMsg *msg)
 139 {
 140   EmptyMsg m;
 141   CmiFree(msg);
 142   if((CmiMyNode()+1) != CmiNumNodes()) {
 143     CmiSetHandler(&m, pva(nbrHandler));
 144     CmiSyncSend(pva(nodeList)[CmiMyNode()+1], sizeof(EmptyMsg), &m);
 145   }
 146 }
 147
 148 static void startNextNbr(EmptyMsg *msg)
 149 {
 150   EmptyMsg m;
 151   TimeMessage *tm;
 152   int i, size;
 153
 154 //CmiAssert(CmiMyPe()==0);
 155
 156   CmiFree(msg);
 157   pva(nextNbr)++;
 158   if(pva(nextNbr) == CmiMyNode()) {
 159     CmiSetHandler(&m, pva(nbrHandler));
 160     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 161     return;
 162   }
 163   if(pva(nextNbr) == CmiNumNodes()) {
 164     pva(nextNbr) = -1;
 165     CmiSetHandler(&m, pva(nodeHandler));
 166     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 167     size = sizeof(TimeMessage)+pva(numSizes)*CmiNumNodes()*sizeof(double);
 168     tm = (TimeMessage *) CmiAlloc(size);
 169     for(i=0;i<CmiNumNodes();i++)
 170       memcpy(tm->data+i*pva(numSizes),pva(times)[i],
 171              sizeof(double)*pva(numSizes));
 172     tm->srcNode = CmiMyNode();
 173     CmiSetHandler(tm, pva(timeHandler));
 174     CmiSyncSendAndFree(0, size, tm);
 175   } else {
 176     CmiSetHandler(&m, pva(sizeHandler));
 177     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 178   }
 179 }
 180
 181 static void startNextSize(EmptyMsg *msg)
 182 {
 183   EmptyMsg m;
 184   Message *mm;
 185   int i;
 186
 187  //CmiAssert(CmiMyPe()==0);
 188
 189   pva(nextSize)++;
 190   if(pva(nextSize) == pva(numSizes)) {
 191     pva(nextSize) = -1;
 192     CmiSetHandler(&m, pva(nbrHandler));
 193     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 194   } else {
 195     int size = sizeof(Message)+sizes[pva(nextSize)].size;
 196
 197     buffer_msgs = (char*)malloc((sizes[pva(nextSize)].numiter) * sizeof(Message*));
 198     for(i=0; i<sizes[pva(nextSize)].numiter; i++)
 199     {
 200         mm = CmiAlloc(size);
 201         mm->srcpe = CmiMyPe();
 202         mm->idx = pva(nextSize);
 203         CmiSetHandler(mm, pva(bounceHandler));
 204         //*((Message**)(buffer_msgs+i*sizeof(char*))) = mm;
 205         ((Message**)buffer_msgs)[i] = mm;
 206     }
 207     mm = (Message *) CmiAlloc(size);
 208     mm->srcpe = CmiMyPe();
 209     mm->idx = pva(nextSize);
 210     CmiSetHandler(mm, pva(setupHandler));
 211     //fillMessage(mm);
 212     CmiSyncSendAndFree(pva(nextNbr), size, mm);
 213   }
 214   CmiFree(msg);
 215 }
 216
 217 static void startNextIter(Message *msg)
 218 {
 219   EmptyMsg m;
 220   Message *mm;
 221   pva(nextIter)++;
 222   if(pva(nextIter) >= sizes[pva(nextSize)].numiter) {
 223     pva(endtime) = CmiWallTimer();
 224     //checkMessage(msg);
 225     pva(times)[pva(nextNbr)][pva(nextSize)] =
 226       (pva(endtime) - pva(starttime))/(2.0*sizes[pva(nextSize)].numiter);
 227     pva(nextIter) = -1;
 228     CmiSetHandler(&m, pva(sizeHandler));
 229     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 230     CmiFree(msg);
 231   } else {
 232       //mm = *((Message**)(buffer_msgs + pva(nextIter)*sizeof(char*)));
 233       mm = ((Message**)buffer_msgs)[pva(nextIter)];
 234       mm->iter = pva(nextIter);
 235       //CmiSetHandler(mm, pva(bounceHandler));
 236       CmiSyncSendAndFree(pva(nextNbr), sizeof(Message)+sizes[mm->idx].size, mm);
 237       CmiFree(msg);
 238   }
 239 }
 240
 241 static void bounceMessage(Message *msg)
 242 {
 243     Message *mm;
 244     int iter = msg->iter;
 245     mm = *((Message**)(buffer_msgs + iter*sizeof(char*)));
 246     CmiSetHandler(mm, pva(iterHandler));
 247 //CmiAssert(msg->srcpe==0);
 248     CmiSyncSendAndFree(msg->srcpe, sizeof(Message)+sizes[msg->idx].size, mm);
 249     CmiFree(msg);
 250 }
 251 static void setupMessage(Message *msg)
 252 {
 253     Message *mm;
 254     int     i, size;
 255
 256     int nextSize =  msg->idx;
 257     size = sizeof(Message)+sizes[nextSize].size;
 258     buffer_msgs = (char*)malloc((sizes[nextSize].numiter) * sizeof(Message*));
 259
 260     for(i=0; i<sizes[nextSize].numiter; i++)
 261     {
 262         mm = (Message*)CmiAlloc(size);
 263         mm->srcpe = CmiMyPe();
 264         CmiSetHandler(mm, pva(iterHandler));
 265         //mm->idx = pva(nextSize);
 266         //*((Message**)buffer_msgs+i*sizeof(char*)) = mm;
 267         ((Message**)buffer_msgs)[i] = mm;
 268     }
 269     mm = (Message *) CmiAlloc(size);
 270     mm->srcpe = CmiMyPe();
 271     mm->idx = nextSize;
 272     CmiSetHandler(mm, pva(startHandler));
 273     //fillMessage(mm);
 274 //CmiAssert(msg->srcpe == 0);
 275     CmiSyncSendAndFree(msg->srcpe, size, mm);
 276     CmiFree(msg);
 277 }
 278 static void startMessage(Message *msg)
 279 {
 280 #if 0
 281     EmptyMsg m;
 282
 283     CmiFree(msg);
 284
 285     CmiSetHandler(&m, pva(iterHandler));
 286     //fillMessage(mm);
 287     pva(starttime) = CmiWallTimer();
 288     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 289 #else
 290    Message *mm;
 291    int size;
 292    size = sizeof(Message)+sizes[pva(nextSize)].size;
 293    mm = (Message *) CmiAlloc(size);
 294    mm->srcpe = CmiMyPe();
 295    mm->idx = pva(nextSize);
 296    CmiSetHandler(mm, pva(iterHandler));
 297    //fillMessage(mm);
 298    pva(starttime) = CmiWallTimer();
 299    CmiSyncSendAndFree(mm->srcpe, size, mm);
 300    CmiFree(msg);
 301 #endif
 302 }
 303
 304
 305 void pingpong_init(void)
 306 {
 307   EmptyMsg m;
 308
 309   if(CmiNumNodes()==1) {
 310     CmiPrintf("[pingpong] This benchmark requires > 1 nodes.\n");
 311     CmiSetHandler(&m, pva(ack_handler));
 312     CmiSyncSend(0, sizeof(EmptyMsg), &m);
 313     return;
 314   }
 315   CmiSetHandler(&m, pva(nbrHandler));
 316   CmiSyncSend(0, sizeof(EmptyMsg), &m);
 317 }
 318
 319 void pingpong_moduleinit(void)
 320 {
 321   int i,j;
 322   pvi(int, numRecv);
 323   pva(numRecv) = 0;
 324   pvi(int, nextIter);
 325   pva(nextIter) = -1;
 326   pvi(int, nextSize);
 327   pva(nextSize) = -1;
 328   pvi(int, nextNbr);
 329   pva(nextNbr) = -1;
 330   pvi(double, starttime);
 331   pva(starttime) = 0.0;
 332   pvi(double, endtime);
 333   pva(endtime) = 0.0;
 334   pvi(int, numSizes);
 335   for(i=0; sizes[i].size != (-1); i++);
 336   pva(numSizes) = i;
 337   pvi(double **, times);
 338   pva(times) = (double **) malloc(CmiNumNodes()*sizeof(double *));
 339   for(i=0;i<CmiNumNodes();i++)
 340     pva(times)[i] = (double *) malloc(pva(numSizes)*sizeof(double));
 341   for(i=0;i<CmiNumNodes();i++)
 342     for(j=0;j<pva(numSizes);j++)
 343       pva(times)[i][j] = 0.0;
 344   pvi(int *, nodeList);
 345   pva(nodeList) = (int *) malloc(CmiNumNodes()*sizeof(int));
 346   for(i=0;i<CmiNumNodes();i++)
 347     pva(nodeList)[i] = CmiNodeFirst(i);
 348   pvi(double *, gavg);
 349   pva(gavg) = (double *) malloc(sizeof(double)*pva(numSizes));
 350   pvi(double *, gmax);
 351   pva(gmax) = (double *) malloc(sizeof(double)*pva(numSizes));
 352   pvi(double *, gmin);
 353   pva(gmin) = (double *) malloc(sizeof(double)*pva(numSizes));
 354   pvi(int *, gmaxSrc);
 355   pva(gmaxSrc) = (int *) malloc(sizeof(int)*pva(numSizes));
 356   pvi(int *, gmaxDest);
 357   pva(gmaxDest) = (int *) malloc(sizeof(int)*pva(numSizes));
 358   pvi(int *, gminSrc);
 359   pva(gminSrc) = (int *) malloc(sizeof(int)*pva(numSizes));
 360   pvi(int *, gminDest);
 361   pva(gminDest) = (int *) malloc(sizeof(int)*pva(numSizes));
 362   for(i=0;i<pva(numSizes);i++) {
 363     pva(gavg)[i] = 0.0;
 364     pva(gmax)[i] = 0.0;
 365     pva(gmin)[i] = 1000000000.0;
 366     pva(gmaxSrc)[i] = 0;
 367     pva(gmaxDest)[i] = 0;
 368     pva(gminSrc)[i] = 0;
 369     pva(gminDest)[i] = 0;
 370   }
 371   pvi(int, timeHandler);
 372   pva(timeHandler) = CmiRegisterHandler((CmiHandler)recvTime);
 373   pvi(int, nodeHandler);
 374   pva(nodeHandler) = CmiRegisterHandler((CmiHandler)startNextNode);
 375   pvi(int, nbrHandler);
 376   pva(nbrHandler) = CmiRegisterHandler((CmiHandler)startNextNbr);
 377   pvi(int, sizeHandler);
 378   pva(sizeHandler) = CmiRegisterHandler((CmiHandler)startNextSize);
 379   pvi(int, iterHandler);
 380   pva(iterHandler) = CmiRegisterHandler((CmiHandler)startNextIter);
 381   pvi(int, bounceHandler);
 382   pva(bounceHandler) = CmiRegisterHandler((CmiHandler)bounceMessage);
 383   pvi(int, setupHandler);
 384   pva(setupHandler) = CmiRegisterHandler((CmiHandler)setupMessage);
 385   pvi(int, startHandler);
 386   pva(startHandler) = CmiRegisterHandler((CmiHandler)startMessage);
 387 }