tests/converse/commbench/pingpong-cachemiss.c

   1 #include <stdlib.h>
   2 #include <converse.h>
   3 #include "commbench.h"
   4
   5 #define pva CpvAccess
   6 #define pvd CpvStaticDeclare
   7 #define pvi CpvInitialize
   8
   9 static struct testdata {
  10   int size;
  11   int numiter;
  12 } sizes[] = {
  13   {16, 4000},
  14   {128, 4000},
  15   {512, 1000},
  16   {2048, 1000},
  17   {8192, 1000},
  18   {32768, 100},
  19   {131072, 100},
  20   {1048576, 40},
  21   {4194304, 10},
  22   {-1, -1},
  23 };
  24
  25 typedef struct message_ {
  26   char core[CmiMsgHeaderSizeBytes];
  27   int srcpe;
  28   int idx;
  29   int iter;
  30   int data[1];
  31 } Message;
  32
  33
  34 #if 0
  35 static void fillMessage(Message* msg) {
  36   int i, size, items;
  37   size = sizes[msg->idx].size + sizeof(double);
  38   items = size / sizeof(double);
  39   for (i = 0; i < items; i++) msg->data[i] = i + 0x1234;
  40 }
  41
  42 static void checkMessage(Message* msg) {
  43   int i, size, items;
  44   size = sizes[msg->idx].size + sizeof(double);
  45   items = size / sizeof(double);
  46   for (i = 0; i < items; i++) {
  47     if (msg->data[i] != (i + 0x1234))
  48       CmiAbort("[pingpong] Data corrupted. Run megacon !!\n");
  49   }
  50 }
  51 #endif
  52
  53 typedef struct Time_ {
  54   char core[CmiMsgHeaderSizeBytes];
  55   int srcNode;
  56   double data[1];
  57 } TimeMessage;
  58
  59 pvd(int, nextIter);
  60 pvd(int, nextSize);
  61 pvd(int, nextNbr);
  62
  63 pvd(double, starttime);
  64 pvd(double, endtime);
  65 pvd(double**, times); /* stores times for all nbrs and sizes */
  66 pvd(int*, nodeList);  /* stores nums of pes rank 0 on all nodes*/
  67 pvd(double*, gavg);
  68 pvd(double*, gmax);
  69 pvd(double*, gmin);
  70 pvd(int*, gmaxSrc);
  71 pvd(int*, gmaxDest);
  72 pvd(int*, gminSrc);
  73 pvd(int*, gminDest);
  74
  75 pvd(int, numSizes); /* how many test sizes exist */
  76 pvd(int, numRecv);
  77
  78 pvd(int, timeHandler);
  79 pvd(int, nodeHandler);
  80 pvd(int, nbrHandler);
  81 pvd(int, sizeHandler);
  82 pvd(int, iterHandler);
  83 pvd(int, bounceHandler);
  84 pvd(int, setupHandler);
  85 pvd(int, startHandler);
  86
  87 pvd(char*, buffer_msgs);
  88
  89 static void recvTime(TimeMessage* msg) {
  90   EmptyMsg m;
  91   int i, j;
  92   double time;
  93
  94   CmiInitMsgHeader(m.core, sizeof(EmptyMsg));
  95   pva(numRecv)++;
  96   for (i = 0; i < CmiNumNodes(); i++) {
  97     if (i == msg->srcNode) continue;
  98     for (j = 0; j < pva(numSizes); j++) {
  99       time = *(msg->data + i * pva(numSizes) + j);
 100       pva(gavg)[j] += time;
 101       if (time > pva(gmax)[j]) {
 102         pva(gmax)[j] = time;
 103         pva(gmaxSrc)[j] = msg->srcNode;
 104         pva(gmaxDest)[j] = i;
 105       }
 106       if (time < pva(gmin)[j]) {
 107         pva(gmin)[j] = time;
 108         pva(gminSrc)[j] = msg->srcNode;
 109         pva(gminDest)[j] = i;
 110       }
 111     }
 112   }
 113   if (pva(numRecv) == CmiNumNodes()) {
 114     for (j = 0; j < pva(numSizes); j++)
 115       pva(gavg)[j] /= (CmiNumNodes() * (CmiNumNodes() - 1));
 116     for (j = 0; j < pva(numSizes); j++) {
 117       CmiPrintf("%d\t\t%le\n", sizes[j].size, pva(gavg)[j]);
 118
 119 #if 0
 120       CmiPrintf("[pingpong] size=%d\tmaxTime=%le seconds\t[%d->%d]\n",
 121           sizes[j].size,
 122           pva(gmax)[j],pva(gmaxSrc)[j],pva(gmaxDest)[j]);
 123       CmiPrintf("[pingpong] size=%d\tminTime=%le seconds\t[%d->%d]\n",
 124           sizes[j].size,
 125           pva(gmin)[j],pva(gminSrc)[j],pva(gminDest)[j]);
 126 #endif
 127
 128     }
 129     CmiSetHandler(&m, pva(ack_handler));
 130     CmiSyncSend(0, sizeof(EmptyMsg), &m);
 131   }
 132   CmiFree(msg);
 133 }
 134
 135 static void startNextNode(EmptyMsg* msg) {
 136   EmptyMsg m;
 137   CmiInitMsgHeader(m.core, sizeof(EmptyMsg));
 138   CmiFree(msg);
 139   if ((CmiMyNode() + 1) != CmiNumNodes()) {
 140     CmiSetHandler(&m, pva(nbrHandler));
 141     CmiSyncSend(pva(nodeList)[CmiMyNode() + 1], sizeof(EmptyMsg), &m);
 142   }
 143 }
 144
 145 static void startNextNbr(EmptyMsg* msg) {
 146   EmptyMsg m;
 147   TimeMessage* tm;
 148   int i, size;
 149
 150   /* CmiAssert(CmiMyPe()==0); */
 151
 152   CmiInitMsgHeader(m.core, sizeof(EmptyMsg));
 153   CmiFree(msg);
 154   pva(nextNbr)++;
 155   if (pva(nextNbr) == CmiMyNode()) {
 156     CmiSetHandler(&m, pva(nbrHandler));
 157     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 158     return;
 159   }
 160   if (pva(nextNbr) == CmiNumNodes()) {
 161     pva(nextNbr) = -1;
 162     CmiSetHandler(&m, pva(nodeHandler));
 163     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 164     size = sizeof(TimeMessage) + pva(numSizes) * CmiNumNodes() * sizeof(double);
 165     tm = (TimeMessage*)CmiAlloc(size);
 166     for (i = 0; i < CmiNumNodes(); i++)
 167       memcpy(tm->data + i * pva(numSizes), pva(times)[i],
 168           sizeof(double) * pva(numSizes));
 169     tm->srcNode = CmiMyNode();
 170     CmiSetHandler(tm, pva(timeHandler));
 171     CmiSyncSendAndFree(0, size, tm);
 172   } else {
 173     CmiSetHandler(&m, pva(sizeHandler));
 174     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 175   }
 176 }
 177
 178 static void startNextSize(EmptyMsg* msg) {
 179   EmptyMsg m;
 180   Message* mm;
 181   int i;
 182
 183   /* CmiAssert(CmiMyPe()==0); */
 184
 185   CmiInitMsgHeader(m.core, sizeof(EmptyMsg));
 186   pva(nextSize)++;
 187   if (pva(nextSize) == pva(numSizes)) {
 188     pva(nextSize) = -1;
 189     CmiSetHandler(&m, pva(nbrHandler));
 190     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 191   } else {
 192     int size = sizeof(Message) + sizes[pva(nextSize)].size;
 193
 194     pva(buffer_msgs) =
 195       (char*)malloc((sizes[pva(nextSize)].numiter) * sizeof(Message*));
 196     for (i = 0; i < sizes[pva(nextSize)].numiter; i++) {
 197       mm = (Message *)CmiAlloc(size);
 198       mm->srcpe = CmiMyPe();
 199       mm->idx = pva(nextSize);
 200       CmiSetHandler(mm, pva(bounceHandler));
 201       ((Message**)pva(buffer_msgs))[i] = mm;
 202     }
 203     mm = (Message*)CmiAlloc(size);
 204     mm->srcpe = CmiMyPe();
 205     mm->idx = pva(nextSize);
 206     CmiSetHandler(mm, pva(setupHandler));
 207     /* fillMessage(mm); */
 208     CmiSyncSendAndFree(pva(nextNbr), size, mm);
 209   }
 210   CmiFree(msg);
 211 }
 212
 213 static void startNextIter(Message* msg) {
 214   EmptyMsg m;
 215   Message* mm;
 216   pva(nextIter)++;
 217
 218   CmiInitMsgHeader(m.core, sizeof(EmptyMsg));
 219   if (pva(nextIter) >= sizes[pva(nextSize)].numiter) {
 220     pva(endtime) = CmiWallTimer();
 221     /* checkMessage(msg); */
 222     CmiAssert(CpvInitialized(starttime));
 223
 224     CmiAssert(pva(nextNbr) < CmiNumNodes());
 225     CmiAssert(pva(nextNbr) >= 0);
 226     CmiAssert(pva(nextSize) < pva(numSizes));
 227     CmiAssert(pva(nextSize) >= 0);
 228
 229     free(pva(buffer_msgs));
 230     pva(buffer_msgs) = 0;
 231
 232     pva(times)[pva(nextNbr)][pva(nextSize)] =
 233       (pva(endtime) - pva(starttime)) / (2.0 * sizes[pva(nextSize)].numiter);
 234     pva(nextIter) = -1;
 235     CmiSetHandler(&m, pva(sizeHandler));
 236     CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 237     CmiFree(msg);
 238   } else {
 239     mm = ((Message**)pva(buffer_msgs))[pva(nextIter)];
 240     mm->iter = pva(nextIter);
 241     /* CmiSetHandler(mm, pva(bounceHandler)); */
 242     CmiSyncSendAndFree(pva(nextNbr), sizeof(Message) + sizes[mm->idx].size, mm);
 243     CmiFree(msg);
 244   }
 245 }
 246
 247 static void bounceMessage(Message* msg) {
 248   Message* mm;
 249   int iter = msg->iter;
 250   mm = *((Message**)(pva(buffer_msgs) + iter * sizeof(char*)));
 251   CmiSetHandler(mm, pva(iterHandler));
 252   /* CmiAssert(msg->srcpe==0); */
 253   CmiSyncSendAndFree(msg->srcpe, sizeof(Message) + sizes[msg->idx].size, mm);
 254   CmiFree(msg);
 255 }
 256 static void setupMessage(Message* msg) {
 257   Message* mm;
 258   int i, size;
 259
 260   int nextSize = msg->idx;
 261   size = sizeof(Message) + sizes[nextSize].size;
 262   pva(buffer_msgs) = (char*)malloc((sizes[nextSize].numiter) * sizeof(Message*));
 263
 264   for (i = 0; i < sizes[nextSize].numiter; i++) {
 265     mm = (Message*)CmiAlloc(size);
 266     mm->srcpe = CmiMyPe();
 267     CmiSetHandler(mm, pva(iterHandler));
 268     /* mm->idx = pva(nextSize); */
 269     /* *((Message**)buffer_msgs+i*sizeof(char*)) = mm; */
 270     ((Message**)pva(buffer_msgs))[i] = mm;
 271   }
 272   mm = (Message*)CmiAlloc(size);
 273   mm->srcpe = CmiMyPe();
 274   mm->idx = nextSize;
 275   CmiSetHandler(mm, pva(startHandler));
 276   /* fillMessage(mm); */
 277   /* CmiAssert(msg->srcpe == 0); */
 278   CmiSyncSendAndFree(msg->srcpe, size, mm);
 279   CmiFree(msg);
 280 }
 281 static void startMessage(Message* msg) {
 282 #if 0
 283   EmptyMsg m;
 284
 285   CmiFree(msg);
 286
 287   CmiSetHandler(&m, pva(iterHandler));
 288   /* fillMessage(mm); */
 289   pva(starttime) = CmiWallTimer();
 290   CmiSyncSend(CmiMyPe(), sizeof(EmptyMsg), &m);
 291 #else
 292   Message* mm;
 293   int size;
 294   size = sizeof(Message) + sizes[pva(nextSize)].size;
 295   mm = (Message*)CmiAlloc(size);
 296   mm->srcpe = CmiMyPe();
 297   mm->idx = pva(nextSize);
 298   CmiSetHandler(mm, pva(iterHandler));
 299   /* fillMessage(mm); */
 300   pva(starttime) = CmiWallTimer();
 301   CmiSyncSendAndFree(mm->srcpe, size, mm);
 302   CmiFree(msg);
 303 #endif
 304 }
 305
 306 void pingpong_cachemiss_init(void) {
 307   EmptyMsg m;
 308   CmiInitMsgHeader(m.core, sizeof(EmptyMsg));
 309
 310   if (CmiNumNodes() == 1) {
 311     CmiPrintf("[pingpong] This benchmark requires > 1 nodes.\n");
 312     CmiSetHandler(&m, pva(ack_handler));
 313     CmiSyncSend(0, sizeof(EmptyMsg), &m);
 314     return;
 315   }
 316   CmiSetHandler(&m, pva(nbrHandler));
 317   CmiSyncSend(0, sizeof(EmptyMsg), &m);
 318 }
 319
 320 void pingpong_cachemiss_moduleinit(void) {
 321   int i, j;
 322   pvi(int, numRecv);
 323   pva(numRecv) = 0;
 324   pvi(int, nextIter);
 325   pva(nextIter) = -1;
 326   pvi(int, nextSize);
 327   pva(nextSize) = -1;
 328   pvi(int, nextNbr);
 329   pva(nextNbr) = -1;
 330   pvi(double, starttime);
 331   pva(starttime) = 0.0;
 332   pvi(double, endtime);
 333   pva(endtime) = 0.0;
 334   pvi(int, numSizes);
 335   for (i = 0; sizes[i].size != (-1); i++)
 336     ;
 337   pva(numSizes) = i;
 338   pvi(double**, times);
 339   pva(times) = (double**)malloc(CmiNumNodes() * sizeof(double*));
 340   for (i = 0; i < CmiNumNodes(); i++)
 341     pva(times)[i] = (double*)malloc(pva(numSizes) * sizeof(double));
 342   for (i = 0; i < CmiNumNodes(); i++)
 343     for (j = 0; j < pva(numSizes); j++) pva(times)[i][j] = 0.0;
 344   pvi(int*, nodeList);
 345   pva(nodeList) = (int*)malloc(CmiNumNodes() * sizeof(int));
 346   for (i = 0; i < CmiNumNodes(); i++) pva(nodeList)[i] = CmiNodeFirst(i);
 347   pvi(double*, gavg);
 348   pva(gavg) = (double*)malloc(sizeof(double) * pva(numSizes));
 349   pvi(double*, gmax);
 350   pva(gmax) = (double*)malloc(sizeof(double) * pva(numSizes));
 351   pvi(double*, gmin);
 352   pva(gmin) = (double*)malloc(sizeof(double) * pva(numSizes));
 353   pvi(int*, gmaxSrc);
 354   pva(gmaxSrc) = (int*)malloc(sizeof(int) * pva(numSizes));
 355   pvi(int*, gmaxDest);
 356   pva(gmaxDest) = (int*)malloc(sizeof(int) * pva(numSizes));
 357   pvi(int*, gminSrc);
 358   pva(gminSrc) = (int*)malloc(sizeof(int) * pva(numSizes));
 359   pvi(int*, gminDest);
 360   pva(gminDest) = (int*)malloc(sizeof(int) * pva(numSizes));
 361   for (i = 0; i < pva(numSizes); i++) {
 362     pva(gavg)[i] = 0.0;
 363     pva(gmax)[i] = 0.0;
 364     pva(gmin)[i] = 1000000000.0;
 365     pva(gmaxSrc)[i] = 0;
 366     pva(gmaxDest)[i] = 0;
 367     pva(gminSrc)[i] = 0;
 368     pva(gminDest)[i] = 0;
 369   }
 370   pvi(int, timeHandler);
 371   pva(timeHandler) = CmiRegisterHandler((CmiHandler)recvTime);
 372   pvi(int, nodeHandler);
 373   pva(nodeHandler) = CmiRegisterHandler((CmiHandler)startNextNode);
 374   pvi(int, nbrHandler);
 375   pva(nbrHandler) = CmiRegisterHandler((CmiHandler)startNextNbr);
 376   pvi(int, sizeHandler);
 377   pva(sizeHandler) = CmiRegisterHandler((CmiHandler)startNextSize);
 378   pvi(int, iterHandler);
 379   pva(iterHandler) = CmiRegisterHandler((CmiHandler)startNextIter);
 380   pvi(int, bounceHandler);
 381   pva(bounceHandler) = CmiRegisterHandler((CmiHandler)bounceMessage);
 382   pvi(int, setupHandler);
 383   pva(setupHandler) = CmiRegisterHandler((CmiHandler)setupMessage);
 384   pvi(int, startHandler);
 385   pva(startHandler) = CmiRegisterHandler((CmiHandler)startMessage);
 386
 387   pvi(char *, buffer_msgs);
 388   pva(buffer_msgs) = 0;
 389 }