src/util/XTTorus.h

   1 /** \file XTTorus.h
   2  *  Author: Abhinav S Bhatele
   3  *  Date created: August 19th, 2008
   4  *
   5  */
   6
   7 #ifndef _XT_TORUS_H_
   8 #define _XT_TORUS_H_
   9
  10 #include "converse.h"
  11
  12 #include <stdlib.h>
  13 #include <stdio.h>
  14
  15 #if XT4_TOPOLOGY || XT5_TOPOLOGY || XE6_TOPOLOGY
  16
  17 // XDIM, YDIM, ZDIM and MAXNID depend on a specific Cray installation.
  18 // Please do NOT expect things to work if you use this code on a new
  19 // Cray machine.
  20
  21 #if XT4_TOPOLOGY
  22 #define MAXNID 14000
  23 #define XDIM 21
  24 #define YDIM 16
  25 #define ZDIM 24
  26 #define TDIM 4
  27
  28 #elif XT5_TOPOLOGY
  29 #define MAXNID 22020
  30 #define XDIM 25
  31 #define YDIM 32
  32 #define ZDIM 24
  33 #define TDIM 12
  34
  35 #elif XE6_TOPOLOGY
  36 /* hopper */
  37 #define MAXNID 6384
  38 #define XDIM 17
  39 #define YDIM 8
  40 #define ZDIM 24
  41 #define TDIM 24
  42
  43 #endif
  44
  45 extern "C" int *pid2nid;
  46 extern "C" int nid2pid[MAXNID][TDIM];
  47 extern "C" int pidtonid(int numpes);
  48 extern "C" int getMeshCoord(int nid, int *x, int *y, int *z);
  49
  50 struct loc {
  51   int x;
  52   int y;
  53   int z;
  54   int t;
  55 };
  56
  57 class XTTorusManager {
  58   private:
  59     int dimX;   // dimension of the allocation in X (processors)
  60     int dimY;   // dimension of the allocation in Y (processors)
  61     int dimZ;   // dimension of the allocation in Z (processors)
  62     int dimNX;  // dimension of the allocation in X (nodes)
  63     int dimNY;  // dimension of the allocation in Y (nodes)
  64     int dimNZ;  // dimension of the allocation in Z (nodes)
  65     int dimNT;  // number of processors per node (2 for XT3)
  66
  67     int torus[4];
  68     int procsPerNode;   // number of cores per node
  69
  70     int coords2pid[XDIM][YDIM][ZDIM][TDIM];     // coordinates to rank
  71     struct loc *pid2coords;                     // rank to coordinates
  72     struct loc origin;
  73
  74   public:
  75     XTTorusManager() {
  76       int nid = 0, oldnid = -1, lx, ly, lz;
  77       int i, j, k, l;
  78       int minX=XDIM, minY=YDIM, minZ=ZDIM, minT=0, maxX=0, maxY=0, maxZ=0;
  79
  80       int numPes = CmiNumPes();
  81       pid2coords = (struct loc*)malloc(sizeof(struct loc) * numPes);
  82
  83       // fill the nid2pid and pid2nid data structures
  84       pidtonid(numPes);
  85
  86       for(i=0; i<XDIM; i++)
  87         for(j=0; j<YDIM; j++)
  88           for(k=0; k<ZDIM; k++)
  89             for(l=0; l<TDIM; l++)
  90               coords2pid[i][j][k][l] = -1;
  91
  92       dimNT = 1;                        // assume SN mode first
  93       // now fill the coords2pid and pid2coords data structures
  94       for(i=0; i<numPes; i++)
  95       {
  96         nid = pid2nid[i];
  97         if (nid != oldnid)
  98           getMeshCoord(nid, &lx, &ly, &lz);
  99         oldnid = nid;
 100
 101         pid2coords[i].x = lx;
 102         pid2coords[i].y = ly;
 103         pid2coords[i].z = lz;
 104
 105         l = 0;
 106         while(coords2pid[lx][ly][lz][l] != -1)
 107           l++;
 108         coords2pid[lx][ly][lz][l] = i;
 109         pid2coords[i].t = l;
 110
 111         if (lx<minX) minX = lx; if (lx>maxX) maxX = lx;
 112         if (ly<minY) minY = ly; if (ly>maxY) maxY = ly;
 113         if (lz<minZ) minZ = lz; if (lz>maxZ) maxZ = lz;
 114       }
 115
 116       // set the origin as the element on the lower end of the torus
 117       origin.x =  minX;
 118       origin.y =  minY;
 119       origin.z =  minZ;
 120       origin.t =  minT;
 121
 122       // assuming a contiguous allocation find the dimensions of
 123       // the torus
 124       dimNX = maxX - minX + 1;
 125       dimNY = maxY - minY + 1;
 126       dimNZ = maxZ - minZ + 1;
 127       procsPerNode = dimNT;
 128       dimX = dimNX * dimNT;
 129       dimY = dimNY;
 130       dimZ = dimNZ;
 131
 132       // pick a random node (1) to find the number of cores per node being
 133       // actually used - assumes same number of cores per node
 134       lx = pid2coords[1].x;
 135       ly = pid2coords[1].y;
 136       lz = pid2coords[1].z;
 137       for(l=0; l<TDIM; l++) {
 138         if(coords2pid[lx][ly][lz][l] == -1)
 139           break;
 140       }
 141       dimNT = l;
 142
 143       // we get a torus only if the size of the dimension is the biggest
 144       torus[0] = 0;             // Jaguar is a mesh in X dimension always
 145       torus[1] = (dimNY == YDIM) ? 1 : 0;
 146       torus[2] = (dimNZ == ZDIM) ? 1 : 0;
 147       torus[3] = 0;
 148     }
 149
 150     ~XTTorusManager() { }
 151
 152     inline int getDimX() { return dimX; }
 153     inline int getDimY() { return dimY; }
 154     inline int getDimZ() { return dimZ; }
 155
 156     inline int getDimNX() { return dimNX; }
 157     inline int getDimNY() { return dimNY; }
 158     inline int getDimNZ() { return dimNZ; }
 159     inline int getDimNT() { return dimNT; }
 160
 161     inline int getProcsPerNode() { return procsPerNode; }
 162
 163     inline int* isTorus() { return torus; }
 164
 165     inline void rankToCoordinates(int pe, int &x, int &y, int &z, int &t) {
 166       x = pid2coords[pe].x - origin.x;
 167       y = pid2coords[pe].y - origin.y;
 168       z = pid2coords[pe].z - origin.z;
 169       t = pid2coords[pe].t - origin.t;
 170     }
 171
 172     inline void realRankToCoordinates(int pe, int &x, int &y, int &z, int &t) {
 173       x = pid2coords[pe].x;
 174       y = pid2coords[pe].y;
 175       z = pid2coords[pe].z;
 176       t = pid2coords[pe].t;
 177     }
 178
 179     inline int coordinatesToRank(int x, int y, int z, int t) {
 180       return coords2pid[x+origin.x][y+origin.y][z+origin.z][t+origin.t];
 181     }
 182 };
 183
 184 #endif // XT4_TOPOLOGY || XT5_TOPOLOGY
 185 #endif //_XT_TORUS_H_