src/mdlib/gmx_wallcycle.c

   1 /*  -*- mode: c; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4; c-file-style: "stroustrup"; -*-
   2  *
   3  *
   4  *                This source code is part of
   5  *
   6  *                 G   R   O   M   A   C   S
   7  *
   8  *          GROningen MAchine for Chemical Simulations
   9  *
  10  * Written by David van der Spoel, Erik Lindahl, Berk Hess, and others.
  11  * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
  12  * Copyright (c) 2001-2008, The GROMACS development team,
  13  * check out http://www.gromacs.org for more information.
  14
  15  * This program is free software; you can redistribute it and/or
  16  * modify it under the terms of the GNU General Public License
  17  * as published by the Free Software Foundation; either version 2
  18  * of the License, or (at your option) any later version.
  19  *
  20  * If you want to redistribute modifications, please consider that
  21  * scientific software is very special. Version control is crucial -
  22  * bugs must be traceable. We will be happy to consider code for
  23  * inclusion in the official distribution, but derived work must not
  24  * be called official GROMACS. Details are found in the README & COPYING
  25  * files - if they are missing, get the official version at www.gromacs.org.
  26  *
  27  * To help us fund GROMACS development, we humbly ask that you cite
  28  * the papers on the package - you can find them in the top README file.
  29  *
  30  * For more info, check our website at http://www.gromacs.org
  31  *
  32  * And Hey:
  33  * Gallium Rubidium Oxygen Manganese Argon Carbon Silicon
  34  */
  35
  36
  37 #ifdef HAVE_CONFIG_H
  38 #include <config.h>
  39 #endif
  40
  41 #include <string.h>
  42 #include "gmx_wallcycle.h"
  43 #include "gmx_cyclecounter.h"
  44 #include "smalloc.h"
  45
  46 #ifdef GMX_LIB_MPI
  47 #include <mpi.h>
  48 #endif
  49 #ifdef GMX_THREAD_MPI
  50 #include "thread_mpi.h"
  51 #endif
  52
  53 typedef struct
  54 {
  55     int          n;
  56     gmx_cycles_t c;
  57     gmx_cycles_t start;
  58     gmx_cycles_t last;
  59 } wallcc_t;
  60
  61 typedef struct gmx_wallcycle
  62 {
  63     wallcc_t     *wcc;
  64     /* variables for testing/debugging */
  65     bool         wc_barrier;
  66     wallcc_t     *wcc_all;
  67     int          wc_depth;
  68     int          ewc_prev;
  69     gmx_cycles_t cycle_prev;
  70     gmx_step_t   reset_counters;
  71 #ifdef GMX_MPI
  72     MPI_Comm     mpi_comm_mygroup;
  73 #endif
  74 } gmx_wallcycle_t_t;
  75
  76 /* Each name should not exceed 19 characters */
  77 static const char *wcn[ewcNR] =
  78   { "Run", "Step", "PP during PME", "Domain decomp.", "DD comm. load", "DD comm. bounds", "Vsite constr.", "Send X to PME", "Comm. coord.", "Neighbor search", "Force", "Wait + Comm. F", "PME mesh", "PME mesh", "Wait + Comm. X/F", "Wait + Recv. PME F", "Vsite spread", "Write traj.", "Update", "Constraints", "Comm. energies", "Test", "Born radii" };
  79
  80 bool wallcycle_have_counter(void)
  81 {
  82   return gmx_cycles_have_counter();
  83 }
  84
  85 gmx_wallcycle_t wallcycle_init(FILE *fplog,t_commrec *cr)
  86 {
  87     gmx_wallcycle_t wc;
  88     char *env_ptr;
  89
  90
  91     if (!wallcycle_have_counter())
  92     {
  93         return NULL;
  94     }
  95
  96     snew(wc,1);
  97
  98     wc->wc_barrier = FALSE;
  99     wc->wcc_all    = NULL;
 100     wc->wc_depth   = 0;
 101     wc->ewc_prev   = -1;
 102
 103 #ifdef GMX_MPI
 104     if (PAR(cr) && getenv("GMX_CYCLE_BARRIER") != NULL)
 105     {
 106         if (fplog)
 107         {
 108             fprintf(fplog,"\nWill call MPI_Barrier before each cycle start/stop call\n\n");
 109         }
 110         wc->wc_barrier = TRUE;
 111         wc->mpi_comm_mygroup = cr->mpi_comm_mygroup;
 112     }
 113 #endif
 114
 115     snew(wc->wcc,ewcNR);
 116     if (getenv("GMX_CYCLE_ALL") != NULL)
 117     {
 118         if (fplog)
 119         {
 120             fprintf(fplog,"\nWill time all the code during the run\n\n");
 121         }
 122         snew(wc->wcc_all,ewcNR*ewcNR);
 123     }
 124
 125     /* Read variable GMX_RESET_COUNTER from environment */
 126     wc->reset_counters = -1;
 127     if ((env_ptr=getenv("GMX_RESET_COUNTERS")) != NULL)
 128     {
 129         sscanf(env_ptr,gmx_step_pfmt,&wc->reset_counters);
 130     }
 131
 132     return wc;
 133 }
 134
 135 void wallcycle_destroy(gmx_wallcycle_t wc)
 136 {
 137     if (wc == NULL)
 138     {
 139         return;
 140     }
 141
 142     if (wc->wcc != NULL)
 143     {
 144         sfree(wc->wcc);
 145     }
 146     if (wc->wcc_all != NULL)
 147     {
 148         sfree(wc->wcc_all);
 149     }
 150     sfree(wc);
 151 }
 152
 153 static void wallcycle_all_start(gmx_wallcycle_t wc,int ewc,gmx_cycles_t cycle)
 154 {
 155     wc->ewc_prev = ewc;
 156     wc->cycle_prev = cycle;
 157 }
 158
 159 static void wallcycle_all_stop(gmx_wallcycle_t wc,int ewc,gmx_cycles_t cycle)
 160 {
 161     wc->wcc_all[wc->ewc_prev*ewcNR+ewc].n += 1;
 162     wc->wcc_all[wc->ewc_prev*ewcNR+ewc].c += cycle - wc->cycle_prev;
 163 }
 164
 165 void wallcycle_start(gmx_wallcycle_t wc, int ewc)
 166 {
 167     gmx_cycles_t cycle;
 168
 169     if (wc == NULL)
 170     {
 171         return;
 172     }
 173
 174 #ifdef GMX_MPI
 175     if (wc->wc_barrier)
 176     {
 177         MPI_Barrier(wc->mpi_comm_mygroup);
 178     }
 179 #endif
 180
 181     cycle = gmx_cycles_read();
 182     wc->wcc[ewc].start = cycle;
 183     if (wc->wcc_all != NULL)
 184     {
 185         wc->wc_depth++;
 186         if (ewc == ewcRUN)
 187         {
 188             wallcycle_all_start(wc,ewc,cycle);
 189         }
 190         else if (wc->wc_depth == 3)
 191         {
 192             wallcycle_all_stop(wc,ewc,cycle);
 193         }
 194     }
 195 }
 196
 197 double wallcycle_stop(gmx_wallcycle_t wc, int ewc)
 198 {
 199     gmx_cycles_t cycle,last;
 200
 201     if (wc == NULL)
 202     {
 203         return 0;
 204     }
 205
 206 #ifdef GMX_MPI
 207     if (wc->wc_barrier)
 208     {
 209         MPI_Barrier(wc->mpi_comm_mygroup);
 210     }
 211 #endif
 212
 213     cycle = gmx_cycles_read();
 214     last = cycle - wc->wcc[ewc].start;
 215     wc->wcc[ewc].c += last;
 216     wc->wcc[ewc].n++;
 217     if (wc->wcc_all)
 218     {
 219         wc->wc_depth--;
 220         if (ewc == ewcRUN)
 221         {
 222             wallcycle_all_stop(wc,ewc,cycle);
 223         }
 224         else if (wc->wc_depth == 2)
 225         {
 226             wallcycle_all_start(wc,ewc,cycle);
 227         }
 228     }
 229
 230     return last;
 231 }
 232
 233 void wallcycle_reset_all(gmx_wallcycle_t wc)
 234 {
 235     int i;
 236
 237     if (wc == NULL)
 238     {
 239         return;
 240     }
 241
 242     for(i=0; i<ewcNR; i++)
 243     {
 244         wc->wcc[i].n = 0;
 245         wc->wcc[i].c = 0;
 246         wc->wcc[i].start = 0;
 247         wc->wcc[i].last = 0;
 248     }
 249 }
 250
 251 void wallcycle_sum(t_commrec *cr, gmx_wallcycle_t wc,double cycles[])
 252 {
 253     wallcc_t *wcc;
 254     double buf[ewcNR],*cyc_all,*buf_all;
 255     int    i;
 256
 257     if (wc == NULL)
 258     {
 259         return;
 260     }
 261
 262     wcc = wc->wcc;
 263
 264     if (wcc[ewcDDCOMMLOAD].n > 0)
 265     {
 266         wcc[ewcDOMDEC].c -= wcc[ewcDDCOMMLOAD].c;
 267     }
 268     if (wcc[ewcDDCOMMBOUND].n > 0)
 269     {
 270         wcc[ewcDOMDEC].c -= wcc[ewcDDCOMMBOUND].c;
 271     }
 272     if (wcc[ewcPMEMESH].n > 0)
 273     {
 274         wcc[ewcFORCE].c -= wcc[ewcPMEMESH].c;
 275     }
 276
 277     if (wcc[ewcPMEMESH_SEP].n > 0)
 278     {
 279         /* This must be a PME only node, calculate the Wait + Comm. time */
 280         wcc[ewcPMEWAITCOMM].c = wcc[ewcRUN].c - wcc[ewcPMEMESH_SEP].c;
 281     }
 282     else
 283     {
 284         /* Correct the PME mesh only call count */
 285         wcc[ewcPMEMESH_SEP].n = wcc[ewcFORCE].n;
 286         wcc[ewcPMEWAITCOMM].n = wcc[ewcFORCE].n;
 287     }
 288
 289     /* Store the cycles in a double buffer for summing */
 290     for(i=0; i<ewcNR; i++)
 291     {
 292         cycles[i] = (double)wcc[i].c;
 293     }
 294
 295     if (wcc[ewcUPDATE].n > 0)
 296     {
 297         /* Remove the constraint part from the update count */
 298         cycles[ewcUPDATE] -= cycles[ewcCONSTR];
 299     }
 300
 301 #ifdef GMX_MPI
 302     if (cr->nnodes > 1)
 303     {
 304         MPI_Allreduce(cycles,buf,ewcNR,MPI_DOUBLE,MPI_SUM,
 305                       cr->mpi_comm_mysim);
 306         for(i=0; i<ewcNR; i++)
 307         {
 308             cycles[i] = buf[i];
 309         }
 310         if (wc->wcc_all != NULL)
 311         {
 312             snew(cyc_all,ewcNR*ewcNR);
 313             snew(buf_all,ewcNR*ewcNR);
 314             for(i=0; i<ewcNR*ewcNR; i++)
 315             {
 316                 cyc_all[i] = wc->wcc_all[i].c;
 317             }
 318             MPI_Allreduce(cyc_all,buf_all,ewcNR*ewcNR,MPI_DOUBLE,MPI_SUM,
 319                           cr->mpi_comm_mysim);
 320             for(i=0; i<ewcNR*ewcNR; i++)
 321             {
 322                 wc->wcc_all[i].c = buf_all[i];
 323             }
 324             sfree(buf_all);
 325             sfree(cyc_all);
 326         }
 327     }
 328 #endif
 329 }
 330
 331 static void print_cycles(FILE *fplog, double c2t, const char *name, int nnodes,
 332                          int n, gmx_cycles_t c, gmx_cycles_t tot)
 333 {
 334   char num[11];
 335
 336   if (c > 0) {
 337     if (n > 0)
 338       sprintf(num,"%10d",n);
 339     else
 340       sprintf(num,"          ");
 341     fprintf(fplog," %-19s %4d %10s %12.3f %10.1f   %5.1f\n",
 342             name,nnodes,num,c*1e-9,c*c2t,100*(double)c/(double)tot);
 343   }
 344 }
 345
 346 void wallcycle_print(FILE *fplog, int nnodes, int npme, double realtime,
 347                      gmx_wallcycle_t wc, double cycles[])
 348 {
 349     double c2t,tot,sum;
 350     int    i,j,npp;
 351     char   buf[STRLEN];
 352     const char *myline = "-----------------------------------------------------------------------";
 353
 354     if (wc == NULL)
 355     {
 356         return;
 357     }
 358
 359     if (npme > 0)
 360     {
 361         npp = nnodes - npme;
 362     }
 363     else
 364     {
 365         npp = nnodes;
 366     }
 367     tot = cycles[ewcRUN];
 368     /* Conversion factor from cycles to seconds */
 369     if (tot > 0)
 370     {
 371       c2t = nnodes*realtime/tot;
 372     }
 373     else
 374     {
 375       c2t = 0;
 376     }
 377
 378     fprintf(fplog,"\n     R E A L   C Y C L E   A N D   T I M E   A C C O U N T I N G\n\n");
 379
 380     fprintf(fplog," Computing:         Nodes     Number     G-Cycles    Seconds     %c\n",'%');
 381     fprintf(fplog,"%s\n",myline);
 382     sum = 0;
 383     for(i=ewcPPDURINGPME+1; i<ewcNR; i++)
 384     {
 385         print_cycles(fplog,c2t,wcn[i],
 386                      (i==ewcPMEMESH_SEP || i==ewcPMEWAITCOMM) ? npme : npp,
 387                      wc->wcc[i].n,cycles[i],tot);
 388         sum += cycles[i];
 389     }
 390     if (wc->wcc_all != NULL)
 391     {
 392         for(i=0; i<ewcNR; i++)
 393         {
 394             for(j=0; j<ewcNR; j++)
 395             {
 396                 sprintf(buf,"%-9s",wcn[i]);
 397                 buf[9] = ' ';
 398                 sprintf(buf+10,"%-9s",wcn[j]);
 399                 buf[19] = '\0';
 400                 print_cycles(fplog,c2t,buf,
 401                              (i==ewcPMEMESH_SEP || i==ewcPMEWAITCOMM) ? npme : npp,
 402                              wc->wcc_all[i*ewcNR+j].n,
 403                              wc->wcc_all[i*ewcNR+j].c,
 404                              tot);
 405                 sum += wc->wcc_all[i*ewcNR+j].c;
 406             }
 407         }
 408     }
 409     print_cycles(fplog,c2t,"Rest",npp,0,tot-sum,tot);
 410     fprintf(fplog,"%s\n",myline);
 411     print_cycles(fplog,c2t,"Total",nnodes,0,tot,tot);
 412     fprintf(fplog,"%s\n",myline);
 413
 414     if (cycles[ewcMoveE] > tot*0.05)
 415     {
 416         sprintf(buf,
 417                 "NOTE: %d %% of the run time was spent communicating energies,\n"
 418                 "      you might want to use the -gcom option of mdrun\n",
 419                 (int)(100*cycles[ewcMoveE]/tot+0.5));
 420         if (fplog)
 421         {
 422             fprintf(fplog,"\n%s\n",buf);
 423         }
 424         /* Only the sim master calls this function, so always print to stderr */
 425         fprintf(stderr,"\n%s\n",buf);
 426     }
 427 }
 428
 429 extern gmx_step_t wcycle_get_reset_counters(gmx_wallcycle_t wc)
 430 {
 431     return wc->reset_counters;
 432 }
 433
 434 extern void wcycle_set_reset_counters(gmx_wallcycle_t wc, gmx_step_t reset_counters)
 435 {
 436     wc->reset_counters = reset_counters;
 437 }