2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 1991-2000, University of Groningen, The Netherlands.
5 * Copyright (c) 2001-2004, The GROMACS development team,
6 * check out http://www.gromacs.org for more information.
7 * Copyright (c) 2012,2013, by the GROMACS development team, led by
8 * David van der Spoel, Berk Hess, Erik Lindahl, and including many
9 * others, as listed in the AUTHORS file in the top-level source
10 * directory and at http://www.gromacs.org.
12 * GROMACS is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU Lesser General Public License
14 * as published by the Free Software Foundation; either version 2.1
15 * of the License, or (at your option) any later version.
17 * GROMACS is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 * Lesser General Public License for more details.
22 * You should have received a copy of the GNU Lesser General Public
23 * License along with GROMACS; if not, see
24 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
25 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
27 * If you want to redistribute modifications to GROMACS, please
28 * consider that scientific software is very special. Version
29 * control is crucial - bugs must be traceable. We will be happy to
30 * consider code for inclusion in the official distribution, but
31 * derived work must not be called official GROMACS. Details are found
32 * in the README & COPYING files - if they are missing, get the
33 * official version at http://www.gromacs.org.
35 * To help us fund GROMACS development, we humbly ask that you cite
36 * the research papers on the package. Check out http://www.gromacs.org.
48 #ifdef HAVE_SYS_TIME_H
53 /* windows-specific include for _chdir() */
59 #include "gmx_fatal.h"
70 #include "thread_mpi.h"
73 /* The source code in this file should be thread-safe.
74 Please keep it that way. */
81 #ifdef GMX_NATIVE_WINDOWS
86 /* Portable version of ctime_r implemented in src/gmxlib/string2.c, but we do not want it declared in public installed headers */
88 gmx_ctime_r(const time_t *clock
, char *buf
, int n
);
94 static void par_fn(char *base
, int ftp
, const t_commrec
*cr
,
95 gmx_bool bAppendSimId
, gmx_bool bAppendNodeId
,
96 char buf
[], int bufsize
)
100 if ((size_t)bufsize
< (strlen(base
)+10))
102 gmx_mem("Character buffer too small!");
105 /* Copy to buf, and strip extension */
107 buf
[strlen(base
) - strlen(ftp2ext(fn2ftp(base
))) - 1] = '\0';
111 sprintf(buf
+strlen(buf
), "%d", cr
->ms
->sim
);
115 strcat(buf
, "_node");
116 sprintf(buf
+strlen(buf
), "%d", cr
->nodeid
);
120 /* Add extension again */
121 strcat(buf
, (ftp
== efTPX
) ? "tpr" : (ftp
== efEDR
) ? "edr" : ftp2ext(ftp
));
124 fprintf(debug
, "node %d par_fn '%s'\n", cr
->nodeid
, buf
);
125 if (fn2ftp(buf
) == efLOG
)
127 fprintf(debug
, "log\n");
132 void check_multi_int(FILE *log
, const gmx_multisim_t
*ms
, int val
,
137 gmx_bool bCompatible
;
139 if (NULL
!= log
&& !bQuiet
)
141 fprintf(log
, "Multi-checking %s ... ", name
);
147 "check_multi_int called with a NULL communication pointer");
150 snew(ibuf
, ms
->nsim
);
152 gmx_sumi_sim(ms
->nsim
, ibuf
, ms
);
155 for (p
= 1; p
< ms
->nsim
; p
++)
157 bCompatible
= bCompatible
&& (ibuf
[p
-1] == ibuf
[p
]);
162 if (NULL
!= log
&& !bQuiet
)
164 fprintf(log
, "OK\n");
171 fprintf(log
, "\n%s is not equal for all subsystems\n", name
);
172 for (p
= 0; p
< ms
->nsim
; p
++)
174 fprintf(log
, " subsystem %d: %d\n", p
, ibuf
[p
]);
177 gmx_fatal(FARGS
, "The %d subsystems are not compatible\n", ms
->nsim
);
183 void check_multi_large_int(FILE *log
, const gmx_multisim_t
*ms
,
184 gmx_large_int_t val
, const char *name
,
187 gmx_large_int_t
*ibuf
;
189 gmx_bool bCompatible
;
191 if (NULL
!= log
&& !bQuiet
)
193 fprintf(log
, "Multi-checking %s ... ", name
);
199 "check_multi_int called with a NULL communication pointer");
202 snew(ibuf
, ms
->nsim
);
204 gmx_sumli_sim(ms
->nsim
, ibuf
, ms
);
207 for (p
= 1; p
< ms
->nsim
; p
++)
209 bCompatible
= bCompatible
&& (ibuf
[p
-1] == ibuf
[p
]);
214 if (NULL
!= log
&& !bQuiet
)
216 fprintf(log
, "OK\n");
223 fprintf(log
, "\n%s is not equal for all subsystems\n", name
);
224 for (p
= 0; p
< ms
->nsim
; p
++)
227 /* first make the format string */
228 snprintf(strbuf
, 255, " subsystem %%d: %s\n",
230 fprintf(log
, strbuf
, p
, ibuf
[p
]);
233 gmx_fatal(FARGS
, "The %d subsystems are not compatible\n", ms
->nsim
);
240 int gmx_gethostname(char *name
, size_t len
)
244 gmx_incons("gmx_gethostname called with len<8");
246 #if defined(HAVE_UNISTD_H) && !defined(__native_client__)
247 if (gethostname(name
, len
-1) != 0)
249 strncpy(name
, "unknown", 8);
254 strncpy(name
, "unknown", 8);
260 void gmx_log_open(const char *lognm
, const t_commrec
*cr
, gmx_bool bMasterOnly
,
261 gmx_bool bAppendFiles
, FILE** fplog
)
263 int len
, testlen
, pid
;
264 char buf
[256], host
[256];
266 char timebuf
[STRLEN
];
272 /* Communicate the filename for logfile */
273 if (cr
->nnodes
> 1 && !bMasterOnly
274 #ifdef GMX_THREAD_MPI
275 /* With thread MPI the non-master log files are opened later
276 * when the files names are already known on all nodes.
284 len
= strlen(lognm
) + 1;
286 gmx_bcast(sizeof(len
), &len
, cr
);
293 tmpnm
= gmx_strdup(lognm
);
295 gmx_bcast(len
*sizeof(*tmpnm
), tmpnm
, cr
);
299 tmpnm
= gmx_strdup(lognm
);
304 if (!bMasterOnly
&& !MASTER(cr
))
306 /* Since log always ends with '.log' let's use this info */
307 par_fn(tmpnm
, efLOG
, cr
, FALSE
, !bMasterOnly
, buf
, 255);
308 fp
= gmx_fio_fopen(buf
, bAppendFiles
? "a+" : "w+" );
310 else if (!bAppendFiles
)
312 fp
= gmx_fio_fopen(tmpnm
, bAppendFiles
? "a+" : "w+" );
317 gmx_fatal_set_log_file(fp
);
319 /* Get some machine parameters */
320 gmx_gethostname(host
, 256);
325 # ifdef GMX_NATIVE_WINDOWS
339 "-----------------------------------------------------------\n"
340 "Restarting from checkpoint, appending to previous log file.\n"
345 gmx_ctime_r(&t
, timebuf
, STRLEN
);
348 "Log file opened on %s"
349 "Host: %s pid: %d nodeid: %d nnodes: %d\n",
350 timebuf
, host
, pid
, cr
->nodeid
, cr
->nnodes
);
351 gmx_print_version_info(fp
);
360 void gmx_log_close(FILE *fp
)
364 gmx_fatal_set_log_file(NULL
);
369 static void comm_args(const t_commrec
*cr
, int *argc
, char ***argv
)
375 gmx_bcast(sizeof(*argc
), argc
, cr
);
380 snew(*argv
, *argc
+1);
384 fprintf(debug
, "NODEID=%d argc=%d\n", cr
->nodeid
, *argc
);
386 for (i
= 0; (i
< *argc
); i
++)
390 len
= strlen((*argv
)[i
])+1;
392 gmx_bcast(sizeof(len
), &len
, cr
);
395 snew((*argv
)[i
], len
);
397 /*gmx_bcast(len*sizeof((*argv)[i][0]),(*argv)[i],cr);*/
398 gmx_bcast(len
*sizeof(char), (*argv
)[i
], cr
);
403 void init_multisystem(t_commrec
*cr
, int nsim
, char **multidirs
,
404 int nfile
, const t_filenm fnm
[], gmx_bool bParFn
)
407 int nnodes
, nnodpersim
, sim
, i
, ftp
;
410 MPI_Group mpi_group_world
;
417 gmx_fatal(FARGS
, "This binary is compiled without MPI support, can not do multiple simulations.");
422 if (nnodes
% nsim
!= 0)
424 gmx_fatal(FARGS
, "The number of nodes (%d) is not a multiple of the number of simulations (%d)", nnodes
, nsim
);
427 nnodpersim
= nnodes
/nsim
;
428 sim
= cr
->nodeid
/nnodpersim
;
432 fprintf(debug
, "We have %d simulations, %d nodes per simulation, local simulation is %d\n", nsim
, nnodpersim
, sim
);
440 /* Create a communicator for the master nodes */
441 snew(rank
, ms
->nsim
);
442 for (i
= 0; i
< ms
->nsim
; i
++)
444 rank
[i
] = i
*nnodpersim
;
446 MPI_Comm_group(MPI_COMM_WORLD
, &mpi_group_world
);
447 MPI_Group_incl(mpi_group_world
, nsim
, rank
, &ms
->mpi_group_masters
);
449 MPI_Comm_create(MPI_COMM_WORLD
, ms
->mpi_group_masters
,
450 &ms
->mpi_comm_masters
);
452 #if !defined(GMX_THREAD_MPI) && !defined(MPI_IN_PLACE_EXISTS)
453 /* initialize the MPI_IN_PLACE replacement buffers */
455 ms
->mpb
->ibuf
= NULL
;
456 ms
->mpb
->libuf
= NULL
;
457 ms
->mpb
->fbuf
= NULL
;
458 ms
->mpb
->dbuf
= NULL
;
459 ms
->mpb
->ibuf_alloc
= 0;
460 ms
->mpb
->libuf_alloc
= 0;
461 ms
->mpb
->fbuf_alloc
= 0;
462 ms
->mpb
->dbuf_alloc
= 0;
467 /* Reduce the intra-simulation communication */
468 cr
->sim_nodeid
= cr
->nodeid
% nnodpersim
;
469 cr
->nnodes
= nnodpersim
;
471 MPI_Comm_split(MPI_COMM_WORLD
, sim
, cr
->sim_nodeid
, &cr
->mpi_comm_mysim
);
472 cr
->mpi_comm_mygroup
= cr
->mpi_comm_mysim
;
473 cr
->nodeid
= cr
->sim_nodeid
;
478 fprintf(debug
, "This is simulation %d", cr
->ms
->sim
);
481 fprintf(debug
, ", local number of nodes %d, local nodeid %d",
482 cr
->nnodes
, cr
->sim_nodeid
);
484 fprintf(debug
, "\n\n");
492 fprintf(debug
, "Changing to directory %s\n", multidirs
[cr
->ms
->sim
]);
494 if (chdir(multidirs
[cr
->ms
->sim
]) != 0)
496 gmx_fatal(FARGS
, "Couldn't change directory to %s: %s",
497 multidirs
[cr
->ms
->sim
],
503 /* Patch output and tpx, cpt and rerun input file names */
504 for (i
= 0; (i
< nfile
); i
++)
506 /* Because of possible multiple extensions per type we must look
507 * at the actual file name
509 if (is_output(&fnm
[i
]) ||
510 fnm
[i
].ftp
== efTPX
|| fnm
[i
].ftp
== efCPT
||
511 strcmp(fnm
[i
].opt
, "-rerun") == 0)
513 ftp
= fn2ftp(fnm
[i
].fns
[0]);
514 par_fn(fnm
[i
].fns
[0], ftp
, cr
, TRUE
, FALSE
, buf
, 255);
515 sfree(fnm
[i
].fns
[0]);
516 fnm
[i
].fns
[0] = gmx_strdup(buf
);
522 t_commrec
*init_par(int *argc
, char ***argv_ptr
)
531 argv
= argv_ptr
? *argv_ptr
: NULL
;
533 #if defined GMX_MPI && !defined GMX_THREAD_MPI
534 cr
->sim_nodeid
= gmx_setup(argc
, argv
, &cr
->nnodes
);
536 if (!PAR(cr
) && (cr
->sim_nodeid
!= 0))
538 gmx_comm("(!PAR(cr) && (cr->sim_nodeid != 0))");
541 cr
->mpi_comm_mysim
= MPI_COMM_WORLD
;
542 cr
->mpi_comm_mygroup
= cr
->mpi_comm_mysim
;
544 /* These should never be accessed */
545 cr
->mpi_comm_mysim
= NULL
;
546 cr
->mpi_comm_mygroup
= NULL
;
551 cr
->nodeid
= cr
->sim_nodeid
;
553 cr
->duty
= (DUTY_PP
| DUTY_PME
);
555 /* Communicate arguments if parallel */
556 #ifndef GMX_THREAD_MPI
559 comm_args(cr
, argc
, argv_ptr
);
561 #endif /* GMX_THREAD_MPI */
564 #if !defined(GMX_THREAD_MPI) && !defined(MPI_IN_PLACE_EXISTS)
565 /* initialize the MPI_IN_PLACE replacement buffers */
567 cr
->mpb
->ibuf
= NULL
;
568 cr
->mpb
->libuf
= NULL
;
569 cr
->mpb
->fbuf
= NULL
;
570 cr
->mpb
->dbuf
= NULL
;
571 cr
->mpb
->ibuf_alloc
= 0;
572 cr
->mpb
->libuf_alloc
= 0;
573 cr
->mpb
->fbuf_alloc
= 0;
574 cr
->mpb
->dbuf_alloc
= 0;
581 t_commrec
*init_par_threads(const t_commrec
*cro
)
583 #ifdef GMX_THREAD_MPI
587 /* make a thread-specific commrec */
589 /* now copy the whole thing, so settings like the number of PME nodes
593 /* and we start setting our own thread-specific values for things */
594 MPI_Initialized(&initialized
);
597 gmx_comm("Initializing threads without comm");
599 /* once threads will be used together with MPI, we'll
600 fill the cr structure with distinct data here. This might even work: */
601 cr
->sim_nodeid
= gmx_setup(0, NULL
, &cr
->nnodes
);
603 cr
->mpi_comm_mysim
= MPI_COMM_WORLD
;
604 cr
->mpi_comm_mygroup
= cr
->mpi_comm_mysim
;
605 cr
->nodeid
= cr
->sim_nodeid
;
606 cr
->duty
= (DUTY_PP
| DUTY_PME
);