2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
51 #include "gromacs/domdec/domdec_network.h"
52 #include "gromacs/domdec/ga2la.h"
53 #include "gromacs/ewald/pme.h"
54 #include "gromacs/fileio/gmxfio.h"
55 #include "gromacs/fileio/pdbio.h"
56 #include "gromacs/gmxlib/chargegroup.h"
57 #include "gromacs/gmxlib/network.h"
58 #include "gromacs/gmxlib/nrnb.h"
59 #include "gromacs/gpu_utils/gpu_utils.h"
60 #include "gromacs/hardware/hw_info.h"
61 #include "gromacs/imd/imd.h"
62 #include "gromacs/listed-forces/manage-threading.h"
63 #include "gromacs/math/functions.h"
64 #include "gromacs/math/vec.h"
65 #include "gromacs/math/vectypes.h"
66 #include "gromacs/mdlib/constr.h"
67 #include "gromacs/mdlib/force.h"
68 #include "gromacs/mdlib/forcerec.h"
69 #include "gromacs/mdlib/genborn.h"
70 #include "gromacs/mdlib/gmx_omp_nthreads.h"
71 #include "gromacs/mdlib/mdatoms.h"
72 #include "gromacs/mdlib/mdrun.h"
73 #include "gromacs/mdlib/mdsetup.h"
74 #include "gromacs/mdlib/nb_verlet.h"
75 #include "gromacs/mdlib/nbnxn_grid.h"
76 #include "gromacs/mdlib/nsgrid.h"
77 #include "gromacs/mdlib/vsite.h"
78 #include "gromacs/mdtypes/commrec.h"
79 #include "gromacs/mdtypes/df_history.h"
80 #include "gromacs/mdtypes/forcerec.h"
81 #include "gromacs/mdtypes/inputrec.h"
82 #include "gromacs/mdtypes/md_enums.h"
83 #include "gromacs/mdtypes/mdatom.h"
84 #include "gromacs/mdtypes/nblist.h"
85 #include "gromacs/mdtypes/state.h"
86 #include "gromacs/pbcutil/ishift.h"
87 #include "gromacs/pbcutil/pbc.h"
88 #include "gromacs/pulling/pull.h"
89 #include "gromacs/pulling/pull_rotation.h"
90 #include "gromacs/swap/swapcoords.h"
91 #include "gromacs/timing/wallcycle.h"
92 #include "gromacs/topology/block.h"
93 #include "gromacs/topology/idef.h"
94 #include "gromacs/topology/ifunc.h"
95 #include "gromacs/topology/mtop_lookup.h"
96 #include "gromacs/topology/mtop_util.h"
97 #include "gromacs/topology/topology.h"
98 #include "gromacs/utility/basedefinitions.h"
99 #include "gromacs/utility/basenetwork.h"
100 #include "gromacs/utility/cstringutil.h"
101 #include "gromacs/utility/exceptions.h"
102 #include "gromacs/utility/fatalerror.h"
103 #include "gromacs/utility/gmxmpi.h"
104 #include "gromacs/utility/qsort_threadsafe.h"
105 #include "gromacs/utility/real.h"
106 #include "gromacs/utility/smalloc.h"
108 #include "domdec_constraints.h"
109 #include "domdec_internal.h"
110 #include "domdec_vsite.h"
112 #define DDRANK(dd, rank) (rank)
113 #define DDMASTERRANK(dd) (dd->masterrank)
115 struct gmx_domdec_master_t
117 /* The cell boundaries */
119 /* The global charge group division */
120 int *ncg
; /* Number of home charge groups for each node */
121 int *index
; /* Index of nnodes+1 into cg */
122 int *cg
; /* Global charge group index */
123 int *nat
; /* Number of home atoms for each node. */
124 int *ibuf
; /* Buffer for communication */
125 rvec
*vbuf
; /* Buffer for state scattering and gathering */
128 #define DD_NLOAD_MAX 9
130 const char *edlbs_names
[edlbsNR
] = { "off", "auto", "locked", "on" };
132 /* The size per charge group of the cggl_flag buffer in gmx_domdec_comm_t */
135 /* The flags for the cggl_flag buffer in gmx_domdec_comm_t */
136 #define DD_FLAG_NRCG 65535
137 #define DD_FLAG_FW(d) (1<<(16+(d)*2))
138 #define DD_FLAG_BW(d) (1<<(16+(d)*2+1))
140 /* The DD zone order */
141 static const ivec dd_zo
[DD_MAXZONE
] =
142 {{0, 0, 0}, {1, 0, 0}, {1, 1, 0}, {0, 1, 0}, {0, 1, 1}, {0, 0, 1}, {1, 0, 1}, {1, 1, 1}};
144 /* The non-bonded zone-pair setup for domain decomposition
145 * The first number is the i-zone, the second number the first j-zone seen by
146 * this i-zone, the third number the last+1 j-zone seen by this i-zone.
147 * As is, this is for 3D decomposition, where there are 4 i-zones.
148 * With 2D decomposition use only the first 2 i-zones and a last+1 j-zone of 4.
149 * With 1D decomposition use only the first i-zone and a last+1 j-zone of 2.
152 ddNonbondedZonePairRanges
[DD_MAXIZONE
][3] = {{0, 0, 8},
157 /* Factors used to avoid problems due to rounding issues */
158 #define DD_CELL_MARGIN 1.0001
159 #define DD_CELL_MARGIN2 1.00005
160 /* Factor to account for pressure scaling during nstlist steps */
161 #define DD_PRES_SCALE_MARGIN 1.02
163 /* Turn on DLB when the load imbalance causes this amount of total loss.
164 * There is a bit of overhead with DLB and it's difficult to achieve
165 * a load imbalance of less than 2% with DLB.
167 #define DD_PERF_LOSS_DLB_ON 0.02
169 /* Warn about imbalance due to PP or PP/PME load imbalance at this loss */
170 #define DD_PERF_LOSS_WARN 0.05
172 #define DD_CELL_F_SIZE(dd, di) ((dd)->nc[(dd)->dim[(di)]]+1+(di)*2+1+(di))
174 /* Use separate MPI send and receive commands
175 * when nnodes <= GMX_DD_NNODES_SENDRECV.
176 * This saves memory (and some copying for small nnodes).
177 * For high parallelization scatter and gather calls are used.
179 #define GMX_DD_NNODES_SENDRECV 4
182 /* We check if to turn on DLB at the first and every 100 DD partitionings.
183 * With large imbalance DLB will turn on at the first step, so we can
184 * make the interval so large that the MPI overhead of the check is negligible.
186 static const int c_checkTurnDlbOnInterval
= 100;
187 /* We need to check if DLB results in worse performance and then turn it off.
188 * We check this more often then for turning DLB on, because the DLB can scale
189 * the domains very rapidly, so if unlucky the load imbalance can go up quickly
190 * and furthermore, we are already synchronizing often with DLB, so
191 * the overhead of the MPI Bcast is not that high.
193 static const int c_checkTurnDlbOffInterval
= 20;
195 /* Forward declaration */
196 static void dd_dlb_set_should_check_whether_to_turn_dlb_on(gmx_domdec_t
*dd
, gmx_bool bValue
);
200 #define dd_index(n,i) ((((i)[ZZ]*(n)[YY] + (i)[YY])*(n)[XX]) + (i)[XX])
202 static void index2xyz(ivec nc,int ind,ivec xyz)
204 xyz[XX] = ind % nc[XX];
205 xyz[YY] = (ind / nc[XX]) % nc[YY];
206 xyz[ZZ] = ind / (nc[YY]*nc[XX]);
210 /* This order is required to minimize the coordinate communication in PME
211 * which uses decomposition in the x direction.
213 #define dd_index(n, i) ((((i)[XX]*(n)[YY] + (i)[YY])*(n)[ZZ]) + (i)[ZZ])
215 static void ddindex2xyz(ivec nc
, int ind
, ivec xyz
)
217 xyz
[XX
] = ind
/ (nc
[YY
]*nc
[ZZ
]);
218 xyz
[YY
] = (ind
/ nc
[ZZ
]) % nc
[YY
];
219 xyz
[ZZ
] = ind
% nc
[ZZ
];
222 static int ddcoord2ddnodeid(gmx_domdec_t
*dd
, ivec c
)
227 ddindex
= dd_index(dd
->nc
, c
);
228 if (dd
->comm
->bCartesianPP_PME
)
230 ddnodeid
= dd
->comm
->ddindex2ddnodeid
[ddindex
];
232 else if (dd
->comm
->bCartesianPP
)
235 MPI_Cart_rank(dd
->mpi_comm_all
, c
, &ddnodeid
);
246 static gmx_bool
dynamic_dd_box(const gmx_ddbox_t
*ddbox
, const t_inputrec
*ir
)
248 return (ddbox
->nboundeddim
< DIM
|| inputrecDynamicBox(ir
));
251 int ddglatnr(const gmx_domdec_t
*dd
, int i
)
261 if (i
>= dd
->comm
->nat
[ddnatNR
-1])
263 gmx_fatal(FARGS
, "glatnr called with %d, which is larger than the local number of atoms (%d)", i
, dd
->comm
->nat
[ddnatNR
-1]);
265 atnr
= dd
->gatindex
[i
] + 1;
271 t_block
*dd_charge_groups_global(gmx_domdec_t
*dd
)
273 return &dd
->comm
->cgs_gl
;
276 static bool dlbIsOn(const gmx_domdec_comm_t
*comm
)
278 return (comm
->dlbState
== edlbsOnCanTurnOff
||
279 comm
->dlbState
== edlbsOnForever
);
282 static void vec_rvec_init(vec_rvec_t
*v
)
288 static void vec_rvec_check_alloc(vec_rvec_t
*v
, int n
)
292 v
->nalloc
= over_alloc_dd(n
);
293 srenew(v
->v
, v
->nalloc
);
297 void dd_store_state(gmx_domdec_t
*dd
, t_state
*state
)
301 if (state
->ddp_count
!= dd
->ddp_count
)
303 gmx_incons("The state does not the domain decomposition state");
306 state
->cg_gl
.resize(dd
->ncg_home
);
307 for (i
= 0; i
< dd
->ncg_home
; i
++)
309 state
->cg_gl
[i
] = dd
->index_gl
[i
];
312 state
->ddp_count_cg_gl
= dd
->ddp_count
;
315 gmx_domdec_zones_t
*domdec_zones(gmx_domdec_t
*dd
)
317 return &dd
->comm
->zones
;
320 void dd_get_ns_ranges(const gmx_domdec_t
*dd
, int icg
,
321 int *jcg0
, int *jcg1
, ivec shift0
, ivec shift1
)
323 gmx_domdec_zones_t
*zones
;
326 zones
= &dd
->comm
->zones
;
329 while (icg
>= zones
->izone
[izone
].cg1
)
338 else if (izone
< zones
->nizone
)
340 *jcg0
= zones
->izone
[izone
].jcg0
;
344 gmx_fatal(FARGS
, "DD icg %d out of range: izone (%d) >= nizone (%d)",
345 icg
, izone
, zones
->nizone
);
348 *jcg1
= zones
->izone
[izone
].jcg1
;
350 for (d
= 0; d
< dd
->ndim
; d
++)
353 shift0
[dim
] = zones
->izone
[izone
].shift0
[dim
];
354 shift1
[dim
] = zones
->izone
[izone
].shift1
[dim
];
355 if (dd
->comm
->tric_dir
[dim
] || (dlbIsOn(dd
->comm
) && d
> 0))
357 /* A conservative approach, this can be optimized */
364 int dd_natoms_mdatoms(const gmx_domdec_t
*dd
)
366 /* We currently set mdatoms entries for all atoms:
367 * local + non-local + communicated for vsite + constraints
370 return dd
->comm
->nat
[ddnatNR
- 1];
373 int dd_natoms_vsite(const gmx_domdec_t
*dd
)
375 return dd
->comm
->nat
[ddnatVSITE
];
378 void dd_get_constraint_range(const gmx_domdec_t
*dd
, int *at_start
, int *at_end
)
380 *at_start
= dd
->comm
->nat
[ddnatCON
-1];
381 *at_end
= dd
->comm
->nat
[ddnatCON
];
384 void dd_move_x(gmx_domdec_t
*dd
, matrix box
, rvec x
[])
386 int nzone
, nat_tot
, n
, d
, p
, i
, j
, at0
, at1
, zone
;
387 int *index
, *cgindex
;
388 gmx_domdec_comm_t
*comm
;
389 gmx_domdec_comm_dim_t
*cd
;
390 gmx_domdec_ind_t
*ind
;
391 rvec shift
= {0, 0, 0}, *buf
, *rbuf
;
392 gmx_bool bPBC
, bScrew
;
396 cgindex
= dd
->cgindex
;
401 nat_tot
= dd
->nat_home
;
402 for (d
= 0; d
< dd
->ndim
; d
++)
404 bPBC
= (dd
->ci
[dd
->dim
[d
]] == 0);
405 bScrew
= (bPBC
&& dd
->bScrewPBC
&& dd
->dim
[d
] == XX
);
408 copy_rvec(box
[dd
->dim
[d
]], shift
);
411 for (p
= 0; p
< cd
->np
; p
++)
418 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
420 at0
= cgindex
[index
[i
]];
421 at1
= cgindex
[index
[i
]+1];
422 for (j
= at0
; j
< at1
; j
++)
424 copy_rvec(x
[j
], buf
[n
]);
431 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
433 at0
= cgindex
[index
[i
]];
434 at1
= cgindex
[index
[i
]+1];
435 for (j
= at0
; j
< at1
; j
++)
437 /* We need to shift the coordinates */
438 rvec_add(x
[j
], shift
, buf
[n
]);
445 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
447 at0
= cgindex
[index
[i
]];
448 at1
= cgindex
[index
[i
]+1];
449 for (j
= at0
; j
< at1
; j
++)
452 buf
[n
][XX
] = x
[j
][XX
] + shift
[XX
];
454 * This operation requires a special shift force
455 * treatment, which is performed in calc_vir.
457 buf
[n
][YY
] = box
[YY
][YY
] - x
[j
][YY
];
458 buf
[n
][ZZ
] = box
[ZZ
][ZZ
] - x
[j
][ZZ
];
470 rbuf
= comm
->vbuf2
.v
;
472 /* Send and receive the coordinates */
473 dd_sendrecv_rvec(dd
, d
, dddirBackward
,
474 buf
, ind
->nsend
[nzone
+1],
475 rbuf
, ind
->nrecv
[nzone
+1]);
479 for (zone
= 0; zone
< nzone
; zone
++)
481 for (i
= ind
->cell2at0
[zone
]; i
< ind
->cell2at1
[zone
]; i
++)
483 copy_rvec(rbuf
[j
], x
[i
]);
488 nat_tot
+= ind
->nrecv
[nzone
+1];
494 void dd_move_f(gmx_domdec_t
*dd
, rvec f
[], rvec
*fshift
)
496 int nzone
, nat_tot
, n
, d
, p
, i
, j
, at0
, at1
, zone
;
497 int *index
, *cgindex
;
498 gmx_domdec_comm_t
*comm
;
499 gmx_domdec_comm_dim_t
*cd
;
500 gmx_domdec_ind_t
*ind
;
504 gmx_bool bShiftForcesNeedPbc
, bScrew
;
508 cgindex
= dd
->cgindex
;
512 nzone
= comm
->zones
.n
/2;
513 nat_tot
= dd
->nat_tot
;
514 for (d
= dd
->ndim
-1; d
>= 0; d
--)
516 /* Only forces in domains near the PBC boundaries need to
517 consider PBC in the treatment of fshift */
518 bShiftForcesNeedPbc
= (dd
->ci
[dd
->dim
[d
]] == 0);
519 bScrew
= (bShiftForcesNeedPbc
&& dd
->bScrewPBC
&& dd
->dim
[d
] == XX
);
520 if (fshift
== nullptr && !bScrew
)
522 bShiftForcesNeedPbc
= FALSE
;
524 /* Determine which shift vector we need */
530 for (p
= cd
->np
-1; p
>= 0; p
--)
533 nat_tot
-= ind
->nrecv
[nzone
+1];
540 sbuf
= comm
->vbuf2
.v
;
542 for (zone
= 0; zone
< nzone
; zone
++)
544 for (i
= ind
->cell2at0
[zone
]; i
< ind
->cell2at1
[zone
]; i
++)
546 copy_rvec(f
[i
], sbuf
[j
]);
551 /* Communicate the forces */
552 dd_sendrecv_rvec(dd
, d
, dddirForward
,
553 sbuf
, ind
->nrecv
[nzone
+1],
554 buf
, ind
->nsend
[nzone
+1]);
556 /* Add the received forces */
558 if (!bShiftForcesNeedPbc
)
560 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
562 at0
= cgindex
[index
[i
]];
563 at1
= cgindex
[index
[i
]+1];
564 for (j
= at0
; j
< at1
; j
++)
566 rvec_inc(f
[j
], buf
[n
]);
573 /* fshift should always be defined if this function is
574 * called when bShiftForcesNeedPbc is true */
575 assert(NULL
!= fshift
);
576 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
578 at0
= cgindex
[index
[i
]];
579 at1
= cgindex
[index
[i
]+1];
580 for (j
= at0
; j
< at1
; j
++)
582 rvec_inc(f
[j
], buf
[n
]);
583 /* Add this force to the shift force */
584 rvec_inc(fshift
[is
], buf
[n
]);
591 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
593 at0
= cgindex
[index
[i
]];
594 at1
= cgindex
[index
[i
]+1];
595 for (j
= at0
; j
< at1
; j
++)
597 /* Rotate the force */
598 f
[j
][XX
] += buf
[n
][XX
];
599 f
[j
][YY
] -= buf
[n
][YY
];
600 f
[j
][ZZ
] -= buf
[n
][ZZ
];
603 /* Add this force to the shift force */
604 rvec_inc(fshift
[is
], buf
[n
]);
615 void dd_atom_spread_real(gmx_domdec_t
*dd
, real v
[])
617 int nzone
, nat_tot
, n
, d
, p
, i
, j
, at0
, at1
, zone
;
618 int *index
, *cgindex
;
619 gmx_domdec_comm_t
*comm
;
620 gmx_domdec_comm_dim_t
*cd
;
621 gmx_domdec_ind_t
*ind
;
626 cgindex
= dd
->cgindex
;
628 buf
= &comm
->vbuf
.v
[0][0];
631 nat_tot
= dd
->nat_home
;
632 for (d
= 0; d
< dd
->ndim
; d
++)
635 for (p
= 0; p
< cd
->np
; p
++)
640 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
642 at0
= cgindex
[index
[i
]];
643 at1
= cgindex
[index
[i
]+1];
644 for (j
= at0
; j
< at1
; j
++)
657 rbuf
= &comm
->vbuf2
.v
[0][0];
659 /* Send and receive the coordinates */
660 dd_sendrecv_real(dd
, d
, dddirBackward
,
661 buf
, ind
->nsend
[nzone
+1],
662 rbuf
, ind
->nrecv
[nzone
+1]);
666 for (zone
= 0; zone
< nzone
; zone
++)
668 for (i
= ind
->cell2at0
[zone
]; i
< ind
->cell2at1
[zone
]; i
++)
675 nat_tot
+= ind
->nrecv
[nzone
+1];
681 void dd_atom_sum_real(gmx_domdec_t
*dd
, real v
[])
683 int nzone
, nat_tot
, n
, d
, p
, i
, j
, at0
, at1
, zone
;
684 int *index
, *cgindex
;
685 gmx_domdec_comm_t
*comm
;
686 gmx_domdec_comm_dim_t
*cd
;
687 gmx_domdec_ind_t
*ind
;
692 cgindex
= dd
->cgindex
;
694 buf
= &comm
->vbuf
.v
[0][0];
696 nzone
= comm
->zones
.n
/2;
697 nat_tot
= dd
->nat_tot
;
698 for (d
= dd
->ndim
-1; d
>= 0; d
--)
701 for (p
= cd
->np
-1; p
>= 0; p
--)
704 nat_tot
-= ind
->nrecv
[nzone
+1];
711 sbuf
= &comm
->vbuf2
.v
[0][0];
713 for (zone
= 0; zone
< nzone
; zone
++)
715 for (i
= ind
->cell2at0
[zone
]; i
< ind
->cell2at1
[zone
]; i
++)
722 /* Communicate the forces */
723 dd_sendrecv_real(dd
, d
, dddirForward
,
724 sbuf
, ind
->nrecv
[nzone
+1],
725 buf
, ind
->nsend
[nzone
+1]);
727 /* Add the received forces */
729 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
731 at0
= cgindex
[index
[i
]];
732 at1
= cgindex
[index
[i
]+1];
733 for (j
= at0
; j
< at1
; j
++)
744 static void print_ddzone(FILE *fp
, int d
, int i
, int j
, gmx_ddzone_t
*zone
)
746 fprintf(fp
, "zone d0 %d d1 %d d2 %d min0 %6.3f max1 %6.3f mch0 %6.3f mch1 %6.3f p1_0 %6.3f p1_1 %6.3f\n",
748 zone
->min0
, zone
->max1
,
749 zone
->mch0
, zone
->mch0
,
750 zone
->p1_0
, zone
->p1_1
);
754 #define DDZONECOMM_MAXZONE 5
755 #define DDZONECOMM_BUFSIZE 3
757 static void dd_sendrecv_ddzone(const gmx_domdec_t
*dd
,
758 int ddimind
, int direction
,
759 gmx_ddzone_t
*buf_s
, int n_s
,
760 gmx_ddzone_t
*buf_r
, int n_r
)
762 #define ZBS DDZONECOMM_BUFSIZE
763 rvec vbuf_s
[DDZONECOMM_MAXZONE
*ZBS
];
764 rvec vbuf_r
[DDZONECOMM_MAXZONE
*ZBS
];
767 for (i
= 0; i
< n_s
; i
++)
769 vbuf_s
[i
*ZBS
][0] = buf_s
[i
].min0
;
770 vbuf_s
[i
*ZBS
][1] = buf_s
[i
].max1
;
771 vbuf_s
[i
*ZBS
][2] = buf_s
[i
].min1
;
772 vbuf_s
[i
*ZBS
+1][0] = buf_s
[i
].mch0
;
773 vbuf_s
[i
*ZBS
+1][1] = buf_s
[i
].mch1
;
774 vbuf_s
[i
*ZBS
+1][2] = 0;
775 vbuf_s
[i
*ZBS
+2][0] = buf_s
[i
].p1_0
;
776 vbuf_s
[i
*ZBS
+2][1] = buf_s
[i
].p1_1
;
777 vbuf_s
[i
*ZBS
+2][2] = 0;
780 dd_sendrecv_rvec(dd
, ddimind
, direction
,
784 for (i
= 0; i
< n_r
; i
++)
786 buf_r
[i
].min0
= vbuf_r
[i
*ZBS
][0];
787 buf_r
[i
].max1
= vbuf_r
[i
*ZBS
][1];
788 buf_r
[i
].min1
= vbuf_r
[i
*ZBS
][2];
789 buf_r
[i
].mch0
= vbuf_r
[i
*ZBS
+1][0];
790 buf_r
[i
].mch1
= vbuf_r
[i
*ZBS
+1][1];
791 buf_r
[i
].p1_0
= vbuf_r
[i
*ZBS
+2][0];
792 buf_r
[i
].p1_1
= vbuf_r
[i
*ZBS
+2][1];
798 static void dd_move_cellx(gmx_domdec_t
*dd
, gmx_ddbox_t
*ddbox
,
799 rvec cell_ns_x0
, rvec cell_ns_x1
)
801 int d
, d1
, dim
, pos
, buf_size
, i
, j
, p
, npulse
, npulse_min
;
803 gmx_ddzone_t buf_s
[DDZONECOMM_MAXZONE
];
804 gmx_ddzone_t buf_r
[DDZONECOMM_MAXZONE
];
805 gmx_ddzone_t buf_e
[DDZONECOMM_MAXZONE
];
806 rvec extr_s
[2], extr_r
[2];
808 real dist_d
, c
= 0, det
;
809 gmx_domdec_comm_t
*comm
;
814 for (d
= 1; d
< dd
->ndim
; d
++)
817 zp
= (d
== 1) ? &comm
->zone_d1
[0] : &comm
->zone_d2
[0][0];
818 zp
->min0
= cell_ns_x0
[dim
];
819 zp
->max1
= cell_ns_x1
[dim
];
820 zp
->min1
= cell_ns_x1
[dim
];
821 zp
->mch0
= cell_ns_x0
[dim
];
822 zp
->mch1
= cell_ns_x1
[dim
];
823 zp
->p1_0
= cell_ns_x0
[dim
];
824 zp
->p1_1
= cell_ns_x1
[dim
];
827 for (d
= dd
->ndim
-2; d
>= 0; d
--)
830 bPBC
= (dim
< ddbox
->npbcdim
);
832 /* Use an rvec to store two reals */
833 extr_s
[d
][0] = comm
->cell_f0
[d
+1];
834 extr_s
[d
][1] = comm
->cell_f1
[d
+1];
835 extr_s
[d
][2] = comm
->cell_f1
[d
+1];
838 /* Store the extremes in the backward sending buffer,
839 * so the get updated separately from the forward communication.
841 for (d1
= d
; d1
< dd
->ndim
-1; d1
++)
843 /* We invert the order to be able to use the same loop for buf_e */
844 buf_s
[pos
].min0
= extr_s
[d1
][1];
845 buf_s
[pos
].max1
= extr_s
[d1
][0];
846 buf_s
[pos
].min1
= extr_s
[d1
][2];
849 /* Store the cell corner of the dimension we communicate along */
850 buf_s
[pos
].p1_0
= comm
->cell_x0
[dim
];
855 buf_s
[pos
] = (dd
->ndim
== 2) ? comm
->zone_d1
[0] : comm
->zone_d2
[0][0];
858 if (dd
->ndim
== 3 && d
== 0)
860 buf_s
[pos
] = comm
->zone_d2
[0][1];
862 buf_s
[pos
] = comm
->zone_d1
[0];
866 /* We only need to communicate the extremes
867 * in the forward direction
869 npulse
= comm
->cd
[d
].np
;
872 /* Take the minimum to avoid double communication */
873 npulse_min
= std::min(npulse
, dd
->nc
[dim
]-1-npulse
);
877 /* Without PBC we should really not communicate over
878 * the boundaries, but implementing that complicates
879 * the communication setup and therefore we simply
880 * do all communication, but ignore some data.
884 for (p
= 0; p
< npulse_min
; p
++)
886 /* Communicate the extremes forward */
887 bUse
= (bPBC
|| dd
->ci
[dim
] > 0);
889 dd_sendrecv_rvec(dd
, d
, dddirForward
,
890 extr_s
+d
, dd
->ndim
-d
-1,
891 extr_r
+d
, dd
->ndim
-d
-1);
895 for (d1
= d
; d1
< dd
->ndim
-1; d1
++)
897 extr_s
[d1
][0] = std::max(extr_s
[d1
][0], extr_r
[d1
][0]);
898 extr_s
[d1
][1] = std::min(extr_s
[d1
][1], extr_r
[d1
][1]);
899 extr_s
[d1
][2] = std::min(extr_s
[d1
][2], extr_r
[d1
][2]);
905 for (p
= 0; p
< npulse
; p
++)
907 /* Communicate all the zone information backward */
908 bUse
= (bPBC
|| dd
->ci
[dim
] < dd
->nc
[dim
] - 1);
910 dd_sendrecv_ddzone(dd
, d
, dddirBackward
,
917 for (d1
= d
+1; d1
< dd
->ndim
; d1
++)
919 /* Determine the decrease of maximum required
920 * communication height along d1 due to the distance along d,
921 * this avoids a lot of useless atom communication.
923 dist_d
= comm
->cell_x1
[dim
] - buf_r
[0].p1_0
;
925 if (ddbox
->tric_dir
[dim
])
927 /* c is the off-diagonal coupling between the cell planes
928 * along directions d and d1.
930 c
= ddbox
->v
[dim
][dd
->dim
[d1
]][dim
];
936 det
= (1 + c
*c
)*comm
->cutoff
*comm
->cutoff
- dist_d
*dist_d
;
939 dh
[d1
] = comm
->cutoff
- (c
*dist_d
+ std::sqrt(det
))/(1 + c
*c
);
943 /* A negative value signals out of range */
949 /* Accumulate the extremes over all pulses */
950 for (i
= 0; i
< buf_size
; i
++)
960 buf_e
[i
].min0
= std::min(buf_e
[i
].min0
, buf_r
[i
].min0
);
961 buf_e
[i
].max1
= std::max(buf_e
[i
].max1
, buf_r
[i
].max1
);
962 buf_e
[i
].min1
= std::min(buf_e
[i
].min1
, buf_r
[i
].min1
);
965 if (dd
->ndim
== 3 && d
== 0 && i
== buf_size
- 1)
973 if (bUse
&& dh
[d1
] >= 0)
975 buf_e
[i
].mch0
= std::max(buf_e
[i
].mch0
, buf_r
[i
].mch0
-dh
[d1
]);
976 buf_e
[i
].mch1
= std::max(buf_e
[i
].mch1
, buf_r
[i
].mch1
-dh
[d1
]);
979 /* Copy the received buffer to the send buffer,
980 * to pass the data through with the next pulse.
984 if (((bPBC
|| dd
->ci
[dim
]+npulse
< dd
->nc
[dim
]) && p
== npulse
-1) ||
985 (!bPBC
&& dd
->ci
[dim
]+1+p
== dd
->nc
[dim
]-1))
987 /* Store the extremes */
990 for (d1
= d
; d1
< dd
->ndim
-1; d1
++)
992 extr_s
[d1
][1] = std::min(extr_s
[d1
][1], buf_e
[pos
].min0
);
993 extr_s
[d1
][0] = std::max(extr_s
[d1
][0], buf_e
[pos
].max1
);
994 extr_s
[d1
][2] = std::min(extr_s
[d1
][2], buf_e
[pos
].min1
);
998 if (d
== 1 || (d
== 0 && dd
->ndim
== 3))
1000 for (i
= d
; i
< 2; i
++)
1002 comm
->zone_d2
[1-d
][i
] = buf_e
[pos
];
1008 comm
->zone_d1
[1] = buf_e
[pos
];
1018 for (i
= 0; i
< 2; i
++)
1022 print_ddzone(debug
, 1, i
, 0, &comm
->zone_d1
[i
]);
1024 cell_ns_x0
[dim
] = std::min(cell_ns_x0
[dim
], comm
->zone_d1
[i
].min0
);
1025 cell_ns_x1
[dim
] = std::max(cell_ns_x1
[dim
], comm
->zone_d1
[i
].max1
);
1031 for (i
= 0; i
< 2; i
++)
1033 for (j
= 0; j
< 2; j
++)
1037 print_ddzone(debug
, 2, i
, j
, &comm
->zone_d2
[i
][j
]);
1039 cell_ns_x0
[dim
] = std::min(cell_ns_x0
[dim
], comm
->zone_d2
[i
][j
].min0
);
1040 cell_ns_x1
[dim
] = std::max(cell_ns_x1
[dim
], comm
->zone_d2
[i
][j
].max1
);
1044 for (d
= 1; d
< dd
->ndim
; d
++)
1046 comm
->cell_f_max0
[d
] = extr_s
[d
-1][0];
1047 comm
->cell_f_min1
[d
] = extr_s
[d
-1][1];
1050 fprintf(debug
, "Cell fraction d %d, max0 %f, min1 %f\n",
1051 d
, comm
->cell_f_max0
[d
], comm
->cell_f_min1
[d
]);
1056 static void dd_collect_cg(gmx_domdec_t
*dd
,
1057 t_state
*state_local
)
1059 gmx_domdec_master_t
*ma
= nullptr;
1060 int buf2
[2], *ibuf
, i
, ncg_home
= 0, *cg
= nullptr, nat_home
= 0;
1062 if (state_local
->ddp_count
== dd
->comm
->master_cg_ddp_count
)
1064 /* The master has the correct distribution */
1068 if (state_local
->ddp_count
== dd
->ddp_count
)
1070 /* The local state and DD are in sync, use the DD indices */
1071 ncg_home
= dd
->ncg_home
;
1073 nat_home
= dd
->nat_home
;
1075 else if (state_local
->ddp_count_cg_gl
== state_local
->ddp_count
)
1077 /* The DD is out of sync with the local state, but we have stored
1078 * the cg indices with the local state, so we can use those.
1082 cgs_gl
= &dd
->comm
->cgs_gl
;
1084 ncg_home
= state_local
->cg_gl
.size();
1085 cg
= state_local
->cg_gl
.data();
1087 for (i
= 0; i
< ncg_home
; i
++)
1089 nat_home
+= cgs_gl
->index
[cg
[i
]+1] - cgs_gl
->index
[cg
[i
]];
1094 gmx_incons("Attempted to collect a vector for a state for which the charge group distribution is unknown");
1108 /* Collect the charge group and atom counts on the master */
1109 dd_gather(dd
, 2*sizeof(int), buf2
, ibuf
);
1114 for (i
= 0; i
< dd
->nnodes
; i
++)
1116 ma
->ncg
[i
] = ma
->ibuf
[2*i
];
1117 ma
->nat
[i
] = ma
->ibuf
[2*i
+1];
1118 ma
->index
[i
+1] = ma
->index
[i
] + ma
->ncg
[i
];
1121 /* Make byte counts and indices */
1122 for (i
= 0; i
< dd
->nnodes
; i
++)
1124 ma
->ibuf
[i
] = ma
->ncg
[i
]*sizeof(int);
1125 ma
->ibuf
[dd
->nnodes
+i
] = ma
->index
[i
]*sizeof(int);
1129 fprintf(debug
, "Initial charge group distribution: ");
1130 for (i
= 0; i
< dd
->nnodes
; i
++)
1132 fprintf(debug
, " %d", ma
->ncg
[i
]);
1134 fprintf(debug
, "\n");
1138 /* Collect the charge group indices on the master */
1140 ncg_home
*sizeof(int), cg
,
1141 DDMASTER(dd
) ? ma
->ibuf
: nullptr,
1142 DDMASTER(dd
) ? ma
->ibuf
+dd
->nnodes
: nullptr,
1143 DDMASTER(dd
) ? ma
->cg
: nullptr);
1145 dd
->comm
->master_cg_ddp_count
= state_local
->ddp_count
;
1148 static void dd_collect_vec_sendrecv(gmx_domdec_t
*dd
,
1149 const rvec
*lv
, rvec
*v
)
1151 gmx_domdec_master_t
*ma
;
1152 int n
, i
, c
, a
, nalloc
= 0;
1153 rvec
*buf
= nullptr;
1161 MPI_Send(const_cast<void *>(static_cast<const void *>(lv
)), dd
->nat_home
*sizeof(rvec
), MPI_BYTE
,
1162 DDMASTERRANK(dd
), dd
->rank
, dd
->mpi_comm_all
);
1167 /* Copy the master coordinates to the global array */
1168 cgs_gl
= &dd
->comm
->cgs_gl
;
1170 n
= DDMASTERRANK(dd
);
1172 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1174 for (c
= cgs_gl
->index
[ma
->cg
[i
]]; c
< cgs_gl
->index
[ma
->cg
[i
]+1]; c
++)
1176 copy_rvec(lv
[a
++], v
[c
]);
1180 for (n
= 0; n
< dd
->nnodes
; n
++)
1184 if (ma
->nat
[n
] > nalloc
)
1186 nalloc
= over_alloc_dd(ma
->nat
[n
]);
1187 srenew(buf
, nalloc
);
1190 MPI_Recv(buf
, ma
->nat
[n
]*sizeof(rvec
), MPI_BYTE
, DDRANK(dd
, n
),
1191 n
, dd
->mpi_comm_all
, MPI_STATUS_IGNORE
);
1194 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1196 for (c
= cgs_gl
->index
[ma
->cg
[i
]]; c
< cgs_gl
->index
[ma
->cg
[i
]+1]; c
++)
1198 copy_rvec(buf
[a
++], v
[c
]);
1207 static void get_commbuffer_counts(gmx_domdec_t
*dd
,
1208 int **counts
, int **disps
)
1210 gmx_domdec_master_t
*ma
;
1215 /* Make the rvec count and displacment arrays */
1217 *disps
= ma
->ibuf
+ dd
->nnodes
;
1218 for (n
= 0; n
< dd
->nnodes
; n
++)
1220 (*counts
)[n
] = ma
->nat
[n
]*sizeof(rvec
);
1221 (*disps
)[n
] = (n
== 0 ? 0 : (*disps
)[n
-1] + (*counts
)[n
-1]);
1225 static void dd_collect_vec_gatherv(gmx_domdec_t
*dd
,
1226 const rvec
*lv
, rvec
*v
)
1228 gmx_domdec_master_t
*ma
;
1229 int *rcounts
= nullptr, *disps
= nullptr;
1231 rvec
*buf
= nullptr;
1238 get_commbuffer_counts(dd
, &rcounts
, &disps
);
1243 dd_gatherv(dd
, dd
->nat_home
*sizeof(rvec
), lv
, rcounts
, disps
, buf
);
1247 cgs_gl
= &dd
->comm
->cgs_gl
;
1250 for (n
= 0; n
< dd
->nnodes
; n
++)
1252 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1254 for (c
= cgs_gl
->index
[ma
->cg
[i
]]; c
< cgs_gl
->index
[ma
->cg
[i
]+1]; c
++)
1256 copy_rvec(buf
[a
++], v
[c
]);
1263 void dd_collect_vec(gmx_domdec_t
*dd
,
1264 t_state
*state_local
,
1265 const PaddedRVecVector
*localVector
,
1268 dd_collect_cg(dd
, state_local
);
1270 const rvec
*lv
= as_rvec_array(localVector
->data());
1272 if (dd
->nnodes
<= GMX_DD_NNODES_SENDRECV
)
1274 dd_collect_vec_sendrecv(dd
, lv
, v
);
1278 dd_collect_vec_gatherv(dd
, lv
, v
);
1282 void dd_collect_vec(gmx_domdec_t
*dd
,
1283 t_state
*state_local
,
1284 const PaddedRVecVector
*localVector
,
1285 PaddedRVecVector
*vector
)
1287 dd_collect_vec(dd
, state_local
, localVector
, as_rvec_array(vector
->data()));
1291 void dd_collect_state(gmx_domdec_t
*dd
,
1292 t_state
*state_local
, t_state
*state
)
1294 int nh
= state
->nhchainlength
;
1298 for (int i
= 0; i
< efptNR
; i
++)
1300 state
->lambda
[i
] = state_local
->lambda
[i
];
1302 state
->fep_state
= state_local
->fep_state
;
1303 state
->veta
= state_local
->veta
;
1304 state
->vol0
= state_local
->vol0
;
1305 copy_mat(state_local
->box
, state
->box
);
1306 copy_mat(state_local
->boxv
, state
->boxv
);
1307 copy_mat(state_local
->svir_prev
, state
->svir_prev
);
1308 copy_mat(state_local
->fvir_prev
, state
->fvir_prev
);
1309 copy_mat(state_local
->pres_prev
, state
->pres_prev
);
1311 for (int i
= 0; i
< state_local
->ngtc
; i
++)
1313 for (int j
= 0; j
< nh
; j
++)
1315 state
->nosehoover_xi
[i
*nh
+j
] = state_local
->nosehoover_xi
[i
*nh
+j
];
1316 state
->nosehoover_vxi
[i
*nh
+j
] = state_local
->nosehoover_vxi
[i
*nh
+j
];
1318 state
->therm_integral
[i
] = state_local
->therm_integral
[i
];
1320 for (int i
= 0; i
< state_local
->nnhpres
; i
++)
1322 for (int j
= 0; j
< nh
; j
++)
1324 state
->nhpres_xi
[i
*nh
+j
] = state_local
->nhpres_xi
[i
*nh
+j
];
1325 state
->nhpres_vxi
[i
*nh
+j
] = state_local
->nhpres_vxi
[i
*nh
+j
];
1329 if (state_local
->flags
& (1 << estX
))
1331 dd_collect_vec(dd
, state_local
, &state_local
->x
, &state
->x
);
1333 if (state_local
->flags
& (1 << estV
))
1335 dd_collect_vec(dd
, state_local
, &state_local
->v
, &state
->v
);
1337 if (state_local
->flags
& (1 << estCGP
))
1339 dd_collect_vec(dd
, state_local
, &state_local
->cg_p
, &state
->cg_p
);
1343 static void dd_resize_state(t_state
*state
, PaddedRVecVector
*f
, int natoms
)
1347 fprintf(debug
, "Resizing state: currently %d, required %d\n", state
->natoms
, natoms
);
1350 state_change_natoms(state
, natoms
);
1354 /* We need to allocate one element extra, since we might use
1355 * (unaligned) 4-wide SIMD loads to access rvec entries.
1357 f
->resize(natoms
+ 1);
1361 static void dd_check_alloc_ncg(t_forcerec
*fr
,
1363 PaddedRVecVector
*f
,
1364 int numChargeGroups
)
1366 if (numChargeGroups
> fr
->cg_nalloc
)
1370 fprintf(debug
, "Reallocating forcerec: currently %d, required %d, allocating %d\n", fr
->cg_nalloc
, numChargeGroups
, over_alloc_dd(numChargeGroups
));
1372 fr
->cg_nalloc
= over_alloc_dd(numChargeGroups
);
1373 srenew(fr
->cginfo
, fr
->cg_nalloc
);
1374 if (fr
->cutoff_scheme
== ecutsGROUP
)
1376 srenew(fr
->cg_cm
, fr
->cg_nalloc
);
1379 if (fr
->cutoff_scheme
== ecutsVERLET
)
1381 /* We don't use charge groups, we use x in state to set up
1382 * the atom communication.
1384 dd_resize_state(state
, f
, numChargeGroups
);
1388 static void dd_distribute_vec_sendrecv(gmx_domdec_t
*dd
, t_block
*cgs
,
1391 gmx_domdec_master_t
*ma
;
1392 int n
, i
, c
, a
, nalloc
= 0;
1393 rvec
*buf
= nullptr;
1399 for (n
= 0; n
< dd
->nnodes
; n
++)
1403 if (ma
->nat
[n
] > nalloc
)
1405 nalloc
= over_alloc_dd(ma
->nat
[n
]);
1406 srenew(buf
, nalloc
);
1408 /* Use lv as a temporary buffer */
1410 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1412 for (c
= cgs
->index
[ma
->cg
[i
]]; c
< cgs
->index
[ma
->cg
[i
]+1]; c
++)
1414 copy_rvec(v
[c
], buf
[a
++]);
1417 if (a
!= ma
->nat
[n
])
1419 gmx_fatal(FARGS
, "Internal error a (%d) != nat (%d)",
1424 MPI_Send(buf
, ma
->nat
[n
]*sizeof(rvec
), MPI_BYTE
,
1425 DDRANK(dd
, n
), n
, dd
->mpi_comm_all
);
1430 n
= DDMASTERRANK(dd
);
1432 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1434 for (c
= cgs
->index
[ma
->cg
[i
]]; c
< cgs
->index
[ma
->cg
[i
]+1]; c
++)
1436 copy_rvec(v
[c
], lv
[a
++]);
1443 MPI_Recv(lv
, dd
->nat_home
*sizeof(rvec
), MPI_BYTE
, DDMASTERRANK(dd
),
1444 MPI_ANY_TAG
, dd
->mpi_comm_all
, MPI_STATUS_IGNORE
);
1449 static void dd_distribute_vec_scatterv(gmx_domdec_t
*dd
, t_block
*cgs
,
1452 gmx_domdec_master_t
*ma
;
1453 int *scounts
= nullptr, *disps
= nullptr;
1455 rvec
*buf
= nullptr;
1461 get_commbuffer_counts(dd
, &scounts
, &disps
);
1465 for (n
= 0; n
< dd
->nnodes
; n
++)
1467 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1469 for (c
= cgs
->index
[ma
->cg
[i
]]; c
< cgs
->index
[ma
->cg
[i
]+1]; c
++)
1471 copy_rvec(v
[c
], buf
[a
++]);
1477 dd_scatterv(dd
, scounts
, disps
, buf
, dd
->nat_home
*sizeof(rvec
), lv
);
1480 static void dd_distribute_vec(gmx_domdec_t
*dd
, t_block
*cgs
, rvec
*v
, rvec
*lv
)
1482 if (dd
->nnodes
<= GMX_DD_NNODES_SENDRECV
)
1484 dd_distribute_vec_sendrecv(dd
, cgs
, v
, lv
);
1488 dd_distribute_vec_scatterv(dd
, cgs
, v
, lv
);
1492 static void dd_distribute_dfhist(gmx_domdec_t
*dd
, df_history_t
*dfhist
)
1494 if (dfhist
== nullptr)
1499 dd_bcast(dd
, sizeof(int), &dfhist
->bEquil
);
1500 dd_bcast(dd
, sizeof(int), &dfhist
->nlambda
);
1501 dd_bcast(dd
, sizeof(real
), &dfhist
->wl_delta
);
1503 if (dfhist
->nlambda
> 0)
1505 int nlam
= dfhist
->nlambda
;
1506 dd_bcast(dd
, sizeof(int)*nlam
, dfhist
->n_at_lam
);
1507 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->wl_histo
);
1508 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->sum_weights
);
1509 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->sum_dg
);
1510 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->sum_minvar
);
1511 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->sum_variance
);
1513 for (int i
= 0; i
< nlam
; i
++)
1515 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->accum_p
[i
]);
1516 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->accum_m
[i
]);
1517 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->accum_p2
[i
]);
1518 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->accum_m2
[i
]);
1519 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->Tij
[i
]);
1520 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->Tij_empirical
[i
]);
1525 static void dd_distribute_state(gmx_domdec_t
*dd
, t_block
*cgs
,
1526 t_state
*state
, t_state
*state_local
,
1527 PaddedRVecVector
*f
)
1529 int nh
= state
->nhchainlength
;
1533 for (int i
= 0; i
< efptNR
; i
++)
1535 state_local
->lambda
[i
] = state
->lambda
[i
];
1537 state_local
->fep_state
= state
->fep_state
;
1538 state_local
->veta
= state
->veta
;
1539 state_local
->vol0
= state
->vol0
;
1540 copy_mat(state
->box
, state_local
->box
);
1541 copy_mat(state
->box_rel
, state_local
->box_rel
);
1542 copy_mat(state
->boxv
, state_local
->boxv
);
1543 copy_mat(state
->svir_prev
, state_local
->svir_prev
);
1544 copy_mat(state
->fvir_prev
, state_local
->fvir_prev
);
1545 if (state
->dfhist
!= nullptr)
1547 copy_df_history(state_local
->dfhist
, state
->dfhist
);
1549 for (int i
= 0; i
< state_local
->ngtc
; i
++)
1551 for (int j
= 0; j
< nh
; j
++)
1553 state_local
->nosehoover_xi
[i
*nh
+j
] = state
->nosehoover_xi
[i
*nh
+j
];
1554 state_local
->nosehoover_vxi
[i
*nh
+j
] = state
->nosehoover_vxi
[i
*nh
+j
];
1556 state_local
->therm_integral
[i
] = state
->therm_integral
[i
];
1558 for (int i
= 0; i
< state_local
->nnhpres
; i
++)
1560 for (int j
= 0; j
< nh
; j
++)
1562 state_local
->nhpres_xi
[i
*nh
+j
] = state
->nhpres_xi
[i
*nh
+j
];
1563 state_local
->nhpres_vxi
[i
*nh
+j
] = state
->nhpres_vxi
[i
*nh
+j
];
1567 dd_bcast(dd
, ((efptNR
)*sizeof(real
)), state_local
->lambda
.data());
1568 dd_bcast(dd
, sizeof(int), &state_local
->fep_state
);
1569 dd_bcast(dd
, sizeof(real
), &state_local
->veta
);
1570 dd_bcast(dd
, sizeof(real
), &state_local
->vol0
);
1571 dd_bcast(dd
, sizeof(state_local
->box
), state_local
->box
);
1572 dd_bcast(dd
, sizeof(state_local
->box_rel
), state_local
->box_rel
);
1573 dd_bcast(dd
, sizeof(state_local
->boxv
), state_local
->boxv
);
1574 dd_bcast(dd
, sizeof(state_local
->svir_prev
), state_local
->svir_prev
);
1575 dd_bcast(dd
, sizeof(state_local
->fvir_prev
), state_local
->fvir_prev
);
1576 dd_bcast(dd
, ((state_local
->ngtc
*nh
)*sizeof(double)), state_local
->nosehoover_xi
.data());
1577 dd_bcast(dd
, ((state_local
->ngtc
*nh
)*sizeof(double)), state_local
->nosehoover_vxi
.data());
1578 dd_bcast(dd
, state_local
->ngtc
*sizeof(double), state_local
->therm_integral
.data());
1579 dd_bcast(dd
, ((state_local
->nnhpres
*nh
)*sizeof(double)), state_local
->nhpres_xi
.data());
1580 dd_bcast(dd
, ((state_local
->nnhpres
*nh
)*sizeof(double)), state_local
->nhpres_vxi
.data());
1582 /* communicate df_history -- required for restarting from checkpoint */
1583 dd_distribute_dfhist(dd
, state_local
->dfhist
);
1585 dd_resize_state(state_local
, f
, dd
->nat_home
);
1587 if (state_local
->flags
& (1 << estX
))
1589 dd_distribute_vec(dd
, cgs
, as_rvec_array(state
->x
.data()), as_rvec_array(state_local
->x
.data()));
1591 if (state_local
->flags
& (1 << estV
))
1593 dd_distribute_vec(dd
, cgs
, as_rvec_array(state
->v
.data()), as_rvec_array(state_local
->v
.data()));
1595 if (state_local
->flags
& (1 << estCGP
))
1597 dd_distribute_vec(dd
, cgs
, as_rvec_array(state
->cg_p
.data()), as_rvec_array(state_local
->cg_p
.data()));
1601 static char dim2char(int dim
)
1607 case XX
: c
= 'X'; break;
1608 case YY
: c
= 'Y'; break;
1609 case ZZ
: c
= 'Z'; break;
1610 default: gmx_fatal(FARGS
, "Unknown dim %d", dim
);
1616 static void write_dd_grid_pdb(const char *fn
, gmx_int64_t step
,
1617 gmx_domdec_t
*dd
, matrix box
, gmx_ddbox_t
*ddbox
)
1619 rvec grid_s
[2], *grid_r
= nullptr, cx
, r
;
1620 char fname
[STRLEN
], buf
[22];
1622 int a
, i
, d
, z
, y
, x
;
1626 copy_rvec(dd
->comm
->cell_x0
, grid_s
[0]);
1627 copy_rvec(dd
->comm
->cell_x1
, grid_s
[1]);
1631 snew(grid_r
, 2*dd
->nnodes
);
1634 dd_gather(dd
, 2*sizeof(rvec
), grid_s
, DDMASTER(dd
) ? grid_r
: nullptr);
1638 for (d
= 0; d
< DIM
; d
++)
1640 for (i
= 0; i
< DIM
; i
++)
1648 if (d
< ddbox
->npbcdim
&& dd
->nc
[d
] > 1)
1650 tric
[d
][i
] = box
[i
][d
]/box
[i
][i
];
1659 sprintf(fname
, "%s_%s.pdb", fn
, gmx_step_str(step
, buf
));
1660 out
= gmx_fio_fopen(fname
, "w");
1661 gmx_write_pdb_box(out
, dd
->bScrewPBC
? epbcSCREW
: epbcXYZ
, box
);
1663 for (i
= 0; i
< dd
->nnodes
; i
++)
1665 vol
= dd
->nnodes
/(box
[XX
][XX
]*box
[YY
][YY
]*box
[ZZ
][ZZ
]);
1666 for (d
= 0; d
< DIM
; d
++)
1668 vol
*= grid_r
[i
*2+1][d
] - grid_r
[i
*2][d
];
1670 for (z
= 0; z
< 2; z
++)
1672 for (y
= 0; y
< 2; y
++)
1674 for (x
= 0; x
< 2; x
++)
1676 cx
[XX
] = grid_r
[i
*2+x
][XX
];
1677 cx
[YY
] = grid_r
[i
*2+y
][YY
];
1678 cx
[ZZ
] = grid_r
[i
*2+z
][ZZ
];
1680 gmx_fprintf_pdb_atomline(out
, epdbATOM
, a
++, "CA", ' ', "GLY", ' ', i
+1, ' ',
1681 10*r
[XX
], 10*r
[YY
], 10*r
[ZZ
], 1.0, vol
, "");
1685 for (d
= 0; d
< DIM
; d
++)
1687 for (x
= 0; x
< 4; x
++)
1691 case 0: y
= 1 + i
*8 + 2*x
; break;
1692 case 1: y
= 1 + i
*8 + 2*x
- (x
% 2); break;
1693 case 2: y
= 1 + i
*8 + x
; break;
1695 fprintf(out
, "%6s%5d%5d\n", "CONECT", y
, y
+(1<<d
));
1699 gmx_fio_fclose(out
);
1704 void write_dd_pdb(const char *fn
, gmx_int64_t step
, const char *title
,
1705 const gmx_mtop_t
*mtop
, t_commrec
*cr
,
1706 int natoms
, rvec x
[], matrix box
)
1708 char fname
[STRLEN
], buf
[22];
1710 int i
, ii
, resnr
, c
;
1711 const char *atomname
, *resname
;
1718 natoms
= dd
->comm
->nat
[ddnatVSITE
];
1721 sprintf(fname
, "%s_%s_n%d.pdb", fn
, gmx_step_str(step
, buf
), cr
->sim_nodeid
);
1723 out
= gmx_fio_fopen(fname
, "w");
1725 fprintf(out
, "TITLE %s\n", title
);
1726 gmx_write_pdb_box(out
, dd
->bScrewPBC
? epbcSCREW
: epbcXYZ
, box
);
1728 for (i
= 0; i
< natoms
; i
++)
1730 ii
= dd
->gatindex
[i
];
1731 mtopGetAtomAndResidueName(mtop
, ii
, &molb
, &atomname
, &resnr
, &resname
, nullptr);
1732 if (i
< dd
->comm
->nat
[ddnatZONE
])
1735 while (i
>= dd
->cgindex
[dd
->comm
->zones
.cg_range
[c
+1]])
1741 else if (i
< dd
->comm
->nat
[ddnatVSITE
])
1743 b
= dd
->comm
->zones
.n
;
1747 b
= dd
->comm
->zones
.n
+ 1;
1749 gmx_fprintf_pdb_atomline(out
, epdbATOM
, ii
+1, atomname
, ' ', resname
, ' ', resnr
, ' ',
1750 10*x
[i
][XX
], 10*x
[i
][YY
], 10*x
[i
][ZZ
], 1.0, b
, "");
1752 fprintf(out
, "TER\n");
1754 gmx_fio_fclose(out
);
1757 real
dd_cutoff_multibody(const gmx_domdec_t
*dd
)
1759 gmx_domdec_comm_t
*comm
;
1766 if (comm
->bInterCGBondeds
)
1768 if (comm
->cutoff_mbody
> 0)
1770 r
= comm
->cutoff_mbody
;
1774 /* cutoff_mbody=0 means we do not have DLB */
1775 r
= comm
->cellsize_min
[dd
->dim
[0]];
1776 for (di
= 1; di
< dd
->ndim
; di
++)
1778 r
= std::min(r
, comm
->cellsize_min
[dd
->dim
[di
]]);
1780 if (comm
->bBondComm
)
1782 r
= std::max(r
, comm
->cutoff_mbody
);
1786 r
= std::min(r
, comm
->cutoff
);
1794 real
dd_cutoff_twobody(const gmx_domdec_t
*dd
)
1798 r_mb
= dd_cutoff_multibody(dd
);
1800 return std::max(dd
->comm
->cutoff
, r_mb
);
1804 static void dd_cart_coord2pmecoord(const gmx_domdec_t
*dd
, const ivec coord
,
1809 nc
= dd
->nc
[dd
->comm
->cartpmedim
];
1810 ntot
= dd
->comm
->ntot
[dd
->comm
->cartpmedim
];
1811 copy_ivec(coord
, coord_pme
);
1812 coord_pme
[dd
->comm
->cartpmedim
] =
1813 nc
+ (coord
[dd
->comm
->cartpmedim
]*(ntot
- nc
) + (ntot
- nc
)/2)/nc
;
1816 static int ddindex2pmeindex(const gmx_domdec_t
*dd
, int ddindex
)
1821 npme
= dd
->comm
->npmenodes
;
1823 /* Here we assign a PME node to communicate with this DD node
1824 * by assuming that the major index of both is x.
1825 * We add cr->npmenodes/2 to obtain an even distribution.
1827 return (ddindex
*npme
+ npme
/2)/npp
;
1830 static int *dd_interleaved_pme_ranks(const gmx_domdec_t
*dd
)
1835 snew(pme_rank
, dd
->comm
->npmenodes
);
1837 for (i
= 0; i
< dd
->nnodes
; i
++)
1839 p0
= ddindex2pmeindex(dd
, i
);
1840 p1
= ddindex2pmeindex(dd
, i
+1);
1841 if (i
+1 == dd
->nnodes
|| p1
> p0
)
1845 fprintf(debug
, "pme_rank[%d] = %d\n", n
, i
+1+n
);
1847 pme_rank
[n
] = i
+ 1 + n
;
1855 static int gmx_ddcoord2pmeindex(t_commrec
*cr
, int x
, int y
, int z
)
1863 if (dd->comm->bCartesian) {
1864 gmx_ddindex2xyz(dd->nc,ddindex,coords);
1865 dd_coords2pmecoords(dd,coords,coords_pme);
1866 copy_ivec(dd->ntot,nc);
1867 nc[dd->cartpmedim] -= dd->nc[dd->cartpmedim];
1868 coords_pme[dd->cartpmedim] -= dd->nc[dd->cartpmedim];
1870 slab = (coords_pme[XX]*nc[YY] + coords_pme[YY])*nc[ZZ] + coords_pme[ZZ];
1872 slab = (ddindex*cr->npmenodes + cr->npmenodes/2)/dd->nnodes;
1878 slab
= ddindex2pmeindex(dd
, dd_index(dd
->nc
, coords
));
1883 static int ddcoord2simnodeid(t_commrec
*cr
, int x
, int y
, int z
)
1885 gmx_domdec_comm_t
*comm
;
1887 int ddindex
, nodeid
= -1;
1889 comm
= cr
->dd
->comm
;
1894 if (comm
->bCartesianPP_PME
)
1897 MPI_Cart_rank(cr
->mpi_comm_mysim
, coords
, &nodeid
);
1902 ddindex
= dd_index(cr
->dd
->nc
, coords
);
1903 if (comm
->bCartesianPP
)
1905 nodeid
= comm
->ddindex2simnodeid
[ddindex
];
1911 nodeid
= ddindex
+ gmx_ddcoord2pmeindex(cr
, x
, y
, z
);
1923 static int dd_simnode2pmenode(const gmx_domdec_t
*dd
,
1924 const t_commrec gmx_unused
*cr
,
1929 const gmx_domdec_comm_t
*comm
= dd
->comm
;
1931 /* This assumes a uniform x domain decomposition grid cell size */
1932 if (comm
->bCartesianPP_PME
)
1935 ivec coord
, coord_pme
;
1936 MPI_Cart_coords(cr
->mpi_comm_mysim
, sim_nodeid
, DIM
, coord
);
1937 if (coord
[comm
->cartpmedim
] < dd
->nc
[comm
->cartpmedim
])
1939 /* This is a PP node */
1940 dd_cart_coord2pmecoord(dd
, coord
, coord_pme
);
1941 MPI_Cart_rank(cr
->mpi_comm_mysim
, coord_pme
, &pmenode
);
1945 else if (comm
->bCartesianPP
)
1947 if (sim_nodeid
< dd
->nnodes
)
1949 pmenode
= dd
->nnodes
+ ddindex2pmeindex(dd
, sim_nodeid
);
1954 /* This assumes DD cells with identical x coordinates
1955 * are numbered sequentially.
1957 if (dd
->comm
->pmenodes
== nullptr)
1959 if (sim_nodeid
< dd
->nnodes
)
1961 /* The DD index equals the nodeid */
1962 pmenode
= dd
->nnodes
+ ddindex2pmeindex(dd
, sim_nodeid
);
1968 while (sim_nodeid
> dd
->comm
->pmenodes
[i
])
1972 if (sim_nodeid
< dd
->comm
->pmenodes
[i
])
1974 pmenode
= dd
->comm
->pmenodes
[i
];
1982 void get_pme_nnodes(const gmx_domdec_t
*dd
,
1983 int *npmenodes_x
, int *npmenodes_y
)
1987 *npmenodes_x
= dd
->comm
->npmenodes_x
;
1988 *npmenodes_y
= dd
->comm
->npmenodes_y
;
1997 void get_pme_ddnodes(t_commrec
*cr
, int pmenodeid
,
1998 int *nmy_ddnodes
, int **my_ddnodes
, int *node_peer
)
2002 ivec coord
, coord_pme
;
2006 snew(*my_ddnodes
, (dd
->nnodes
+cr
->npmenodes
-1)/cr
->npmenodes
);
2009 for (x
= 0; x
< dd
->nc
[XX
]; x
++)
2011 for (y
= 0; y
< dd
->nc
[YY
]; y
++)
2013 for (z
= 0; z
< dd
->nc
[ZZ
]; z
++)
2015 if (dd
->comm
->bCartesianPP_PME
)
2020 dd_cart_coord2pmecoord(dd
, coord
, coord_pme
);
2021 if (dd
->ci
[XX
] == coord_pme
[XX
] &&
2022 dd
->ci
[YY
] == coord_pme
[YY
] &&
2023 dd
->ci
[ZZ
] == coord_pme
[ZZ
])
2025 (*my_ddnodes
)[(*nmy_ddnodes
)++] = ddcoord2simnodeid(cr
, x
, y
, z
);
2030 /* The slab corresponds to the nodeid in the PME group */
2031 if (gmx_ddcoord2pmeindex(cr
, x
, y
, z
) == pmenodeid
)
2033 (*my_ddnodes
)[(*nmy_ddnodes
)++] = ddcoord2simnodeid(cr
, x
, y
, z
);
2040 /* The last PP-only node is the peer node */
2041 *node_peer
= (*my_ddnodes
)[*nmy_ddnodes
-1];
2045 fprintf(debug
, "Receive coordinates from PP ranks:");
2046 for (x
= 0; x
< *nmy_ddnodes
; x
++)
2048 fprintf(debug
, " %d", (*my_ddnodes
)[x
]);
2050 fprintf(debug
, "\n");
2054 static gmx_bool
receive_vir_ener(const gmx_domdec_t
*dd
, const t_commrec
*cr
)
2056 gmx_bool bReceive
= TRUE
;
2058 if (cr
->npmenodes
< dd
->nnodes
)
2060 gmx_domdec_comm_t
*comm
= dd
->comm
;
2061 if (comm
->bCartesianPP_PME
)
2064 int pmenode
= dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
);
2066 MPI_Cart_coords(cr
->mpi_comm_mysim
, cr
->sim_nodeid
, DIM
, coords
);
2067 coords
[comm
->cartpmedim
]++;
2068 if (coords
[comm
->cartpmedim
] < dd
->nc
[comm
->cartpmedim
])
2071 MPI_Cart_rank(cr
->mpi_comm_mysim
, coords
, &rank
);
2072 if (dd_simnode2pmenode(dd
, cr
, rank
) == pmenode
)
2074 /* This is not the last PP node for pmenode */
2079 GMX_RELEASE_ASSERT(false, "Without MPI we should not have Cartesian PP-PME with #PMEnodes < #DDnodes");
2084 int pmenode
= dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
);
2085 if (cr
->sim_nodeid
+1 < cr
->nnodes
&&
2086 dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
+1) == pmenode
)
2088 /* This is not the last PP node for pmenode */
2097 static void set_zones_ncg_home(gmx_domdec_t
*dd
)
2099 gmx_domdec_zones_t
*zones
;
2102 zones
= &dd
->comm
->zones
;
2104 zones
->cg_range
[0] = 0;
2105 for (i
= 1; i
< zones
->n
+1; i
++)
2107 zones
->cg_range
[i
] = dd
->ncg_home
;
2109 /* zone_ncg1[0] should always be equal to ncg_home */
2110 dd
->comm
->zone_ncg1
[0] = dd
->ncg_home
;
2113 static void rebuild_cgindex(gmx_domdec_t
*dd
,
2114 const int *gcgs_index
, const t_state
*state
)
2116 int * gmx_restrict dd_cg_gl
= dd
->index_gl
;
2117 int * gmx_restrict cgindex
= dd
->cgindex
;
2120 /* Copy back the global charge group indices from state
2121 * and rebuild the local charge group to atom index.
2124 for (unsigned int i
= 0; i
< state
->cg_gl
.size(); i
++)
2127 int cg_gl
= state
->cg_gl
[i
];
2128 dd_cg_gl
[i
] = cg_gl
;
2129 nat
+= gcgs_index
[cg_gl
+1] - gcgs_index
[cg_gl
];
2131 cgindex
[state
->cg_gl
.size()] = nat
;
2133 dd
->ncg_home
= state
->cg_gl
.size();
2136 set_zones_ncg_home(dd
);
2139 static int ddcginfo(const cginfo_mb_t
*cginfo_mb
, int cg
)
2141 while (cg
>= cginfo_mb
->cg_end
)
2146 return cginfo_mb
->cginfo
[(cg
- cginfo_mb
->cg_start
) % cginfo_mb
->cg_mod
];
2149 static void dd_set_cginfo(int *index_gl
, int cg0
, int cg1
,
2150 t_forcerec
*fr
, char *bLocalCG
)
2152 cginfo_mb_t
*cginfo_mb
;
2158 cginfo_mb
= fr
->cginfo_mb
;
2159 cginfo
= fr
->cginfo
;
2161 for (cg
= cg0
; cg
< cg1
; cg
++)
2163 cginfo
[cg
] = ddcginfo(cginfo_mb
, index_gl
[cg
]);
2167 if (bLocalCG
!= nullptr)
2169 for (cg
= cg0
; cg
< cg1
; cg
++)
2171 bLocalCG
[index_gl
[cg
]] = TRUE
;
2176 static void make_dd_indices(gmx_domdec_t
*dd
,
2177 const int *gcgs_index
, int cg_start
)
2179 int nzone
, zone
, zone1
, cg0
, cg1
, cg1_p1
, cg
, cg_gl
, a
, a_gl
;
2180 int *zone2cg
, *zone_ncg1
, *index_gl
, *gatindex
;
2183 if (dd
->nat_tot
> dd
->gatindex_nalloc
)
2185 dd
->gatindex_nalloc
= over_alloc_dd(dd
->nat_tot
);
2186 srenew(dd
->gatindex
, dd
->gatindex_nalloc
);
2189 nzone
= dd
->comm
->zones
.n
;
2190 zone2cg
= dd
->comm
->zones
.cg_range
;
2191 zone_ncg1
= dd
->comm
->zone_ncg1
;
2192 index_gl
= dd
->index_gl
;
2193 gatindex
= dd
->gatindex
;
2194 bCGs
= dd
->comm
->bCGs
;
2196 if (zone2cg
[1] != dd
->ncg_home
)
2198 gmx_incons("dd->ncg_zone is not up to date");
2201 /* Make the local to global and global to local atom index */
2202 a
= dd
->cgindex
[cg_start
];
2203 for (zone
= 0; zone
< nzone
; zone
++)
2211 cg0
= zone2cg
[zone
];
2213 cg1
= zone2cg
[zone
+1];
2214 cg1_p1
= cg0
+ zone_ncg1
[zone
];
2216 for (cg
= cg0
; cg
< cg1
; cg
++)
2221 /* Signal that this cg is from more than one pulse away */
2224 cg_gl
= index_gl
[cg
];
2227 for (a_gl
= gcgs_index
[cg_gl
]; a_gl
< gcgs_index
[cg_gl
+1]; a_gl
++)
2230 ga2la_set(dd
->ga2la
, a_gl
, a
, zone1
);
2236 gatindex
[a
] = cg_gl
;
2237 ga2la_set(dd
->ga2la
, cg_gl
, a
, zone1
);
2244 static int check_bLocalCG(gmx_domdec_t
*dd
, int ncg_sys
, const char *bLocalCG
,
2250 if (bLocalCG
== nullptr)
2254 for (i
= 0; i
< dd
->ncg_tot
; i
++)
2256 if (!bLocalCG
[dd
->index_gl
[i
]])
2259 "DD rank %d, %s: cg %d, global cg %d is not marked in bLocalCG (ncg_home %d)\n", dd
->rank
, where
, i
+1, dd
->index_gl
[i
]+1, dd
->ncg_home
);
2264 for (i
= 0; i
< ncg_sys
; i
++)
2271 if (ngl
!= dd
->ncg_tot
)
2273 fprintf(stderr
, "DD rank %d, %s: In bLocalCG %d cgs are marked as local, whereas there are %d\n", dd
->rank
, where
, ngl
, dd
->ncg_tot
);
2280 static void check_index_consistency(gmx_domdec_t
*dd
,
2281 int natoms_sys
, int ncg_sys
,
2284 int nerr
, ngl
, i
, a
, cell
;
2289 if (dd
->comm
->DD_debug
> 1)
2291 snew(have
, natoms_sys
);
2292 for (a
= 0; a
< dd
->nat_tot
; a
++)
2294 if (have
[dd
->gatindex
[a
]] > 0)
2296 fprintf(stderr
, "DD rank %d: global atom %d occurs twice: index %d and %d\n", dd
->rank
, dd
->gatindex
[a
]+1, have
[dd
->gatindex
[a
]], a
+1);
2300 have
[dd
->gatindex
[a
]] = a
+ 1;
2306 snew(have
, dd
->nat_tot
);
2309 for (i
= 0; i
< natoms_sys
; i
++)
2311 if (ga2la_get(dd
->ga2la
, i
, &a
, &cell
))
2313 if (a
>= dd
->nat_tot
)
2315 fprintf(stderr
, "DD rank %d: global atom %d marked as local atom %d, which is larger than nat_tot (%d)\n", dd
->rank
, i
+1, a
+1, dd
->nat_tot
);
2321 if (dd
->gatindex
[a
] != i
)
2323 fprintf(stderr
, "DD rank %d: global atom %d marked as local atom %d, which has global atom index %d\n", dd
->rank
, i
+1, a
+1, dd
->gatindex
[a
]+1);
2330 if (ngl
!= dd
->nat_tot
)
2333 "DD rank %d, %s: %d global atom indices, %d local atoms\n",
2334 dd
->rank
, where
, ngl
, dd
->nat_tot
);
2336 for (a
= 0; a
< dd
->nat_tot
; a
++)
2341 "DD rank %d, %s: local atom %d, global %d has no global index\n",
2342 dd
->rank
, where
, a
+1, dd
->gatindex
[a
]+1);
2347 nerr
+= check_bLocalCG(dd
, ncg_sys
, dd
->comm
->bLocalCG
, where
);
2351 gmx_fatal(FARGS
, "DD rank %d, %s: %d atom/cg index inconsistencies",
2352 dd
->rank
, where
, nerr
);
2356 static void clear_dd_indices(gmx_domdec_t
*dd
, int cg_start
, int a_start
)
2363 /* Clear the whole list without searching */
2364 ga2la_clear(dd
->ga2la
);
2368 for (i
= a_start
; i
< dd
->nat_tot
; i
++)
2370 ga2la_del(dd
->ga2la
, dd
->gatindex
[i
]);
2374 bLocalCG
= dd
->comm
->bLocalCG
;
2377 for (i
= cg_start
; i
< dd
->ncg_tot
; i
++)
2379 bLocalCG
[dd
->index_gl
[i
]] = FALSE
;
2383 dd_clear_local_vsite_indices(dd
);
2385 if (dd
->constraints
)
2387 dd_clear_local_constraint_indices(dd
);
2391 /* This function should be used for moving the domain boudaries during DLB,
2392 * for obtaining the minimum cell size. It checks the initially set limit
2393 * comm->cellsize_min, for bonded and initial non-bonded cut-offs,
2394 * and, possibly, a longer cut-off limit set for PME load balancing.
2396 static real
cellsize_min_dlb(gmx_domdec_comm_t
*comm
, int dim_ind
, int dim
)
2400 cellsize_min
= comm
->cellsize_min
[dim
];
2402 if (!comm
->bVacDLBNoLimit
)
2404 /* The cut-off might have changed, e.g. by PME load balacning,
2405 * from the value used to set comm->cellsize_min, so check it.
2407 cellsize_min
= std::max(cellsize_min
, comm
->cutoff
/comm
->cd
[dim_ind
].np_dlb
);
2409 if (comm
->bPMELoadBalDLBLimits
)
2411 /* Check for the cut-off limit set by the PME load balancing */
2412 cellsize_min
= std::max(cellsize_min
, comm
->PMELoadBal_max_cutoff
/comm
->cd
[dim_ind
].np_dlb
);
2416 return cellsize_min
;
2419 static real
grid_jump_limit(gmx_domdec_comm_t
*comm
, real cutoff
,
2422 real grid_jump_limit
;
2424 /* The distance between the boundaries of cells at distance
2425 * x+-1,y+-1 or y+-1,z+-1 is limited by the cut-off restrictions
2426 * and by the fact that cells should not be shifted by more than
2427 * half their size, such that cg's only shift by one cell
2428 * at redecomposition.
2430 grid_jump_limit
= comm
->cellsize_limit
;
2431 if (!comm
->bVacDLBNoLimit
)
2433 if (comm
->bPMELoadBalDLBLimits
)
2435 cutoff
= std::max(cutoff
, comm
->PMELoadBal_max_cutoff
);
2437 grid_jump_limit
= std::max(grid_jump_limit
,
2438 cutoff
/comm
->cd
[dim_ind
].np
);
2441 return grid_jump_limit
;
2444 static gmx_bool
check_grid_jump(gmx_int64_t step
,
2450 gmx_domdec_comm_t
*comm
;
2459 for (d
= 1; d
< dd
->ndim
; d
++)
2462 limit
= grid_jump_limit(comm
, cutoff
, d
);
2463 bfac
= ddbox
->box_size
[dim
];
2464 if (ddbox
->tric_dir
[dim
])
2466 bfac
*= ddbox
->skew_fac
[dim
];
2468 if ((comm
->cell_f1
[d
] - comm
->cell_f_max0
[d
])*bfac
< limit
||
2469 (comm
->cell_f0
[d
] - comm
->cell_f_min1
[d
])*bfac
> -limit
)
2477 /* This error should never be triggered under normal
2478 * circumstances, but you never know ...
2480 gmx_fatal(FARGS
, "step %s: The domain decomposition grid has shifted too much in the %c-direction around cell %d %d %d. This should not have happened. Running with fewer ranks might avoid this issue.",
2481 gmx_step_str(step
, buf
),
2482 dim2char(dim
), dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
2490 static int dd_load_count(gmx_domdec_comm_t
*comm
)
2492 return (comm
->eFlop
? comm
->flop_n
: comm
->cycl_n
[ddCyclF
]);
2495 static float dd_force_load(gmx_domdec_comm_t
*comm
)
2502 if (comm
->eFlop
> 1)
2504 load
*= 1.0 + (comm
->eFlop
- 1)*(0.1*rand()/RAND_MAX
- 0.05);
2509 load
= comm
->cycl
[ddCyclF
];
2510 if (comm
->cycl_n
[ddCyclF
] > 1)
2512 /* Subtract the maximum of the last n cycle counts
2513 * to get rid of possible high counts due to other sources,
2514 * for instance system activity, that would otherwise
2515 * affect the dynamic load balancing.
2517 load
-= comm
->cycl_max
[ddCyclF
];
2521 if (comm
->cycl_n
[ddCyclWaitGPU
] && comm
->nrank_gpu_shared
> 1)
2523 float gpu_wait
, gpu_wait_sum
;
2525 gpu_wait
= comm
->cycl
[ddCyclWaitGPU
];
2526 if (comm
->cycl_n
[ddCyclF
] > 1)
2528 /* We should remove the WaitGPU time of the same MD step
2529 * as the one with the maximum F time, since the F time
2530 * and the wait time are not independent.
2531 * Furthermore, the step for the max F time should be chosen
2532 * the same on all ranks that share the same GPU.
2533 * But to keep the code simple, we remove the average instead.
2534 * The main reason for artificially long times at some steps
2535 * is spurious CPU activity or MPI time, so we don't expect
2536 * that changes in the GPU wait time matter a lot here.
2538 gpu_wait
*= (comm
->cycl_n
[ddCyclF
] - 1)/(float)comm
->cycl_n
[ddCyclF
];
2540 /* Sum the wait times over the ranks that share the same GPU */
2541 MPI_Allreduce(&gpu_wait
, &gpu_wait_sum
, 1, MPI_FLOAT
, MPI_SUM
,
2542 comm
->mpi_comm_gpu_shared
);
2543 /* Replace the wait time by the average over the ranks */
2544 load
+= -gpu_wait
+ gpu_wait_sum
/comm
->nrank_gpu_shared
;
2552 static void set_slb_pme_dim_f(gmx_domdec_t
*dd
, int dim
, real
**dim_f
)
2554 gmx_domdec_comm_t
*comm
;
2559 snew(*dim_f
, dd
->nc
[dim
]+1);
2561 for (i
= 1; i
< dd
->nc
[dim
]; i
++)
2563 if (comm
->slb_frac
[dim
])
2565 (*dim_f
)[i
] = (*dim_f
)[i
-1] + comm
->slb_frac
[dim
][i
-1];
2569 (*dim_f
)[i
] = (real
)i
/(real
)dd
->nc
[dim
];
2572 (*dim_f
)[dd
->nc
[dim
]] = 1;
2575 static void init_ddpme(gmx_domdec_t
*dd
, gmx_ddpme_t
*ddpme
, int dimind
)
2577 int pmeindex
, slab
, nso
, i
;
2580 if (dimind
== 0 && dd
->dim
[0] == YY
&& dd
->comm
->npmenodes_x
== 1)
2586 ddpme
->dim
= dimind
;
2588 ddpme
->dim_match
= (ddpme
->dim
== dd
->dim
[dimind
]);
2590 ddpme
->nslab
= (ddpme
->dim
== 0 ?
2591 dd
->comm
->npmenodes_x
:
2592 dd
->comm
->npmenodes_y
);
2594 if (ddpme
->nslab
<= 1)
2599 nso
= dd
->comm
->npmenodes
/ddpme
->nslab
;
2600 /* Determine for each PME slab the PP location range for dimension dim */
2601 snew(ddpme
->pp_min
, ddpme
->nslab
);
2602 snew(ddpme
->pp_max
, ddpme
->nslab
);
2603 for (slab
= 0; slab
< ddpme
->nslab
; slab
++)
2605 ddpme
->pp_min
[slab
] = dd
->nc
[dd
->dim
[dimind
]] - 1;
2606 ddpme
->pp_max
[slab
] = 0;
2608 for (i
= 0; i
< dd
->nnodes
; i
++)
2610 ddindex2xyz(dd
->nc
, i
, xyz
);
2611 /* For y only use our y/z slab.
2612 * This assumes that the PME x grid size matches the DD grid size.
2614 if (dimind
== 0 || xyz
[XX
] == dd
->ci
[XX
])
2616 pmeindex
= ddindex2pmeindex(dd
, i
);
2619 slab
= pmeindex
/nso
;
2623 slab
= pmeindex
% ddpme
->nslab
;
2625 ddpme
->pp_min
[slab
] = std::min(ddpme
->pp_min
[slab
], xyz
[dimind
]);
2626 ddpme
->pp_max
[slab
] = std::max(ddpme
->pp_max
[slab
], xyz
[dimind
]);
2630 set_slb_pme_dim_f(dd
, ddpme
->dim
, &ddpme
->slb_dim_f
);
2633 int dd_pme_maxshift_x(const gmx_domdec_t
*dd
)
2635 if (dd
->comm
->ddpme
[0].dim
== XX
)
2637 return dd
->comm
->ddpme
[0].maxshift
;
2645 int dd_pme_maxshift_y(const gmx_domdec_t
*dd
)
2647 if (dd
->comm
->ddpme
[0].dim
== YY
)
2649 return dd
->comm
->ddpme
[0].maxshift
;
2651 else if (dd
->comm
->npmedecompdim
>= 2 && dd
->comm
->ddpme
[1].dim
== YY
)
2653 return dd
->comm
->ddpme
[1].maxshift
;
2661 static void set_pme_maxshift(gmx_domdec_t
*dd
, gmx_ddpme_t
*ddpme
,
2662 gmx_bool bUniform
, const gmx_ddbox_t
*ddbox
,
2665 gmx_domdec_comm_t
*comm
;
2668 real range
, pme_boundary
;
2672 nc
= dd
->nc
[ddpme
->dim
];
2675 if (!ddpme
->dim_match
)
2677 /* PP decomposition is not along dim: the worst situation */
2680 else if (ns
<= 3 || (bUniform
&& ns
== nc
))
2682 /* The optimal situation */
2687 /* We need to check for all pme nodes which nodes they
2688 * could possibly need to communicate with.
2690 xmin
= ddpme
->pp_min
;
2691 xmax
= ddpme
->pp_max
;
2692 /* Allow for atoms to be maximally 2/3 times the cut-off
2693 * out of their DD cell. This is a reasonable balance between
2694 * between performance and support for most charge-group/cut-off
2697 range
= 2.0/3.0*comm
->cutoff
/ddbox
->box_size
[ddpme
->dim
];
2698 /* Avoid extra communication when we are exactly at a boundary */
2702 for (s
= 0; s
< ns
; s
++)
2704 /* PME slab s spreads atoms between box frac. s/ns and (s+1)/ns */
2705 pme_boundary
= (real
)s
/ns
;
2708 cell_f
[xmax
[s
-(sh
+1) ]+1] + range
> pme_boundary
) ||
2710 cell_f
[xmax
[s
-(sh
+1)+ns
]+1] - 1 + range
> pme_boundary
)))
2714 pme_boundary
= (real
)(s
+1)/ns
;
2717 cell_f
[xmin
[s
+(sh
+1) ] ] - range
< pme_boundary
) ||
2719 cell_f
[xmin
[s
+(sh
+1)-ns
] ] + 1 - range
< pme_boundary
)))
2726 ddpme
->maxshift
= sh
;
2730 fprintf(debug
, "PME slab communication range for dim %d is %d\n",
2731 ddpme
->dim
, ddpme
->maxshift
);
2735 static void check_box_size(gmx_domdec_t
*dd
, gmx_ddbox_t
*ddbox
)
2739 for (d
= 0; d
< dd
->ndim
; d
++)
2742 if (dim
< ddbox
->nboundeddim
&&
2743 ddbox
->box_size
[dim
]*ddbox
->skew_fac
[dim
] <
2744 dd
->nc
[dim
]*dd
->comm
->cellsize_limit
*DD_CELL_MARGIN
)
2746 gmx_fatal(FARGS
, "The %c-size of the box (%f) times the triclinic skew factor (%f) is smaller than the number of DD cells (%d) times the smallest allowed cell size (%f)\n",
2747 dim2char(dim
), ddbox
->box_size
[dim
], ddbox
->skew_fac
[dim
],
2748 dd
->nc
[dim
], dd
->comm
->cellsize_limit
);
2754 setcellsizeslbLOCAL
, setcellsizeslbMASTER
, setcellsizeslbPULSE_ONLY
2757 /* Set the domain boundaries. Use for static (or no) load balancing,
2758 * and also for the starting state for dynamic load balancing.
2759 * setmode determine if and where the boundaries are stored, use enum above.
2760 * Returns the number communication pulses in npulse.
2762 static void set_dd_cell_sizes_slb(gmx_domdec_t
*dd
, const gmx_ddbox_t
*ddbox
,
2763 int setmode
, ivec npulse
)
2765 gmx_domdec_comm_t
*comm
;
2768 real
*cell_x
, cell_dx
, cellsize
;
2772 for (d
= 0; d
< DIM
; d
++)
2774 cellsize_min
[d
] = ddbox
->box_size
[d
]*ddbox
->skew_fac
[d
];
2776 if (dd
->nc
[d
] == 1 || comm
->slb_frac
[d
] == nullptr)
2779 cell_dx
= ddbox
->box_size
[d
]/dd
->nc
[d
];
2782 case setcellsizeslbMASTER
:
2783 for (j
= 0; j
< dd
->nc
[d
]+1; j
++)
2785 dd
->ma
->cell_x
[d
][j
] = ddbox
->box0
[d
] + j
*cell_dx
;
2788 case setcellsizeslbLOCAL
:
2789 comm
->cell_x0
[d
] = ddbox
->box0
[d
] + (dd
->ci
[d
] )*cell_dx
;
2790 comm
->cell_x1
[d
] = ddbox
->box0
[d
] + (dd
->ci
[d
]+1)*cell_dx
;
2795 cellsize
= cell_dx
*ddbox
->skew_fac
[d
];
2796 while (cellsize
*npulse
[d
] < comm
->cutoff
)
2800 cellsize_min
[d
] = cellsize
;
2804 /* Statically load balanced grid */
2805 /* Also when we are not doing a master distribution we determine
2806 * all cell borders in a loop to obtain identical values
2807 * to the master distribution case and to determine npulse.
2809 if (setmode
== setcellsizeslbMASTER
)
2811 cell_x
= dd
->ma
->cell_x
[d
];
2815 snew(cell_x
, dd
->nc
[d
]+1);
2817 cell_x
[0] = ddbox
->box0
[d
];
2818 for (j
= 0; j
< dd
->nc
[d
]; j
++)
2820 cell_dx
= ddbox
->box_size
[d
]*comm
->slb_frac
[d
][j
];
2821 cell_x
[j
+1] = cell_x
[j
] + cell_dx
;
2822 cellsize
= cell_dx
*ddbox
->skew_fac
[d
];
2823 while (cellsize
*npulse
[d
] < comm
->cutoff
&&
2824 npulse
[d
] < dd
->nc
[d
]-1)
2828 cellsize_min
[d
] = std::min(cellsize_min
[d
], cellsize
);
2830 if (setmode
== setcellsizeslbLOCAL
)
2832 comm
->cell_x0
[d
] = cell_x
[dd
->ci
[d
]];
2833 comm
->cell_x1
[d
] = cell_x
[dd
->ci
[d
]+1];
2835 if (setmode
!= setcellsizeslbMASTER
)
2840 /* The following limitation is to avoid that a cell would receive
2841 * some of its own home charge groups back over the periodic boundary.
2842 * Double charge groups cause trouble with the global indices.
2844 if (d
< ddbox
->npbcdim
&&
2845 dd
->nc
[d
] > 1 && npulse
[d
] >= dd
->nc
[d
])
2847 char error_string
[STRLEN
];
2849 sprintf(error_string
,
2850 "The box size in direction %c (%f) times the triclinic skew factor (%f) is too small for a cut-off of %f with %d domain decomposition cells, use 1 or more than %d %s or increase the box size in this direction",
2851 dim2char(d
), ddbox
->box_size
[d
], ddbox
->skew_fac
[d
],
2853 dd
->nc
[d
], dd
->nc
[d
],
2854 dd
->nnodes
> dd
->nc
[d
] ? "cells" : "ranks");
2856 if (setmode
== setcellsizeslbLOCAL
)
2858 gmx_fatal_collective(FARGS
, dd
->mpi_comm_all
, DDMASTER(dd
),
2863 gmx_fatal(FARGS
, error_string
);
2870 copy_rvec(cellsize_min
, comm
->cellsize_min
);
2873 for (d
= 0; d
< comm
->npmedecompdim
; d
++)
2875 set_pme_maxshift(dd
, &comm
->ddpme
[d
],
2876 comm
->slb_frac
[dd
->dim
[d
]] == nullptr, ddbox
,
2877 comm
->ddpme
[d
].slb_dim_f
);
2882 static void dd_cell_sizes_dlb_root_enforce_limits(gmx_domdec_t
*dd
,
2883 int d
, int dim
, domdec_root_t
*root
,
2884 const gmx_ddbox_t
*ddbox
,
2885 gmx_bool bUniform
, gmx_int64_t step
, real cellsize_limit_f
, int range
[])
2887 gmx_domdec_comm_t
*comm
;
2888 int ncd
, i
, j
, nmin
, nmin_old
;
2889 gmx_bool bLimLo
, bLimHi
;
2891 real fac
, halfway
, cellsize_limit_f_i
, region_size
;
2892 gmx_bool bPBC
, bLastHi
= FALSE
;
2893 int nrange
[] = {range
[0], range
[1]};
2895 region_size
= root
->cell_f
[range
[1]]-root
->cell_f
[range
[0]];
2901 bPBC
= (dim
< ddbox
->npbcdim
);
2903 cell_size
= root
->buf_ncd
;
2907 fprintf(debug
, "enforce_limits: %d %d\n", range
[0], range
[1]);
2910 /* First we need to check if the scaling does not make cells
2911 * smaller than the smallest allowed size.
2912 * We need to do this iteratively, since if a cell is too small,
2913 * it needs to be enlarged, which makes all the other cells smaller,
2914 * which could in turn make another cell smaller than allowed.
2916 for (i
= range
[0]; i
< range
[1]; i
++)
2918 root
->bCellMin
[i
] = FALSE
;
2924 /* We need the total for normalization */
2926 for (i
= range
[0]; i
< range
[1]; i
++)
2928 if (root
->bCellMin
[i
] == FALSE
)
2930 fac
+= cell_size
[i
];
2933 fac
= ( region_size
- nmin
*cellsize_limit_f
)/fac
; /* substracting cells already set to cellsize_limit_f */
2934 /* Determine the cell boundaries */
2935 for (i
= range
[0]; i
< range
[1]; i
++)
2937 if (root
->bCellMin
[i
] == FALSE
)
2939 cell_size
[i
] *= fac
;
2940 if (!bPBC
&& (i
== 0 || i
== dd
->nc
[dim
] -1))
2942 cellsize_limit_f_i
= 0;
2946 cellsize_limit_f_i
= cellsize_limit_f
;
2948 if (cell_size
[i
] < cellsize_limit_f_i
)
2950 root
->bCellMin
[i
] = TRUE
;
2951 cell_size
[i
] = cellsize_limit_f_i
;
2955 root
->cell_f
[i
+1] = root
->cell_f
[i
] + cell_size
[i
];
2958 while (nmin
> nmin_old
);
2961 cell_size
[i
] = root
->cell_f
[i
+1] - root
->cell_f
[i
];
2962 /* For this check we should not use DD_CELL_MARGIN,
2963 * but a slightly smaller factor,
2964 * since rounding could get use below the limit.
2966 if (bPBC
&& cell_size
[i
] < cellsize_limit_f
*DD_CELL_MARGIN2
/DD_CELL_MARGIN
)
2969 gmx_fatal(FARGS
, "step %s: the dynamic load balancing could not balance dimension %c: box size %f, triclinic skew factor %f, #cells %d, minimum cell size %f\n",
2970 gmx_step_str(step
, buf
),
2971 dim2char(dim
), ddbox
->box_size
[dim
], ddbox
->skew_fac
[dim
],
2972 ncd
, comm
->cellsize_min
[dim
]);
2975 root
->bLimited
= (nmin
> 0) || (range
[0] > 0) || (range
[1] < ncd
);
2979 /* Check if the boundary did not displace more than halfway
2980 * each of the cells it bounds, as this could cause problems,
2981 * especially when the differences between cell sizes are large.
2982 * If changes are applied, they will not make cells smaller
2983 * than the cut-off, as we check all the boundaries which
2984 * might be affected by a change and if the old state was ok,
2985 * the cells will at most be shrunk back to their old size.
2987 for (i
= range
[0]+1; i
< range
[1]; i
++)
2989 halfway
= 0.5*(root
->old_cell_f
[i
] + root
->old_cell_f
[i
-1]);
2990 if (root
->cell_f
[i
] < halfway
)
2992 root
->cell_f
[i
] = halfway
;
2993 /* Check if the change also causes shifts of the next boundaries */
2994 for (j
= i
+1; j
< range
[1]; j
++)
2996 if (root
->cell_f
[j
] < root
->cell_f
[j
-1] + cellsize_limit_f
)
2998 root
->cell_f
[j
] = root
->cell_f
[j
-1] + cellsize_limit_f
;
3002 halfway
= 0.5*(root
->old_cell_f
[i
] + root
->old_cell_f
[i
+1]);
3003 if (root
->cell_f
[i
] > halfway
)
3005 root
->cell_f
[i
] = halfway
;
3006 /* Check if the change also causes shifts of the next boundaries */
3007 for (j
= i
-1; j
>= range
[0]+1; j
--)
3009 if (root
->cell_f
[j
] > root
->cell_f
[j
+1] - cellsize_limit_f
)
3011 root
->cell_f
[j
] = root
->cell_f
[j
+1] - cellsize_limit_f
;
3018 /* nrange is defined as [lower, upper) range for new call to enforce_limits */
3019 /* find highest violation of LimLo (a) and the following violation of LimHi (thus the lowest following) (b)
3020 * then call enforce_limits for (oldb,a), (a,b). In the next step: (b,nexta). oldb and nexta can be the boundaries.
3021 * for a and b nrange is used */
3024 /* Take care of the staggering of the cell boundaries */
3027 for (i
= range
[0]; i
< range
[1]; i
++)
3029 root
->cell_f_max0
[i
] = root
->cell_f
[i
];
3030 root
->cell_f_min1
[i
] = root
->cell_f
[i
+1];
3035 for (i
= range
[0]+1; i
< range
[1]; i
++)
3037 bLimLo
= (root
->cell_f
[i
] < root
->bound_min
[i
]);
3038 bLimHi
= (root
->cell_f
[i
] > root
->bound_max
[i
]);
3039 if (bLimLo
&& bLimHi
)
3041 /* Both limits violated, try the best we can */
3042 /* For this case we split the original range (range) in two parts and care about the other limitiations in the next iteration. */
3043 root
->cell_f
[i
] = 0.5*(root
->bound_min
[i
] + root
->bound_max
[i
]);
3044 nrange
[0] = range
[0];
3046 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3049 nrange
[1] = range
[1];
3050 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3056 /* root->cell_f[i] = root->bound_min[i]; */
3057 nrange
[1] = i
; /* only store violation location. There could be a LimLo violation following with an higher index */
3060 else if (bLimHi
&& !bLastHi
)
3063 if (nrange
[1] < range
[1]) /* found a LimLo before */
3065 root
->cell_f
[nrange
[1]] = root
->bound_min
[nrange
[1]];
3066 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3067 nrange
[0] = nrange
[1];
3069 root
->cell_f
[i
] = root
->bound_max
[i
];
3071 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3073 nrange
[1] = range
[1];
3076 if (nrange
[1] < range
[1]) /* found last a LimLo */
3078 root
->cell_f
[nrange
[1]] = root
->bound_min
[nrange
[1]];
3079 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3080 nrange
[0] = nrange
[1];
3081 nrange
[1] = range
[1];
3082 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3084 else if (nrange
[0] > range
[0]) /* found at least one LimHi */
3086 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3093 static void set_dd_cell_sizes_dlb_root(gmx_domdec_t
*dd
,
3094 int d
, int dim
, domdec_root_t
*root
,
3095 const gmx_ddbox_t
*ddbox
,
3096 gmx_bool bDynamicBox
,
3097 gmx_bool bUniform
, gmx_int64_t step
)
3099 gmx_domdec_comm_t
*comm
;
3100 int ncd
, d1
, i
, pos
;
3102 real load_aver
, load_i
, imbalance
, change
, change_max
, sc
;
3103 real cellsize_limit_f
, dist_min_f
, dist_min_f_hard
, space
;
3107 int range
[] = { 0, 0 };
3111 /* Convert the maximum change from the input percentage to a fraction */
3112 change_limit
= comm
->dlb_scale_lim
*0.01;
3116 bPBC
= (dim
< ddbox
->npbcdim
);
3118 cell_size
= root
->buf_ncd
;
3120 /* Store the original boundaries */
3121 for (i
= 0; i
< ncd
+1; i
++)
3123 root
->old_cell_f
[i
] = root
->cell_f
[i
];
3127 for (i
= 0; i
< ncd
; i
++)
3129 cell_size
[i
] = 1.0/ncd
;
3132 else if (dd_load_count(comm
) > 0)
3134 load_aver
= comm
->load
[d
].sum_m
/ncd
;
3136 for (i
= 0; i
< ncd
; i
++)
3138 /* Determine the relative imbalance of cell i */
3139 load_i
= comm
->load
[d
].load
[i
*comm
->load
[d
].nload
+2];
3140 imbalance
= (load_i
- load_aver
)/(load_aver
> 0 ? load_aver
: 1);
3141 /* Determine the change of the cell size using underrelaxation */
3142 change
= -relax
*imbalance
;
3143 change_max
= std::max(change_max
, std::max(change
, -change
));
3145 /* Limit the amount of scaling.
3146 * We need to use the same rescaling for all cells in one row,
3147 * otherwise the load balancing might not converge.
3150 if (change_max
> change_limit
)
3152 sc
*= change_limit
/change_max
;
3154 for (i
= 0; i
< ncd
; i
++)
3156 /* Determine the relative imbalance of cell i */
3157 load_i
= comm
->load
[d
].load
[i
*comm
->load
[d
].nload
+2];
3158 imbalance
= (load_i
- load_aver
)/(load_aver
> 0 ? load_aver
: 1);
3159 /* Determine the change of the cell size using underrelaxation */
3160 change
= -sc
*imbalance
;
3161 cell_size
[i
] = (root
->cell_f
[i
+1]-root
->cell_f
[i
])*(1 + change
);
3165 cellsize_limit_f
= cellsize_min_dlb(comm
, d
, dim
)/ddbox
->box_size
[dim
];
3166 cellsize_limit_f
*= DD_CELL_MARGIN
;
3167 dist_min_f_hard
= grid_jump_limit(comm
, comm
->cutoff
, d
)/ddbox
->box_size
[dim
];
3168 dist_min_f
= dist_min_f_hard
* DD_CELL_MARGIN
;
3169 if (ddbox
->tric_dir
[dim
])
3171 cellsize_limit_f
/= ddbox
->skew_fac
[dim
];
3172 dist_min_f
/= ddbox
->skew_fac
[dim
];
3174 if (bDynamicBox
&& d
> 0)
3176 dist_min_f
*= DD_PRES_SCALE_MARGIN
;
3178 if (d
> 0 && !bUniform
)
3180 /* Make sure that the grid is not shifted too much */
3181 for (i
= 1; i
< ncd
; i
++)
3183 if (root
->cell_f_min1
[i
] - root
->cell_f_max0
[i
-1] < 2 * dist_min_f_hard
)
3185 gmx_incons("Inconsistent DD boundary staggering limits!");
3187 root
->bound_min
[i
] = root
->cell_f_max0
[i
-1] + dist_min_f
;
3188 space
= root
->cell_f
[i
] - (root
->cell_f_max0
[i
-1] + dist_min_f
);
3191 root
->bound_min
[i
] += 0.5*space
;
3193 root
->bound_max
[i
] = root
->cell_f_min1
[i
] - dist_min_f
;
3194 space
= root
->cell_f
[i
] - (root
->cell_f_min1
[i
] - dist_min_f
);
3197 root
->bound_max
[i
] += 0.5*space
;
3202 "dim %d boundary %d %.3f < %.3f < %.3f < %.3f < %.3f\n",
3204 root
->cell_f_max0
[i
-1] + dist_min_f
,
3205 root
->bound_min
[i
], root
->cell_f
[i
], root
->bound_max
[i
],
3206 root
->cell_f_min1
[i
] - dist_min_f
);
3211 root
->cell_f
[0] = 0;
3212 root
->cell_f
[ncd
] = 1;
3213 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, range
);
3216 /* After the checks above, the cells should obey the cut-off
3217 * restrictions, but it does not hurt to check.
3219 for (i
= 0; i
< ncd
; i
++)
3223 fprintf(debug
, "Relative bounds dim %d cell %d: %f %f\n",
3224 dim
, i
, root
->cell_f
[i
], root
->cell_f
[i
+1]);
3227 if ((bPBC
|| (i
!= 0 && i
!= dd
->nc
[dim
]-1)) &&
3228 root
->cell_f
[i
+1] - root
->cell_f
[i
] <
3229 cellsize_limit_f
/DD_CELL_MARGIN
)
3233 "\nWARNING step %s: direction %c, cell %d too small: %f\n",
3234 gmx_step_str(step
, buf
), dim2char(dim
), i
,
3235 (root
->cell_f
[i
+1] - root
->cell_f
[i
])
3236 *ddbox
->box_size
[dim
]*ddbox
->skew_fac
[dim
]);
3241 /* Store the cell boundaries of the lower dimensions at the end */
3242 for (d1
= 0; d1
< d
; d1
++)
3244 root
->cell_f
[pos
++] = comm
->cell_f0
[d1
];
3245 root
->cell_f
[pos
++] = comm
->cell_f1
[d1
];
3248 if (d
< comm
->npmedecompdim
)
3250 /* The master determines the maximum shift for
3251 * the coordinate communication between separate PME nodes.
3253 set_pme_maxshift(dd
, &comm
->ddpme
[d
], bUniform
, ddbox
, root
->cell_f
);
3255 root
->cell_f
[pos
++] = comm
->ddpme
[0].maxshift
;
3258 root
->cell_f
[pos
++] = comm
->ddpme
[1].maxshift
;
3262 static void relative_to_absolute_cell_bounds(gmx_domdec_t
*dd
,
3263 const gmx_ddbox_t
*ddbox
,
3266 gmx_domdec_comm_t
*comm
;
3271 /* Set the cell dimensions */
3272 dim
= dd
->dim
[dimind
];
3273 comm
->cell_x0
[dim
] = comm
->cell_f0
[dimind
]*ddbox
->box_size
[dim
];
3274 comm
->cell_x1
[dim
] = comm
->cell_f1
[dimind
]*ddbox
->box_size
[dim
];
3275 if (dim
>= ddbox
->nboundeddim
)
3277 comm
->cell_x0
[dim
] += ddbox
->box0
[dim
];
3278 comm
->cell_x1
[dim
] += ddbox
->box0
[dim
];
3282 static void distribute_dd_cell_sizes_dlb(gmx_domdec_t
*dd
,
3283 int d
, int dim
, real
*cell_f_row
,
3284 const gmx_ddbox_t
*ddbox
)
3286 gmx_domdec_comm_t
*comm
;
3292 /* Each node would only need to know two fractions,
3293 * but it is probably cheaper to broadcast the whole array.
3295 MPI_Bcast(cell_f_row
, DD_CELL_F_SIZE(dd
, d
)*sizeof(real
), MPI_BYTE
,
3296 0, comm
->mpi_comm_load
[d
]);
3298 /* Copy the fractions for this dimension from the buffer */
3299 comm
->cell_f0
[d
] = cell_f_row
[dd
->ci
[dim
] ];
3300 comm
->cell_f1
[d
] = cell_f_row
[dd
->ci
[dim
]+1];
3301 /* The whole array was communicated, so set the buffer position */
3302 pos
= dd
->nc
[dim
] + 1;
3303 for (d1
= 0; d1
<= d
; d1
++)
3307 /* Copy the cell fractions of the lower dimensions */
3308 comm
->cell_f0
[d1
] = cell_f_row
[pos
++];
3309 comm
->cell_f1
[d1
] = cell_f_row
[pos
++];
3311 relative_to_absolute_cell_bounds(dd
, ddbox
, d1
);
3313 /* Convert the communicated shift from float to int */
3314 comm
->ddpme
[0].maxshift
= (int)(cell_f_row
[pos
++] + 0.5);
3317 comm
->ddpme
[1].maxshift
= (int)(cell_f_row
[pos
++] + 0.5);
3321 static void set_dd_cell_sizes_dlb_change(gmx_domdec_t
*dd
,
3322 const gmx_ddbox_t
*ddbox
,
3323 gmx_bool bDynamicBox
,
3324 gmx_bool bUniform
, gmx_int64_t step
)
3326 gmx_domdec_comm_t
*comm
;
3328 gmx_bool bRowMember
, bRowRoot
;
3333 for (d
= 0; d
< dd
->ndim
; d
++)
3338 for (d1
= d
; d1
< dd
->ndim
; d1
++)
3340 if (dd
->ci
[dd
->dim
[d1
]] > 0)
3353 set_dd_cell_sizes_dlb_root(dd
, d
, dim
, comm
->root
[d
],
3354 ddbox
, bDynamicBox
, bUniform
, step
);
3355 cell_f_row
= comm
->root
[d
]->cell_f
;
3359 cell_f_row
= comm
->cell_f_row
;
3361 distribute_dd_cell_sizes_dlb(dd
, d
, dim
, cell_f_row
, ddbox
);
3366 static void set_dd_cell_sizes_dlb_nochange(gmx_domdec_t
*dd
,
3367 const gmx_ddbox_t
*ddbox
)
3371 /* This function assumes the box is static and should therefore
3372 * not be called when the box has changed since the last
3373 * call to dd_partition_system.
3375 for (d
= 0; d
< dd
->ndim
; d
++)
3377 relative_to_absolute_cell_bounds(dd
, ddbox
, d
);
3383 static void set_dd_cell_sizes_dlb(gmx_domdec_t
*dd
,
3384 const gmx_ddbox_t
*ddbox
, gmx_bool bDynamicBox
,
3385 gmx_bool bUniform
, gmx_bool bDoDLB
, gmx_int64_t step
,
3386 gmx_wallcycle_t wcycle
)
3388 gmx_domdec_comm_t
*comm
;
3395 wallcycle_start(wcycle
, ewcDDCOMMBOUND
);
3396 set_dd_cell_sizes_dlb_change(dd
, ddbox
, bDynamicBox
, bUniform
, step
);
3397 wallcycle_stop(wcycle
, ewcDDCOMMBOUND
);
3399 else if (bDynamicBox
)
3401 set_dd_cell_sizes_dlb_nochange(dd
, ddbox
);
3404 /* Set the dimensions for which no DD is used */
3405 for (dim
= 0; dim
< DIM
; dim
++)
3407 if (dd
->nc
[dim
] == 1)
3409 comm
->cell_x0
[dim
] = 0;
3410 comm
->cell_x1
[dim
] = ddbox
->box_size
[dim
];
3411 if (dim
>= ddbox
->nboundeddim
)
3413 comm
->cell_x0
[dim
] += ddbox
->box0
[dim
];
3414 comm
->cell_x1
[dim
] += ddbox
->box0
[dim
];
3420 static void realloc_comm_ind(gmx_domdec_t
*dd
, ivec npulse
)
3423 gmx_domdec_comm_dim_t
*cd
;
3425 for (d
= 0; d
< dd
->ndim
; d
++)
3427 cd
= &dd
->comm
->cd
[d
];
3428 np
= npulse
[dd
->dim
[d
]];
3429 if (np
> cd
->np_nalloc
)
3433 fprintf(debug
, "(Re)allocing cd for %c to %d pulses\n",
3434 dim2char(dd
->dim
[d
]), np
);
3436 if (DDMASTER(dd
) && cd
->np_nalloc
> 0)
3438 fprintf(stderr
, "\nIncreasing the number of cell to communicate in dimension %c to %d for the first time\n", dim2char(dd
->dim
[d
]), np
);
3440 srenew(cd
->ind
, np
);
3441 for (i
= cd
->np_nalloc
; i
< np
; i
++)
3443 cd
->ind
[i
].index
= nullptr;
3444 cd
->ind
[i
].nalloc
= 0;
3453 static void set_dd_cell_sizes(gmx_domdec_t
*dd
,
3454 gmx_ddbox_t
*ddbox
, gmx_bool bDynamicBox
,
3455 gmx_bool bUniform
, gmx_bool bDoDLB
, gmx_int64_t step
,
3456 gmx_wallcycle_t wcycle
)
3458 gmx_domdec_comm_t
*comm
;
3464 /* Copy the old cell boundaries for the cg displacement check */
3465 copy_rvec(comm
->cell_x0
, comm
->old_cell_x0
);
3466 copy_rvec(comm
->cell_x1
, comm
->old_cell_x1
);
3472 check_box_size(dd
, ddbox
);
3474 set_dd_cell_sizes_dlb(dd
, ddbox
, bDynamicBox
, bUniform
, bDoDLB
, step
, wcycle
);
3478 set_dd_cell_sizes_slb(dd
, ddbox
, setcellsizeslbLOCAL
, npulse
);
3479 realloc_comm_ind(dd
, npulse
);
3484 for (d
= 0; d
< DIM
; d
++)
3486 fprintf(debug
, "cell_x[%d] %f - %f skew_fac %f\n",
3487 d
, comm
->cell_x0
[d
], comm
->cell_x1
[d
], ddbox
->skew_fac
[d
]);
3492 static void comm_dd_ns_cell_sizes(gmx_domdec_t
*dd
,
3494 rvec cell_ns_x0
, rvec cell_ns_x1
,
3497 gmx_domdec_comm_t
*comm
;
3502 for (dim_ind
= 0; dim_ind
< dd
->ndim
; dim_ind
++)
3504 dim
= dd
->dim
[dim_ind
];
3506 /* Without PBC we don't have restrictions on the outer cells */
3507 if (!(dim
>= ddbox
->npbcdim
&&
3508 (dd
->ci
[dim
] == 0 || dd
->ci
[dim
] == dd
->nc
[dim
] - 1)) &&
3510 (comm
->cell_x1
[dim
] - comm
->cell_x0
[dim
])*ddbox
->skew_fac
[dim
] <
3511 comm
->cellsize_min
[dim
])
3514 gmx_fatal(FARGS
, "step %s: The %c-size (%f) times the triclinic skew factor (%f) is smaller than the smallest allowed cell size (%f) for domain decomposition grid cell %d %d %d",
3515 gmx_step_str(step
, buf
), dim2char(dim
),
3516 comm
->cell_x1
[dim
] - comm
->cell_x0
[dim
],
3517 ddbox
->skew_fac
[dim
],
3518 dd
->comm
->cellsize_min
[dim
],
3519 dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
3523 if ((dlbIsOn(dd
->comm
) && dd
->ndim
> 1) || ddbox
->nboundeddim
< DIM
)
3525 /* Communicate the boundaries and update cell_ns_x0/1 */
3526 dd_move_cellx(dd
, ddbox
, cell_ns_x0
, cell_ns_x1
);
3527 if (dlbIsOn(dd
->comm
) && dd
->ndim
> 1)
3529 check_grid_jump(step
, dd
, dd
->comm
->cutoff
, ddbox
, TRUE
);
3534 static void make_tric_corr_matrix(int npbcdim
, matrix box
, matrix tcm
)
3538 tcm
[YY
][XX
] = -box
[YY
][XX
]/box
[YY
][YY
];
3546 tcm
[ZZ
][XX
] = -(box
[ZZ
][YY
]*tcm
[YY
][XX
] + box
[ZZ
][XX
])/box
[ZZ
][ZZ
];
3547 tcm
[ZZ
][YY
] = -box
[ZZ
][YY
]/box
[ZZ
][ZZ
];
3556 static void check_screw_box(matrix box
)
3558 /* Mathematical limitation */
3559 if (box
[YY
][XX
] != 0 || box
[ZZ
][XX
] != 0)
3561 gmx_fatal(FARGS
, "With screw pbc the unit cell can not have non-zero off-diagonal x-components");
3564 /* Limitation due to the asymmetry of the eighth shell method */
3565 if (box
[ZZ
][YY
] != 0)
3567 gmx_fatal(FARGS
, "pbc=screw with non-zero box_zy is not supported");
3571 static void distribute_cg(FILE *fplog
,
3572 matrix box
, ivec tric_dir
, t_block
*cgs
, rvec pos
[],
3575 gmx_domdec_master_t
*ma
;
3576 int **tmp_ind
= nullptr, *tmp_nalloc
= nullptr;
3577 int i
, icg
, j
, k
, k0
, k1
, d
;
3581 real nrcg
, inv_ncg
, pos_d
;
3587 snew(tmp_nalloc
, dd
->nnodes
);
3588 snew(tmp_ind
, dd
->nnodes
);
3589 for (i
= 0; i
< dd
->nnodes
; i
++)
3591 tmp_nalloc
[i
] = over_alloc_large(cgs
->nr
/dd
->nnodes
+1);
3592 snew(tmp_ind
[i
], tmp_nalloc
[i
]);
3595 /* Clear the count */
3596 for (i
= 0; i
< dd
->nnodes
; i
++)
3602 make_tric_corr_matrix(dd
->npbcdim
, box
, tcm
);
3604 cgindex
= cgs
->index
;
3606 /* Compute the center of geometry for all charge groups */
3607 for (icg
= 0; icg
< cgs
->nr
; icg
++)
3610 k1
= cgindex
[icg
+1];
3614 copy_rvec(pos
[k0
], cg_cm
);
3621 for (k
= k0
; (k
< k1
); k
++)
3623 rvec_inc(cg_cm
, pos
[k
]);
3625 for (d
= 0; (d
< DIM
); d
++)
3627 cg_cm
[d
] *= inv_ncg
;
3630 /* Put the charge group in the box and determine the cell index */
3631 for (d
= DIM
-1; d
>= 0; d
--)
3634 if (d
< dd
->npbcdim
)
3636 bScrew
= (dd
->bScrewPBC
&& d
== XX
);
3637 if (tric_dir
[d
] && dd
->nc
[d
] > 1)
3639 /* Use triclinic coordintates for this dimension */
3640 for (j
= d
+1; j
< DIM
; j
++)
3642 pos_d
+= cg_cm
[j
]*tcm
[j
][d
];
3645 while (pos_d
>= box
[d
][d
])
3648 rvec_dec(cg_cm
, box
[d
]);
3651 cg_cm
[YY
] = box
[YY
][YY
] - cg_cm
[YY
];
3652 cg_cm
[ZZ
] = box
[ZZ
][ZZ
] - cg_cm
[ZZ
];
3654 for (k
= k0
; (k
< k1
); k
++)
3656 rvec_dec(pos
[k
], box
[d
]);
3659 pos
[k
][YY
] = box
[YY
][YY
] - pos
[k
][YY
];
3660 pos
[k
][ZZ
] = box
[ZZ
][ZZ
] - pos
[k
][ZZ
];
3667 rvec_inc(cg_cm
, box
[d
]);
3670 cg_cm
[YY
] = box
[YY
][YY
] - cg_cm
[YY
];
3671 cg_cm
[ZZ
] = box
[ZZ
][ZZ
] - cg_cm
[ZZ
];
3673 for (k
= k0
; (k
< k1
); k
++)
3675 rvec_inc(pos
[k
], box
[d
]);
3678 pos
[k
][YY
] = box
[YY
][YY
] - pos
[k
][YY
];
3679 pos
[k
][ZZ
] = box
[ZZ
][ZZ
] - pos
[k
][ZZ
];
3684 /* This could be done more efficiently */
3686 while (ind
[d
]+1 < dd
->nc
[d
] && pos_d
>= ma
->cell_x
[d
][ind
[d
]+1])
3691 i
= dd_index(dd
->nc
, ind
);
3692 if (ma
->ncg
[i
] == tmp_nalloc
[i
])
3694 tmp_nalloc
[i
] = over_alloc_large(ma
->ncg
[i
]+1);
3695 srenew(tmp_ind
[i
], tmp_nalloc
[i
]);
3697 tmp_ind
[i
][ma
->ncg
[i
]] = icg
;
3699 ma
->nat
[i
] += cgindex
[icg
+1] - cgindex
[icg
];
3703 for (i
= 0; i
< dd
->nnodes
; i
++)
3706 for (k
= 0; k
< ma
->ncg
[i
]; k
++)
3708 ma
->cg
[k1
++] = tmp_ind
[i
][k
];
3711 ma
->index
[dd
->nnodes
] = k1
;
3713 for (i
= 0; i
< dd
->nnodes
; i
++)
3722 // Use double for the sums to avoid natoms^2 overflowing
3724 int nat_sum
, nat_min
, nat_max
;
3729 nat_min
= ma
->nat
[0];
3730 nat_max
= ma
->nat
[0];
3731 for (i
= 0; i
< dd
->nnodes
; i
++)
3733 nat_sum
+= ma
->nat
[i
];
3734 // cast to double to avoid integer overflows when squaring
3735 nat2_sum
+= gmx::square(static_cast<double>(ma
->nat
[i
]));
3736 nat_min
= std::min(nat_min
, ma
->nat
[i
]);
3737 nat_max
= std::max(nat_max
, ma
->nat
[i
]);
3739 nat_sum
/= dd
->nnodes
;
3740 nat2_sum
/= dd
->nnodes
;
3742 fprintf(fplog
, "Atom distribution over %d domains: av %d stddev %d min %d max %d\n",
3745 static_cast<int>(std::sqrt(nat2_sum
- gmx::square(static_cast<double>(nat_sum
)) + 0.5)),
3750 static void get_cg_distribution(FILE *fplog
, gmx_domdec_t
*dd
,
3751 t_block
*cgs
, matrix box
, gmx_ddbox_t
*ddbox
,
3754 gmx_domdec_master_t
*ma
= nullptr;
3757 int *ibuf
, buf2
[2] = { 0, 0 };
3758 gmx_bool bMaster
= DDMASTER(dd
);
3766 check_screw_box(box
);
3769 set_dd_cell_sizes_slb(dd
, ddbox
, setcellsizeslbMASTER
, npulse
);
3771 distribute_cg(fplog
, box
, ddbox
->tric_dir
, cgs
, pos
, dd
);
3772 for (i
= 0; i
< dd
->nnodes
; i
++)
3774 ma
->ibuf
[2*i
] = ma
->ncg
[i
];
3775 ma
->ibuf
[2*i
+1] = ma
->nat
[i
];
3783 dd_scatter(dd
, 2*sizeof(int), ibuf
, buf2
);
3785 dd
->ncg_home
= buf2
[0];
3786 dd
->nat_home
= buf2
[1];
3787 dd
->ncg_tot
= dd
->ncg_home
;
3788 dd
->nat_tot
= dd
->nat_home
;
3789 if (dd
->ncg_home
> dd
->cg_nalloc
|| dd
->cg_nalloc
== 0)
3791 dd
->cg_nalloc
= over_alloc_dd(dd
->ncg_home
);
3792 srenew(dd
->index_gl
, dd
->cg_nalloc
);
3793 srenew(dd
->cgindex
, dd
->cg_nalloc
+1);
3797 for (i
= 0; i
< dd
->nnodes
; i
++)
3799 ma
->ibuf
[i
] = ma
->ncg
[i
]*sizeof(int);
3800 ma
->ibuf
[dd
->nnodes
+i
] = ma
->index
[i
]*sizeof(int);
3805 bMaster
? ma
->ibuf
: nullptr,
3806 bMaster
? ma
->ibuf
+dd
->nnodes
: nullptr,
3807 bMaster
? ma
->cg
: nullptr,
3808 dd
->ncg_home
*sizeof(int), dd
->index_gl
);
3810 /* Determine the home charge group sizes */
3812 for (i
= 0; i
< dd
->ncg_home
; i
++)
3814 cg_gl
= dd
->index_gl
[i
];
3816 dd
->cgindex
[i
] + cgs
->index
[cg_gl
+1] - cgs
->index
[cg_gl
];
3821 fprintf(debug
, "Home charge groups:\n");
3822 for (i
= 0; i
< dd
->ncg_home
; i
++)
3824 fprintf(debug
, " %d", dd
->index_gl
[i
]);
3827 fprintf(debug
, "\n");
3830 fprintf(debug
, "\n");
3834 static int compact_and_copy_vec_at(int ncg
, int *move
,
3837 rvec
*src
, gmx_domdec_comm_t
*comm
,
3840 int m
, icg
, i
, i0
, i1
, nrcg
;
3846 for (m
= 0; m
< DIM
*2; m
++)
3852 for (icg
= 0; icg
< ncg
; icg
++)
3854 i1
= cgindex
[icg
+1];
3860 /* Compact the home array in place */
3861 for (i
= i0
; i
< i1
; i
++)
3863 copy_rvec(src
[i
], src
[home_pos
++]);
3869 /* Copy to the communication buffer */
3871 pos_vec
[m
] += 1 + vec
*nrcg
;
3872 for (i
= i0
; i
< i1
; i
++)
3874 copy_rvec(src
[i
], comm
->cgcm_state
[m
][pos_vec
[m
]++]);
3876 pos_vec
[m
] += (nvec
- vec
- 1)*nrcg
;
3880 home_pos
+= i1
- i0
;
3888 static int compact_and_copy_vec_cg(int ncg
, int *move
,
3890 int nvec
, rvec
*src
, gmx_domdec_comm_t
*comm
,
3893 int m
, icg
, i0
, i1
, nrcg
;
3899 for (m
= 0; m
< DIM
*2; m
++)
3905 for (icg
= 0; icg
< ncg
; icg
++)
3907 i1
= cgindex
[icg
+1];
3913 /* Compact the home array in place */
3914 copy_rvec(src
[icg
], src
[home_pos
++]);
3920 /* Copy to the communication buffer */
3921 copy_rvec(src
[icg
], comm
->cgcm_state
[m
][pos_vec
[m
]]);
3922 pos_vec
[m
] += 1 + nrcg
*nvec
;
3934 static int compact_ind(int ncg
, int *move
,
3935 int *index_gl
, int *cgindex
,
3937 gmx_ga2la_t
*ga2la
, char *bLocalCG
,
3940 int cg
, nat
, a0
, a1
, a
, a_gl
;
3945 for (cg
= 0; cg
< ncg
; cg
++)
3951 /* Compact the home arrays in place.
3952 * Anything that can be done here avoids access to global arrays.
3954 cgindex
[home_pos
] = nat
;
3955 for (a
= a0
; a
< a1
; a
++)
3958 gatindex
[nat
] = a_gl
;
3959 /* The cell number stays 0, so we don't need to set it */
3960 ga2la_change_la(ga2la
, a_gl
, nat
);
3963 index_gl
[home_pos
] = index_gl
[cg
];
3964 cginfo
[home_pos
] = cginfo
[cg
];
3965 /* The charge group remains local, so bLocalCG does not change */
3970 /* Clear the global indices */
3971 for (a
= a0
; a
< a1
; a
++)
3973 ga2la_del(ga2la
, gatindex
[a
]);
3977 bLocalCG
[index_gl
[cg
]] = FALSE
;
3981 cgindex
[home_pos
] = nat
;
3986 static void clear_and_mark_ind(int ncg
, int *move
,
3987 int *index_gl
, int *cgindex
, int *gatindex
,
3988 gmx_ga2la_t
*ga2la
, char *bLocalCG
,
3993 for (cg
= 0; cg
< ncg
; cg
++)
3999 /* Clear the global indices */
4000 for (a
= a0
; a
< a1
; a
++)
4002 ga2la_del(ga2la
, gatindex
[a
]);
4006 bLocalCG
[index_gl
[cg
]] = FALSE
;
4008 /* Signal that this cg has moved using the ns cell index.
4009 * Here we set it to -1. fill_grid will change it
4010 * from -1 to NSGRID_SIGNAL_MOVED_FAC*grid->ncells.
4012 cell_index
[cg
] = -1;
4017 static void print_cg_move(FILE *fplog
,
4019 gmx_int64_t step
, int cg
, int dim
, int dir
,
4020 gmx_bool bHaveCgcmOld
, real limitd
,
4021 rvec cm_old
, rvec cm_new
, real pos_d
)
4023 gmx_domdec_comm_t
*comm
;
4028 fprintf(fplog
, "\nStep %s:\n", gmx_step_str(step
, buf
));
4031 fprintf(fplog
, "%s %d moved more than the distance allowed by the domain decomposition (%f) in direction %c\n",
4032 dd
->comm
->bCGs
? "The charge group starting at atom" : "Atom",
4033 ddglatnr(dd
, dd
->cgindex
[cg
]), limitd
, dim2char(dim
));
4037 /* We don't have a limiting distance available: don't print it */
4038 fprintf(fplog
, "%s %d moved more than the distance allowed by the domain decomposition in direction %c\n",
4039 dd
->comm
->bCGs
? "The charge group starting at atom" : "Atom",
4040 ddglatnr(dd
, dd
->cgindex
[cg
]), dim2char(dim
));
4042 fprintf(fplog
, "distance out of cell %f\n",
4043 dir
== 1 ? pos_d
- comm
->cell_x1
[dim
] : pos_d
- comm
->cell_x0
[dim
]);
4046 fprintf(fplog
, "Old coordinates: %8.3f %8.3f %8.3f\n",
4047 cm_old
[XX
], cm_old
[YY
], cm_old
[ZZ
]);
4049 fprintf(fplog
, "New coordinates: %8.3f %8.3f %8.3f\n",
4050 cm_new
[XX
], cm_new
[YY
], cm_new
[ZZ
]);
4051 fprintf(fplog
, "Old cell boundaries in direction %c: %8.3f %8.3f\n",
4053 comm
->old_cell_x0
[dim
], comm
->old_cell_x1
[dim
]);
4054 fprintf(fplog
, "New cell boundaries in direction %c: %8.3f %8.3f\n",
4056 comm
->cell_x0
[dim
], comm
->cell_x1
[dim
]);
4059 static void cg_move_error(FILE *fplog
,
4061 gmx_int64_t step
, int cg
, int dim
, int dir
,
4062 gmx_bool bHaveCgcmOld
, real limitd
,
4063 rvec cm_old
, rvec cm_new
, real pos_d
)
4067 print_cg_move(fplog
, dd
, step
, cg
, dim
, dir
,
4068 bHaveCgcmOld
, limitd
, cm_old
, cm_new
, pos_d
);
4070 print_cg_move(stderr
, dd
, step
, cg
, dim
, dir
,
4071 bHaveCgcmOld
, limitd
, cm_old
, cm_new
, pos_d
);
4073 "%s moved too far between two domain decomposition steps\n"
4074 "This usually means that your system is not well equilibrated",
4075 dd
->comm
->bCGs
? "A charge group" : "An atom");
4078 static void rotate_state_atom(t_state
*state
, int a
)
4080 if (state
->flags
& (1 << estX
))
4082 /* Rotate the complete state; for a rectangular box only */
4083 state
->x
[a
][YY
] = state
->box
[YY
][YY
] - state
->x
[a
][YY
];
4084 state
->x
[a
][ZZ
] = state
->box
[ZZ
][ZZ
] - state
->x
[a
][ZZ
];
4086 if (state
->flags
& (1 << estV
))
4088 state
->v
[a
][YY
] = -state
->v
[a
][YY
];
4089 state
->v
[a
][ZZ
] = -state
->v
[a
][ZZ
];
4091 if (state
->flags
& (1 << estCGP
))
4093 state
->cg_p
[a
][YY
] = -state
->cg_p
[a
][YY
];
4094 state
->cg_p
[a
][ZZ
] = -state
->cg_p
[a
][ZZ
];
4098 static int *get_moved(gmx_domdec_comm_t
*comm
, int natoms
)
4100 if (natoms
> comm
->moved_nalloc
)
4102 /* Contents should be preserved here */
4103 comm
->moved_nalloc
= over_alloc_dd(natoms
);
4104 srenew(comm
->moved
, comm
->moved_nalloc
);
4110 static void calc_cg_move(FILE *fplog
, gmx_int64_t step
,
4113 ivec tric_dir
, matrix tcm
,
4114 rvec cell_x0
, rvec cell_x1
,
4115 rvec limitd
, rvec limit0
, rvec limit1
,
4117 int cg_start
, int cg_end
,
4122 int cg
, k
, k0
, k1
, d
, dim
, d2
;
4127 real inv_ncg
, pos_d
;
4130 npbcdim
= dd
->npbcdim
;
4132 for (cg
= cg_start
; cg
< cg_end
; cg
++)
4139 copy_rvec(state
->x
[k0
], cm_new
);
4146 for (k
= k0
; (k
< k1
); k
++)
4148 rvec_inc(cm_new
, state
->x
[k
]);
4150 for (d
= 0; (d
< DIM
); d
++)
4152 cm_new
[d
] = inv_ncg
*cm_new
[d
];
4157 /* Do pbc and check DD cell boundary crossings */
4158 for (d
= DIM
-1; d
>= 0; d
--)
4162 bScrew
= (dd
->bScrewPBC
&& d
== XX
);
4163 /* Determine the location of this cg in lattice coordinates */
4167 for (d2
= d
+1; d2
< DIM
; d2
++)
4169 pos_d
+= cm_new
[d2
]*tcm
[d2
][d
];
4172 /* Put the charge group in the triclinic unit-cell */
4173 if (pos_d
>= cell_x1
[d
])
4175 if (pos_d
>= limit1
[d
])
4177 cg_move_error(fplog
, dd
, step
, cg
, d
, 1,
4178 cg_cm
!= as_rvec_array(state
->x
.data()), limitd
[d
],
4179 cg_cm
[cg
], cm_new
, pos_d
);
4182 if (dd
->ci
[d
] == dd
->nc
[d
] - 1)
4184 rvec_dec(cm_new
, state
->box
[d
]);
4187 cm_new
[YY
] = state
->box
[YY
][YY
] - cm_new
[YY
];
4188 cm_new
[ZZ
] = state
->box
[ZZ
][ZZ
] - cm_new
[ZZ
];
4190 for (k
= k0
; (k
< k1
); k
++)
4192 rvec_dec(state
->x
[k
], state
->box
[d
]);
4195 rotate_state_atom(state
, k
);
4200 else if (pos_d
< cell_x0
[d
])
4202 if (pos_d
< limit0
[d
])
4204 cg_move_error(fplog
, dd
, step
, cg
, d
, -1,
4205 cg_cm
!= as_rvec_array(state
->x
.data()), limitd
[d
],
4206 cg_cm
[cg
], cm_new
, pos_d
);
4211 rvec_inc(cm_new
, state
->box
[d
]);
4214 cm_new
[YY
] = state
->box
[YY
][YY
] - cm_new
[YY
];
4215 cm_new
[ZZ
] = state
->box
[ZZ
][ZZ
] - cm_new
[ZZ
];
4217 for (k
= k0
; (k
< k1
); k
++)
4219 rvec_inc(state
->x
[k
], state
->box
[d
]);
4222 rotate_state_atom(state
, k
);
4228 else if (d
< npbcdim
)
4230 /* Put the charge group in the rectangular unit-cell */
4231 while (cm_new
[d
] >= state
->box
[d
][d
])
4233 rvec_dec(cm_new
, state
->box
[d
]);
4234 for (k
= k0
; (k
< k1
); k
++)
4236 rvec_dec(state
->x
[k
], state
->box
[d
]);
4239 while (cm_new
[d
] < 0)
4241 rvec_inc(cm_new
, state
->box
[d
]);
4242 for (k
= k0
; (k
< k1
); k
++)
4244 rvec_inc(state
->x
[k
], state
->box
[d
]);
4250 copy_rvec(cm_new
, cg_cm
[cg
]);
4252 /* Determine where this cg should go */
4255 for (d
= 0; d
< dd
->ndim
; d
++)
4260 flag
|= DD_FLAG_FW(d
);
4266 else if (dev
[dim
] == -1)
4268 flag
|= DD_FLAG_BW(d
);
4271 if (dd
->nc
[dim
] > 2)
4282 /* Temporarily store the flag in move */
4283 move
[cg
] = mc
+ flag
;
4287 static void dd_redistribute_cg(FILE *fplog
, gmx_int64_t step
,
4288 gmx_domdec_t
*dd
, ivec tric_dir
,
4289 t_state
*state
, PaddedRVecVector
*f
,
4298 int ncg
[DIM
*2] = { 0 }, nat
[DIM
*2] = { 0 };
4299 int i
, cg
, k
, d
, dim
, dim2
, dir
, d2
, d3
;
4300 int mc
, cdd
, nrcg
, ncg_recv
, nvs
, nvr
, nvec
, vec
;
4301 int sbuf
[2], rbuf
[2];
4302 int home_pos_cg
, home_pos_at
, buf_pos
;
4306 rvec
*cg_cm
= nullptr, cell_x0
, cell_x1
, limitd
, limit0
, limit1
;
4308 cginfo_mb_t
*cginfo_mb
;
4309 gmx_domdec_comm_t
*comm
;
4311 int nthread
, thread
;
4315 check_screw_box(state
->box
);
4319 if (fr
->cutoff_scheme
== ecutsGROUP
)
4324 // Positions are always present, so there's nothing to flag
4325 bool bV
= state
->flags
& (1<<estV
);
4326 bool bCGP
= state
->flags
& (1<<estCGP
);
4328 if (dd
->ncg_tot
> comm
->nalloc_int
)
4330 comm
->nalloc_int
= over_alloc_dd(dd
->ncg_tot
);
4331 srenew(comm
->buf_int
, comm
->nalloc_int
);
4333 move
= comm
->buf_int
;
4335 npbcdim
= dd
->npbcdim
;
4337 for (d
= 0; (d
< DIM
); d
++)
4339 limitd
[d
] = dd
->comm
->cellsize_min
[d
];
4340 if (d
>= npbcdim
&& dd
->ci
[d
] == 0)
4342 cell_x0
[d
] = -GMX_FLOAT_MAX
;
4346 cell_x0
[d
] = comm
->cell_x0
[d
];
4348 if (d
>= npbcdim
&& dd
->ci
[d
] == dd
->nc
[d
] - 1)
4350 cell_x1
[d
] = GMX_FLOAT_MAX
;
4354 cell_x1
[d
] = comm
->cell_x1
[d
];
4358 limit0
[d
] = comm
->old_cell_x0
[d
] - limitd
[d
];
4359 limit1
[d
] = comm
->old_cell_x1
[d
] + limitd
[d
];
4363 /* We check after communication if a charge group moved
4364 * more than one cell. Set the pre-comm check limit to float_max.
4366 limit0
[d
] = -GMX_FLOAT_MAX
;
4367 limit1
[d
] = GMX_FLOAT_MAX
;
4371 make_tric_corr_matrix(npbcdim
, state
->box
, tcm
);
4373 cgindex
= dd
->cgindex
;
4375 nthread
= gmx_omp_nthreads_get(emntDomdec
);
4377 /* Compute the center of geometry for all home charge groups
4378 * and put them in the box and determine where they should go.
4380 #pragma omp parallel for num_threads(nthread) schedule(static)
4381 for (thread
= 0; thread
< nthread
; thread
++)
4385 calc_cg_move(fplog
, step
, dd
, state
, tric_dir
, tcm
,
4386 cell_x0
, cell_x1
, limitd
, limit0
, limit1
,
4388 ( thread
*dd
->ncg_home
)/nthread
,
4389 ((thread
+1)*dd
->ncg_home
)/nthread
,
4390 fr
->cutoff_scheme
== ecutsGROUP
? cg_cm
: as_rvec_array(state
->x
.data()),
4393 GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR
;
4396 for (cg
= 0; cg
< dd
->ncg_home
; cg
++)
4401 flag
= mc
& ~DD_FLAG_NRCG
;
4402 mc
= mc
& DD_FLAG_NRCG
;
4405 if (ncg
[mc
]+1 > comm
->cggl_flag_nalloc
[mc
])
4407 comm
->cggl_flag_nalloc
[mc
] = over_alloc_dd(ncg
[mc
]+1);
4408 srenew(comm
->cggl_flag
[mc
], comm
->cggl_flag_nalloc
[mc
]*DD_CGIBS
);
4410 comm
->cggl_flag
[mc
][ncg
[mc
]*DD_CGIBS
] = dd
->index_gl
[cg
];
4411 /* We store the cg size in the lower 16 bits
4412 * and the place where the charge group should go
4413 * in the next 6 bits. This saves some communication volume.
4415 nrcg
= cgindex
[cg
+1] - cgindex
[cg
];
4416 comm
->cggl_flag
[mc
][ncg
[mc
]*DD_CGIBS
+1] = nrcg
| flag
;
4422 inc_nrnb(nrnb
, eNR_CGCM
, dd
->nat_home
);
4423 inc_nrnb(nrnb
, eNR_RESETX
, dd
->ncg_home
);
4426 for (i
= 0; i
< dd
->ndim
*2; i
++)
4428 *ncg_moved
+= ncg
[i
];
4441 /* Make sure the communication buffers are large enough */
4442 for (mc
= 0; mc
< dd
->ndim
*2; mc
++)
4444 nvr
= ncg
[mc
] + nat
[mc
]*nvec
;
4445 if (nvr
> comm
->cgcm_state_nalloc
[mc
])
4447 comm
->cgcm_state_nalloc
[mc
] = over_alloc_dd(nvr
);
4448 srenew(comm
->cgcm_state
[mc
], comm
->cgcm_state_nalloc
[mc
]);
4452 switch (fr
->cutoff_scheme
)
4455 /* Recalculating cg_cm might be cheaper than communicating,
4456 * but that could give rise to rounding issues.
4459 compact_and_copy_vec_cg(dd
->ncg_home
, move
, cgindex
,
4460 nvec
, cg_cm
, comm
, bCompact
);
4463 /* Without charge groups we send the moved atom coordinates
4464 * over twice. This is so the code below can be used without
4465 * many conditionals for both for with and without charge groups.
4468 compact_and_copy_vec_cg(dd
->ncg_home
, move
, cgindex
,
4469 nvec
, as_rvec_array(state
->x
.data()), comm
, FALSE
);
4472 home_pos_cg
-= *ncg_moved
;
4476 gmx_incons("unimplemented");
4482 compact_and_copy_vec_at(dd
->ncg_home
, move
, cgindex
,
4483 nvec
, vec
++, as_rvec_array(state
->x
.data()),
4487 compact_and_copy_vec_at(dd
->ncg_home
, move
, cgindex
,
4488 nvec
, vec
++, as_rvec_array(state
->v
.data()),
4493 compact_and_copy_vec_at(dd
->ncg_home
, move
, cgindex
,
4494 nvec
, vec
++, as_rvec_array(state
->cg_p
.data()),
4500 compact_ind(dd
->ncg_home
, move
,
4501 dd
->index_gl
, dd
->cgindex
, dd
->gatindex
,
4502 dd
->ga2la
, comm
->bLocalCG
,
4507 if (fr
->cutoff_scheme
== ecutsVERLET
)
4509 moved
= get_moved(comm
, dd
->ncg_home
);
4511 for (k
= 0; k
< dd
->ncg_home
; k
++)
4518 moved
= fr
->ns
->grid
->cell_index
;
4521 clear_and_mark_ind(dd
->ncg_home
, move
,
4522 dd
->index_gl
, dd
->cgindex
, dd
->gatindex
,
4523 dd
->ga2la
, comm
->bLocalCG
,
4527 cginfo_mb
= fr
->cginfo_mb
;
4529 *ncg_stay_home
= home_pos_cg
;
4530 for (d
= 0; d
< dd
->ndim
; d
++)
4535 for (dir
= 0; dir
< (dd
->nc
[dim
] == 2 ? 1 : 2); dir
++)
4538 /* Communicate the cg and atom counts */
4543 fprintf(debug
, "Sending ddim %d dir %d: ncg %d nat %d\n",
4544 d
, dir
, sbuf
[0], sbuf
[1]);
4546 dd_sendrecv_int(dd
, d
, dir
, sbuf
, 2, rbuf
, 2);
4548 if ((ncg_recv
+rbuf
[0])*DD_CGIBS
> comm
->nalloc_int
)
4550 comm
->nalloc_int
= over_alloc_dd((ncg_recv
+rbuf
[0])*DD_CGIBS
);
4551 srenew(comm
->buf_int
, comm
->nalloc_int
);
4554 /* Communicate the charge group indices, sizes and flags */
4555 dd_sendrecv_int(dd
, d
, dir
,
4556 comm
->cggl_flag
[cdd
], sbuf
[0]*DD_CGIBS
,
4557 comm
->buf_int
+ncg_recv
*DD_CGIBS
, rbuf
[0]*DD_CGIBS
);
4559 nvs
= ncg
[cdd
] + nat
[cdd
]*nvec
;
4560 i
= rbuf
[0] + rbuf
[1] *nvec
;
4561 vec_rvec_check_alloc(&comm
->vbuf
, nvr
+i
);
4563 /* Communicate cgcm and state */
4564 dd_sendrecv_rvec(dd
, d
, dir
,
4565 comm
->cgcm_state
[cdd
], nvs
,
4566 comm
->vbuf
.v
+nvr
, i
);
4567 ncg_recv
+= rbuf
[0];
4571 dd_check_alloc_ncg(fr
, state
, f
, home_pos_cg
+ ncg_recv
);
4572 if (fr
->cutoff_scheme
== ecutsGROUP
)
4574 /* Here we resize to more than necessary and shrink later */
4575 dd_resize_state(state
, f
, home_pos_at
+ ncg_recv
*MAX_CGCGSIZE
);
4578 /* Process the received charge groups */
4580 for (cg
= 0; cg
< ncg_recv
; cg
++)
4582 flag
= comm
->buf_int
[cg
*DD_CGIBS
+1];
4584 if (dim
>= npbcdim
&& dd
->nc
[dim
] > 2)
4586 /* No pbc in this dim and more than one domain boundary.
4587 * We do a separate check if a charge group didn't move too far.
4589 if (((flag
& DD_FLAG_FW(d
)) &&
4590 comm
->vbuf
.v
[buf_pos
][dim
] > cell_x1
[dim
]) ||
4591 ((flag
& DD_FLAG_BW(d
)) &&
4592 comm
->vbuf
.v
[buf_pos
][dim
] < cell_x0
[dim
]))
4594 cg_move_error(fplog
, dd
, step
, cg
, dim
,
4595 (flag
& DD_FLAG_FW(d
)) ? 1 : 0,
4596 fr
->cutoff_scheme
== ecutsGROUP
, 0,
4597 comm
->vbuf
.v
[buf_pos
],
4598 comm
->vbuf
.v
[buf_pos
],
4599 comm
->vbuf
.v
[buf_pos
][dim
]);
4606 /* Check which direction this cg should go */
4607 for (d2
= d
+1; (d2
< dd
->ndim
&& mc
== -1); d2
++)
4609 if (dlbIsOn(dd
->comm
))
4611 /* The cell boundaries for dimension d2 are not equal
4612 * for each cell row of the lower dimension(s),
4613 * therefore we might need to redetermine where
4614 * this cg should go.
4617 /* If this cg crosses the box boundary in dimension d2
4618 * we can use the communicated flag, so we do not
4619 * have to worry about pbc.
4621 if (!((dd
->ci
[dim2
] == dd
->nc
[dim2
]-1 &&
4622 (flag
& DD_FLAG_FW(d2
))) ||
4623 (dd
->ci
[dim2
] == 0 &&
4624 (flag
& DD_FLAG_BW(d2
)))))
4626 /* Clear the two flags for this dimension */
4627 flag
&= ~(DD_FLAG_FW(d2
) | DD_FLAG_BW(d2
));
4628 /* Determine the location of this cg
4629 * in lattice coordinates
4631 pos_d
= comm
->vbuf
.v
[buf_pos
][dim2
];
4634 for (d3
= dim2
+1; d3
< DIM
; d3
++)
4637 comm
->vbuf
.v
[buf_pos
][d3
]*tcm
[d3
][dim2
];
4640 /* Check of we are not at the box edge.
4641 * pbc is only handled in the first step above,
4642 * but this check could move over pbc while
4643 * the first step did not due to different rounding.
4645 if (pos_d
>= cell_x1
[dim2
] &&
4646 dd
->ci
[dim2
] != dd
->nc
[dim2
]-1)
4648 flag
|= DD_FLAG_FW(d2
);
4650 else if (pos_d
< cell_x0
[dim2
] &&
4653 flag
|= DD_FLAG_BW(d2
);
4655 comm
->buf_int
[cg
*DD_CGIBS
+1] = flag
;
4658 /* Set to which neighboring cell this cg should go */
4659 if (flag
& DD_FLAG_FW(d2
))
4663 else if (flag
& DD_FLAG_BW(d2
))
4665 if (dd
->nc
[dd
->dim
[d2
]] > 2)
4677 nrcg
= flag
& DD_FLAG_NRCG
;
4680 if (home_pos_cg
+1 > dd
->cg_nalloc
)
4682 dd
->cg_nalloc
= over_alloc_dd(home_pos_cg
+1);
4683 srenew(dd
->index_gl
, dd
->cg_nalloc
);
4684 srenew(dd
->cgindex
, dd
->cg_nalloc
+1);
4686 /* Set the global charge group index and size */
4687 dd
->index_gl
[home_pos_cg
] = comm
->buf_int
[cg
*DD_CGIBS
];
4688 dd
->cgindex
[home_pos_cg
+1] = dd
->cgindex
[home_pos_cg
] + nrcg
;
4689 /* Copy the state from the buffer */
4690 if (fr
->cutoff_scheme
== ecutsGROUP
)
4693 copy_rvec(comm
->vbuf
.v
[buf_pos
], cg_cm
[home_pos_cg
]);
4697 /* Set the cginfo */
4698 fr
->cginfo
[home_pos_cg
] = ddcginfo(cginfo_mb
,
4699 dd
->index_gl
[home_pos_cg
]);
4702 comm
->bLocalCG
[dd
->index_gl
[home_pos_cg
]] = TRUE
;
4705 for (i
= 0; i
< nrcg
; i
++)
4707 copy_rvec(comm
->vbuf
.v
[buf_pos
++],
4708 state
->x
[home_pos_at
+i
]);
4712 for (i
= 0; i
< nrcg
; i
++)
4714 copy_rvec(comm
->vbuf
.v
[buf_pos
++],
4715 state
->v
[home_pos_at
+i
]);
4720 for (i
= 0; i
< nrcg
; i
++)
4722 copy_rvec(comm
->vbuf
.v
[buf_pos
++],
4723 state
->cg_p
[home_pos_at
+i
]);
4727 home_pos_at
+= nrcg
;
4731 /* Reallocate the buffers if necessary */
4732 if (ncg
[mc
]+1 > comm
->cggl_flag_nalloc
[mc
])
4734 comm
->cggl_flag_nalloc
[mc
] = over_alloc_dd(ncg
[mc
]+1);
4735 srenew(comm
->cggl_flag
[mc
], comm
->cggl_flag_nalloc
[mc
]*DD_CGIBS
);
4737 nvr
= ncg
[mc
] + nat
[mc
]*nvec
;
4738 if (nvr
+ 1 + nrcg
*nvec
> comm
->cgcm_state_nalloc
[mc
])
4740 comm
->cgcm_state_nalloc
[mc
] = over_alloc_dd(nvr
+ 1 + nrcg
*nvec
);
4741 srenew(comm
->cgcm_state
[mc
], comm
->cgcm_state_nalloc
[mc
]);
4743 /* Copy from the receive to the send buffers */
4744 memcpy(comm
->cggl_flag
[mc
] + ncg
[mc
]*DD_CGIBS
,
4745 comm
->buf_int
+ cg
*DD_CGIBS
,
4746 DD_CGIBS
*sizeof(int));
4747 memcpy(comm
->cgcm_state
[mc
][nvr
],
4748 comm
->vbuf
.v
[buf_pos
],
4749 (1+nrcg
*nvec
)*sizeof(rvec
));
4750 buf_pos
+= 1 + nrcg
*nvec
;
4757 /* With sorting (!bCompact) the indices are now only partially up to date
4758 * and ncg_home and nat_home are not the real count, since there are
4759 * "holes" in the arrays for the charge groups that moved to neighbors.
4761 if (fr
->cutoff_scheme
== ecutsVERLET
)
4763 moved
= get_moved(comm
, home_pos_cg
);
4765 for (i
= dd
->ncg_home
; i
< home_pos_cg
; i
++)
4770 dd
->ncg_home
= home_pos_cg
;
4771 dd
->nat_home
= home_pos_at
;
4773 if (fr
->cutoff_scheme
== ecutsGROUP
&& !bCompact
)
4775 /* We overallocated before, we need to set the right size here */
4776 dd_resize_state(state
, f
, dd
->nat_home
);
4782 "Finished repartitioning: cgs moved out %d, new home %d\n",
4783 *ncg_moved
, dd
->ncg_home
-*ncg_moved
);
4788 void dd_cycles_add(gmx_domdec_t
*dd
, float cycles
, int ddCycl
)
4790 /* Note that the cycles value can be incorrect, either 0 or some
4791 * extremely large value, when our thread migrated to another core
4792 * with an unsynchronized cycle counter. If this happens less often
4793 * that once per nstlist steps, this will not cause issues, since
4794 * we later subtract the maximum value from the sum over nstlist steps.
4795 * A zero count will slightly lower the total, but that's a small effect.
4796 * Note that the main purpose of the subtraction of the maximum value
4797 * is to avoid throwing off the load balancing when stalls occur due
4798 * e.g. system activity or network congestion.
4800 dd
->comm
->cycl
[ddCycl
] += cycles
;
4801 dd
->comm
->cycl_n
[ddCycl
]++;
4802 if (cycles
> dd
->comm
->cycl_max
[ddCycl
])
4804 dd
->comm
->cycl_max
[ddCycl
] = cycles
;
4808 static double force_flop_count(t_nrnb
*nrnb
)
4815 for (i
= 0; i
< eNR_NBKERNEL_FREE_ENERGY
; i
++)
4817 /* To get closer to the real timings, we half the count
4818 * for the normal loops and again half it for water loops.
4821 if (strstr(name
, "W3") != nullptr || strstr(name
, "W4") != nullptr)
4823 sum
+= nrnb
->n
[i
]*0.25*cost_nrnb(i
);
4827 sum
+= nrnb
->n
[i
]*0.50*cost_nrnb(i
);
4830 for (i
= eNR_NBKERNEL_FREE_ENERGY
; i
<= eNR_NB14
; i
++)
4833 if (strstr(name
, "W3") != nullptr || strstr(name
, "W4") != nullptr)
4835 sum
+= nrnb
->n
[i
]*cost_nrnb(i
);
4838 for (i
= eNR_BONDS
; i
<= eNR_WALLS
; i
++)
4840 sum
+= nrnb
->n
[i
]*cost_nrnb(i
);
4846 void dd_force_flop_start(gmx_domdec_t
*dd
, t_nrnb
*nrnb
)
4848 if (dd
->comm
->eFlop
)
4850 dd
->comm
->flop
-= force_flop_count(nrnb
);
4853 void dd_force_flop_stop(gmx_domdec_t
*dd
, t_nrnb
*nrnb
)
4855 if (dd
->comm
->eFlop
)
4857 dd
->comm
->flop
+= force_flop_count(nrnb
);
4862 static void clear_dd_cycle_counts(gmx_domdec_t
*dd
)
4866 for (i
= 0; i
< ddCyclNr
; i
++)
4868 dd
->comm
->cycl
[i
] = 0;
4869 dd
->comm
->cycl_n
[i
] = 0;
4870 dd
->comm
->cycl_max
[i
] = 0;
4873 dd
->comm
->flop_n
= 0;
4876 static void get_load_distribution(gmx_domdec_t
*dd
, gmx_wallcycle_t wcycle
)
4878 gmx_domdec_comm_t
*comm
;
4879 domdec_load_t
*load
;
4880 domdec_root_t
*root
= nullptr;
4882 float cell_frac
= 0, sbuf
[DD_NLOAD_MAX
];
4887 fprintf(debug
, "get_load_distribution start\n");
4890 wallcycle_start(wcycle
, ewcDDCOMMLOAD
);
4894 bSepPME
= (dd
->pme_nodeid
>= 0);
4896 if (dd
->ndim
== 0 && bSepPME
)
4898 /* Without decomposition, but with PME nodes, we need the load */
4899 comm
->load
[0].mdf
= comm
->cycl
[ddCyclPPduringPME
];
4900 comm
->load
[0].pme
= comm
->cycl
[ddCyclPME
];
4903 for (d
= dd
->ndim
-1; d
>= 0; d
--)
4906 /* Check if we participate in the communication in this dimension */
4907 if (d
== dd
->ndim
-1 ||
4908 (dd
->ci
[dd
->dim
[d
+1]] == 0 && dd
->ci
[dd
->dim
[dd
->ndim
-1]] == 0))
4910 load
= &comm
->load
[d
];
4911 if (dlbIsOn(dd
->comm
))
4913 cell_frac
= comm
->cell_f1
[d
] - comm
->cell_f0
[d
];
4916 if (d
== dd
->ndim
-1)
4918 sbuf
[pos
++] = dd_force_load(comm
);
4919 sbuf
[pos
++] = sbuf
[0];
4920 if (dlbIsOn(dd
->comm
))
4922 sbuf
[pos
++] = sbuf
[0];
4923 sbuf
[pos
++] = cell_frac
;
4926 sbuf
[pos
++] = comm
->cell_f_max0
[d
];
4927 sbuf
[pos
++] = comm
->cell_f_min1
[d
];
4932 sbuf
[pos
++] = comm
->cycl
[ddCyclPPduringPME
];
4933 sbuf
[pos
++] = comm
->cycl
[ddCyclPME
];
4938 sbuf
[pos
++] = comm
->load
[d
+1].sum
;
4939 sbuf
[pos
++] = comm
->load
[d
+1].max
;
4940 if (dlbIsOn(dd
->comm
))
4942 sbuf
[pos
++] = comm
->load
[d
+1].sum_m
;
4943 sbuf
[pos
++] = comm
->load
[d
+1].cvol_min
*cell_frac
;
4944 sbuf
[pos
++] = comm
->load
[d
+1].flags
;
4947 sbuf
[pos
++] = comm
->cell_f_max0
[d
];
4948 sbuf
[pos
++] = comm
->cell_f_min1
[d
];
4953 sbuf
[pos
++] = comm
->load
[d
+1].mdf
;
4954 sbuf
[pos
++] = comm
->load
[d
+1].pme
;
4958 /* Communicate a row in DD direction d.
4959 * The communicators are setup such that the root always has rank 0.
4962 MPI_Gather(sbuf
, load
->nload
*sizeof(float), MPI_BYTE
,
4963 load
->load
, load
->nload
*sizeof(float), MPI_BYTE
,
4964 0, comm
->mpi_comm_load
[d
]);
4966 if (dd
->ci
[dim
] == dd
->master_ci
[dim
])
4968 /* We are the root, process this row */
4971 root
= comm
->root
[d
];
4981 for (i
= 0; i
< dd
->nc
[dim
]; i
++)
4983 load
->sum
+= load
->load
[pos
++];
4984 load
->max
= std::max(load
->max
, load
->load
[pos
]);
4986 if (dlbIsOn(dd
->comm
))
4990 /* This direction could not be load balanced properly,
4991 * therefore we need to use the maximum iso the average load.
4993 load
->sum_m
= std::max(load
->sum_m
, load
->load
[pos
]);
4997 load
->sum_m
+= load
->load
[pos
];
5000 load
->cvol_min
= std::min(load
->cvol_min
, load
->load
[pos
]);
5004 load
->flags
= (int)(load
->load
[pos
++] + 0.5);
5008 root
->cell_f_max0
[i
] = load
->load
[pos
++];
5009 root
->cell_f_min1
[i
] = load
->load
[pos
++];
5014 load
->mdf
= std::max(load
->mdf
, load
->load
[pos
]);
5016 load
->pme
= std::max(load
->pme
, load
->load
[pos
]);
5020 if (dlbIsOn(comm
) && root
->bLimited
)
5022 load
->sum_m
*= dd
->nc
[dim
];
5023 load
->flags
|= (1<<d
);
5031 comm
->nload
+= dd_load_count(comm
);
5032 comm
->load_step
+= comm
->cycl
[ddCyclStep
];
5033 comm
->load_sum
+= comm
->load
[0].sum
;
5034 comm
->load_max
+= comm
->load
[0].max
;
5037 for (d
= 0; d
< dd
->ndim
; d
++)
5039 if (comm
->load
[0].flags
& (1<<d
))
5041 comm
->load_lim
[d
]++;
5047 comm
->load_mdf
+= comm
->load
[0].mdf
;
5048 comm
->load_pme
+= comm
->load
[0].pme
;
5052 wallcycle_stop(wcycle
, ewcDDCOMMLOAD
);
5056 fprintf(debug
, "get_load_distribution finished\n");
5060 static float dd_force_imb_perf_loss(gmx_domdec_t
*dd
)
5062 /* Return the relative performance loss on the total run time
5063 * due to the force calculation load imbalance.
5065 if (dd
->comm
->nload
> 0 && dd
->comm
->load_step
> 0)
5068 (dd
->comm
->load_max
*dd
->nnodes
- dd
->comm
->load_sum
)/
5069 (dd
->comm
->load_step
*dd
->nnodes
);
5077 static void print_dd_load_av(FILE *fplog
, gmx_domdec_t
*dd
)
5080 int npp
, npme
, nnodes
, d
, limp
;
5081 float imbal
, pme_f_ratio
, lossf
= 0, lossp
= 0;
5083 gmx_domdec_comm_t
*comm
;
5086 if (DDMASTER(dd
) && comm
->nload
> 0)
5089 npme
= (dd
->pme_nodeid
>= 0) ? comm
->npmenodes
: 0;
5090 nnodes
= npp
+ npme
;
5091 if (dd
->nnodes
> 1 && comm
->load_sum
> 0)
5093 imbal
= comm
->load_max
*npp
/comm
->load_sum
- 1;
5094 lossf
= dd_force_imb_perf_loss(dd
);
5095 sprintf(buf
, " Average load imbalance: %.1f %%\n", imbal
*100);
5096 fprintf(fplog
, "%s", buf
);
5097 fprintf(stderr
, "\n");
5098 fprintf(stderr
, "%s", buf
);
5099 sprintf(buf
, " Part of the total run time spent waiting due to load imbalance: %.1f %%\n", lossf
*100);
5100 fprintf(fplog
, "%s", buf
);
5101 fprintf(stderr
, "%s", buf
);
5106 sprintf(buf
, " Steps where the load balancing was limited by -rdd, -rcon and/or -dds:");
5107 for (d
= 0; d
< dd
->ndim
; d
++)
5109 limp
= (200*comm
->load_lim
[d
]+1)/(2*comm
->nload
);
5110 sprintf(buf
+strlen(buf
), " %c %d %%", dim2char(dd
->dim
[d
]), limp
);
5116 sprintf(buf
+strlen(buf
), "\n");
5117 fprintf(fplog
, "%s", buf
);
5118 fprintf(stderr
, "%s", buf
);
5120 if (npme
> 0 && comm
->load_mdf
> 0 && comm
->load_step
> 0)
5122 pme_f_ratio
= comm
->load_pme
/comm
->load_mdf
;
5123 lossp
= (comm
->load_pme
- comm
->load_mdf
)/comm
->load_step
;
5126 lossp
*= (float)npme
/(float)nnodes
;
5130 lossp
*= (float)npp
/(float)nnodes
;
5132 sprintf(buf
, " Average PME mesh/force load: %5.3f\n", pme_f_ratio
);
5133 fprintf(fplog
, "%s", buf
);
5134 fprintf(stderr
, "%s", buf
);
5135 sprintf(buf
, " Part of the total run time spent waiting due to PP/PME imbalance: %.1f %%\n", fabs(lossp
)*100);
5136 fprintf(fplog
, "%s", buf
);
5137 fprintf(stderr
, "%s", buf
);
5139 fprintf(fplog
, "\n");
5140 fprintf(stderr
, "\n");
5142 if (lossf
>= DD_PERF_LOSS_WARN
)
5145 "NOTE: %.1f %% of the available CPU time was lost due to load imbalance\n"
5146 " in the domain decomposition.\n", lossf
*100);
5149 sprintf(buf
+strlen(buf
), " You might want to use dynamic load balancing (option -dlb.)\n");
5153 sprintf(buf
+strlen(buf
), " You might want to decrease the cell size limit (options -rdd, -rcon and/or -dds).\n");
5155 fprintf(fplog
, "%s\n", buf
);
5156 fprintf(stderr
, "%s\n", buf
);
5158 if (npme
> 0 && fabs(lossp
) >= DD_PERF_LOSS_WARN
)
5161 "NOTE: %.1f %% performance was lost because the PME ranks\n"
5162 " had %s work to do than the PP ranks.\n"
5163 " You might want to %s the number of PME ranks\n"
5164 " or %s the cut-off and the grid spacing.\n",
5166 (lossp
< 0) ? "less" : "more",
5167 (lossp
< 0) ? "decrease" : "increase",
5168 (lossp
< 0) ? "decrease" : "increase");
5169 fprintf(fplog
, "%s\n", buf
);
5170 fprintf(stderr
, "%s\n", buf
);
5175 static float dd_vol_min(gmx_domdec_t
*dd
)
5177 return dd
->comm
->load
[0].cvol_min
*dd
->nnodes
;
5180 static gmx_bool
dd_load_flags(gmx_domdec_t
*dd
)
5182 return dd
->comm
->load
[0].flags
;
5185 static float dd_f_imbal(gmx_domdec_t
*dd
)
5187 if (dd
->comm
->load
[0].sum
> 0)
5189 return dd
->comm
->load
[0].max
*dd
->nnodes
/dd
->comm
->load
[0].sum
- 1.0f
;
5193 /* Something is wrong in the cycle counting, report no load imbalance */
5198 float dd_pme_f_ratio(gmx_domdec_t
*dd
)
5200 /* Should only be called on the DD master rank */
5201 assert(DDMASTER(dd
));
5203 if (dd
->comm
->load
[0].mdf
> 0 && dd
->comm
->cycl_n
[ddCyclPME
] > 0)
5205 return dd
->comm
->load
[0].pme
/dd
->comm
->load
[0].mdf
;
5213 static void dd_print_load(FILE *fplog
, gmx_domdec_t
*dd
, gmx_int64_t step
)
5218 flags
= dd_load_flags(dd
);
5222 "DD load balancing is limited by minimum cell size in dimension");
5223 for (d
= 0; d
< dd
->ndim
; d
++)
5227 fprintf(fplog
, " %c", dim2char(dd
->dim
[d
]));
5230 fprintf(fplog
, "\n");
5232 fprintf(fplog
, "DD step %s", gmx_step_str(step
, buf
));
5233 if (dlbIsOn(dd
->comm
))
5235 fprintf(fplog
, " vol min/aver %5.3f%c",
5236 dd_vol_min(dd
), flags
? '!' : ' ');
5240 fprintf(fplog
, " load imb.: force %4.1f%%", dd_f_imbal(dd
)*100);
5242 if (dd
->comm
->cycl_n
[ddCyclPME
])
5244 fprintf(fplog
, " pme mesh/force %5.3f", dd_pme_f_ratio(dd
));
5246 fprintf(fplog
, "\n\n");
5249 static void dd_print_load_verbose(gmx_domdec_t
*dd
)
5251 if (dlbIsOn(dd
->comm
))
5253 fprintf(stderr
, "vol %4.2f%c ",
5254 dd_vol_min(dd
), dd_load_flags(dd
) ? '!' : ' ');
5258 fprintf(stderr
, "imb F %2d%% ", (int)(dd_f_imbal(dd
)*100+0.5));
5260 if (dd
->comm
->cycl_n
[ddCyclPME
])
5262 fprintf(stderr
, "pme/F %4.2f ", dd_pme_f_ratio(dd
));
5267 static void make_load_communicator(gmx_domdec_t
*dd
, int dim_ind
, ivec loc
)
5272 domdec_root_t
*root
;
5273 gmx_bool bPartOfGroup
= FALSE
;
5275 dim
= dd
->dim
[dim_ind
];
5276 copy_ivec(loc
, loc_c
);
5277 for (i
= 0; i
< dd
->nc
[dim
]; i
++)
5280 rank
= dd_index(dd
->nc
, loc_c
);
5281 if (rank
== dd
->rank
)
5283 /* This process is part of the group */
5284 bPartOfGroup
= TRUE
;
5287 MPI_Comm_split(dd
->mpi_comm_all
, bPartOfGroup
? 0 : MPI_UNDEFINED
, dd
->rank
,
5291 dd
->comm
->mpi_comm_load
[dim_ind
] = c_row
;
5292 if (dd
->comm
->dlbState
!= edlbsOffForever
)
5294 if (dd
->ci
[dim
] == dd
->master_ci
[dim
])
5296 /* This is the root process of this row */
5297 snew(dd
->comm
->root
[dim_ind
], 1);
5298 root
= dd
->comm
->root
[dim_ind
];
5299 snew(root
->cell_f
, DD_CELL_F_SIZE(dd
, dim_ind
));
5300 snew(root
->old_cell_f
, dd
->nc
[dim
]+1);
5301 snew(root
->bCellMin
, dd
->nc
[dim
]);
5304 snew(root
->cell_f_max0
, dd
->nc
[dim
]);
5305 snew(root
->cell_f_min1
, dd
->nc
[dim
]);
5306 snew(root
->bound_min
, dd
->nc
[dim
]);
5307 snew(root
->bound_max
, dd
->nc
[dim
]);
5309 snew(root
->buf_ncd
, dd
->nc
[dim
]);
5313 /* This is not a root process, we only need to receive cell_f */
5314 snew(dd
->comm
->cell_f_row
, DD_CELL_F_SIZE(dd
, dim_ind
));
5317 if (dd
->ci
[dim
] == dd
->master_ci
[dim
])
5319 snew(dd
->comm
->load
[dim_ind
].load
, dd
->nc
[dim
]*DD_NLOAD_MAX
);
5325 void dd_setup_dlb_resource_sharing(t_commrec gmx_unused
*cr
,
5326 const gmx_hw_info_t gmx_unused
*hwinfo
,
5327 const gmx_hw_opt_t gmx_unused
*hw_opt
)
5330 int physicalnode_id_hash
;
5333 MPI_Comm mpi_comm_pp_physicalnode
;
5335 if (!(cr
->duty
& DUTY_PP
) || hw_opt
->gpu_opt
.n_dev_use
== 0)
5337 /* Only PP nodes (currently) use GPUs.
5338 * If we don't have GPUs, there are no resources to share.
5343 physicalnode_id_hash
= gmx_physicalnode_id_hash();
5345 gpu_id
= get_gpu_device_id(&hwinfo
->gpu_info
, &hw_opt
->gpu_opt
, cr
->rank_pp_intranode
);
5351 fprintf(debug
, "dd_setup_dd_dlb_gpu_sharing:\n");
5352 fprintf(debug
, "DD PP rank %d physical node hash %d gpu_id %d\n",
5353 dd
->rank
, physicalnode_id_hash
, gpu_id
);
5355 /* Split the PP communicator over the physical nodes */
5356 /* TODO: See if we should store this (before), as it's also used for
5357 * for the nodecomm summution.
5359 MPI_Comm_split(dd
->mpi_comm_all
, physicalnode_id_hash
, dd
->rank
,
5360 &mpi_comm_pp_physicalnode
);
5361 MPI_Comm_split(mpi_comm_pp_physicalnode
, gpu_id
, dd
->rank
,
5362 &dd
->comm
->mpi_comm_gpu_shared
);
5363 MPI_Comm_free(&mpi_comm_pp_physicalnode
);
5364 MPI_Comm_size(dd
->comm
->mpi_comm_gpu_shared
, &dd
->comm
->nrank_gpu_shared
);
5368 fprintf(debug
, "nrank_gpu_shared %d\n", dd
->comm
->nrank_gpu_shared
);
5371 /* Note that some ranks could share a GPU, while others don't */
5373 if (dd
->comm
->nrank_gpu_shared
== 1)
5375 MPI_Comm_free(&dd
->comm
->mpi_comm_gpu_shared
);
5380 static void make_load_communicators(gmx_domdec_t gmx_unused
*dd
)
5383 int dim0
, dim1
, i
, j
;
5388 fprintf(debug
, "Making load communicators\n");
5391 snew(dd
->comm
->load
, std::max(dd
->ndim
, 1));
5392 snew(dd
->comm
->mpi_comm_load
, std::max(dd
->ndim
, 1));
5400 make_load_communicator(dd
, 0, loc
);
5404 for (i
= 0; i
< dd
->nc
[dim0
]; i
++)
5407 make_load_communicator(dd
, 1, loc
);
5413 for (i
= 0; i
< dd
->nc
[dim0
]; i
++)
5417 for (j
= 0; j
< dd
->nc
[dim1
]; j
++)
5420 make_load_communicator(dd
, 2, loc
);
5427 fprintf(debug
, "Finished making load communicators\n");
5432 /*! \brief Sets up the relation between neighboring domains and zones */
5433 static void setup_neighbor_relations(gmx_domdec_t
*dd
)
5435 int d
, dim
, i
, j
, m
;
5437 gmx_domdec_zones_t
*zones
;
5438 gmx_domdec_ns_ranges_t
*izone
;
5440 for (d
= 0; d
< dd
->ndim
; d
++)
5443 copy_ivec(dd
->ci
, tmp
);
5444 tmp
[dim
] = (tmp
[dim
] + 1) % dd
->nc
[dim
];
5445 dd
->neighbor
[d
][0] = ddcoord2ddnodeid(dd
, tmp
);
5446 copy_ivec(dd
->ci
, tmp
);
5447 tmp
[dim
] = (tmp
[dim
] - 1 + dd
->nc
[dim
]) % dd
->nc
[dim
];
5448 dd
->neighbor
[d
][1] = ddcoord2ddnodeid(dd
, tmp
);
5451 fprintf(debug
, "DD rank %d neighbor ranks in dir %d are + %d - %d\n",
5454 dd
->neighbor
[d
][1]);
5458 int nzone
= (1 << dd
->ndim
);
5459 int nizone
= (1 << std::max(dd
->ndim
- 1, 0));
5460 assert(nizone
>= 1 && nizone
<= DD_MAXIZONE
);
5462 zones
= &dd
->comm
->zones
;
5464 for (i
= 0; i
< nzone
; i
++)
5467 clear_ivec(zones
->shift
[i
]);
5468 for (d
= 0; d
< dd
->ndim
; d
++)
5470 zones
->shift
[i
][dd
->dim
[d
]] = dd_zo
[i
][m
++];
5475 for (i
= 0; i
< nzone
; i
++)
5477 for (d
= 0; d
< DIM
; d
++)
5479 s
[d
] = dd
->ci
[d
] - zones
->shift
[i
][d
];
5484 else if (s
[d
] >= dd
->nc
[d
])
5490 zones
->nizone
= nizone
;
5491 for (i
= 0; i
< zones
->nizone
; i
++)
5493 assert(ddNonbondedZonePairRanges
[i
][0] == i
);
5495 izone
= &zones
->izone
[i
];
5496 /* dd_zp3 is for 3D decomposition, for fewer dimensions use only
5497 * j-zones up to nzone.
5499 izone
->j0
= std::min(ddNonbondedZonePairRanges
[i
][1], nzone
);
5500 izone
->j1
= std::min(ddNonbondedZonePairRanges
[i
][2], nzone
);
5501 for (dim
= 0; dim
< DIM
; dim
++)
5503 if (dd
->nc
[dim
] == 1)
5505 /* All shifts should be allowed */
5506 izone
->shift0
[dim
] = -1;
5507 izone
->shift1
[dim
] = 1;
5511 /* Determine the min/max j-zone shift wrt the i-zone */
5512 izone
->shift0
[dim
] = 1;
5513 izone
->shift1
[dim
] = -1;
5514 for (j
= izone
->j0
; j
< izone
->j1
; j
++)
5516 int shift_diff
= zones
->shift
[j
][dim
] - zones
->shift
[i
][dim
];
5517 if (shift_diff
< izone
->shift0
[dim
])
5519 izone
->shift0
[dim
] = shift_diff
;
5521 if (shift_diff
> izone
->shift1
[dim
])
5523 izone
->shift1
[dim
] = shift_diff
;
5530 if (dd
->comm
->dlbState
!= edlbsOffForever
)
5532 snew(dd
->comm
->root
, dd
->ndim
);
5535 if (dd
->comm
->bRecordLoad
)
5537 make_load_communicators(dd
);
5541 static void make_pp_communicator(FILE *fplog
,
5543 t_commrec gmx_unused
*cr
,
5544 int gmx_unused reorder
)
5547 gmx_domdec_comm_t
*comm
;
5554 if (comm
->bCartesianPP
)
5556 /* Set up cartesian communication for the particle-particle part */
5559 fprintf(fplog
, "Will use a Cartesian communicator: %d x %d x %d\n",
5560 dd
->nc
[XX
], dd
->nc
[YY
], dd
->nc
[ZZ
]);
5563 for (int i
= 0; i
< DIM
; i
++)
5567 MPI_Cart_create(cr
->mpi_comm_mygroup
, DIM
, dd
->nc
, periods
, reorder
,
5569 /* We overwrite the old communicator with the new cartesian one */
5570 cr
->mpi_comm_mygroup
= comm_cart
;
5573 dd
->mpi_comm_all
= cr
->mpi_comm_mygroup
;
5574 MPI_Comm_rank(dd
->mpi_comm_all
, &dd
->rank
);
5576 if (comm
->bCartesianPP_PME
)
5578 /* Since we want to use the original cartesian setup for sim,
5579 * and not the one after split, we need to make an index.
5581 snew(comm
->ddindex2ddnodeid
, dd
->nnodes
);
5582 comm
->ddindex2ddnodeid
[dd_index(dd
->nc
, dd
->ci
)] = dd
->rank
;
5583 gmx_sumi(dd
->nnodes
, comm
->ddindex2ddnodeid
, cr
);
5584 /* Get the rank of the DD master,
5585 * above we made sure that the master node is a PP node.
5595 MPI_Allreduce(&rank
, &dd
->masterrank
, 1, MPI_INT
, MPI_SUM
, dd
->mpi_comm_all
);
5597 else if (comm
->bCartesianPP
)
5599 if (cr
->npmenodes
== 0)
5601 /* The PP communicator is also
5602 * the communicator for this simulation
5604 cr
->mpi_comm_mysim
= cr
->mpi_comm_mygroup
;
5606 cr
->nodeid
= dd
->rank
;
5608 MPI_Cart_coords(dd
->mpi_comm_all
, dd
->rank
, DIM
, dd
->ci
);
5610 /* We need to make an index to go from the coordinates
5611 * to the nodeid of this simulation.
5613 snew(comm
->ddindex2simnodeid
, dd
->nnodes
);
5614 snew(buf
, dd
->nnodes
);
5615 if (cr
->duty
& DUTY_PP
)
5617 buf
[dd_index(dd
->nc
, dd
->ci
)] = cr
->sim_nodeid
;
5619 /* Communicate the ddindex to simulation nodeid index */
5620 MPI_Allreduce(buf
, comm
->ddindex2simnodeid
, dd
->nnodes
, MPI_INT
, MPI_SUM
,
5621 cr
->mpi_comm_mysim
);
5624 /* Determine the master coordinates and rank.
5625 * The DD master should be the same node as the master of this sim.
5627 for (int i
= 0; i
< dd
->nnodes
; i
++)
5629 if (comm
->ddindex2simnodeid
[i
] == 0)
5631 ddindex2xyz(dd
->nc
, i
, dd
->master_ci
);
5632 MPI_Cart_rank(dd
->mpi_comm_all
, dd
->master_ci
, &dd
->masterrank
);
5637 fprintf(debug
, "The master rank is %d\n", dd
->masterrank
);
5642 /* No Cartesian communicators */
5643 /* We use the rank in dd->comm->all as DD index */
5644 ddindex2xyz(dd
->nc
, dd
->rank
, dd
->ci
);
5645 /* The simulation master nodeid is 0, so the DD master rank is also 0 */
5647 clear_ivec(dd
->master_ci
);
5654 "Domain decomposition rank %d, coordinates %d %d %d\n\n",
5655 dd
->rank
, dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
5660 "Domain decomposition rank %d, coordinates %d %d %d\n\n",
5661 dd
->rank
, dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
5665 static void receive_ddindex2simnodeid(gmx_domdec_t
*dd
,
5669 gmx_domdec_comm_t
*comm
= dd
->comm
;
5671 if (!comm
->bCartesianPP_PME
&& comm
->bCartesianPP
)
5674 snew(comm
->ddindex2simnodeid
, dd
->nnodes
);
5675 snew(buf
, dd
->nnodes
);
5676 if (cr
->duty
& DUTY_PP
)
5678 buf
[dd_index(dd
->nc
, dd
->ci
)] = cr
->sim_nodeid
;
5680 /* Communicate the ddindex to simulation nodeid index */
5681 MPI_Allreduce(buf
, comm
->ddindex2simnodeid
, dd
->nnodes
, MPI_INT
, MPI_SUM
,
5682 cr
->mpi_comm_mysim
);
5686 GMX_UNUSED_VALUE(dd
);
5687 GMX_UNUSED_VALUE(cr
);
5691 static gmx_domdec_master_t
*init_gmx_domdec_master_t(gmx_domdec_t
*dd
,
5692 int ncg
, int natoms
)
5694 gmx_domdec_master_t
*ma
;
5699 snew(ma
->ncg
, dd
->nnodes
);
5700 snew(ma
->index
, dd
->nnodes
+1);
5702 snew(ma
->nat
, dd
->nnodes
);
5703 snew(ma
->ibuf
, dd
->nnodes
*2);
5704 snew(ma
->cell_x
, DIM
);
5705 for (i
= 0; i
< DIM
; i
++)
5707 snew(ma
->cell_x
[i
], dd
->nc
[i
]+1);
5710 if (dd
->nnodes
<= GMX_DD_NNODES_SENDRECV
)
5716 snew(ma
->vbuf
, natoms
);
5722 static void split_communicator(FILE *fplog
, t_commrec
*cr
, gmx_domdec_t
*dd
,
5723 int gmx_unused dd_rank_order
,
5724 int gmx_unused reorder
)
5726 gmx_domdec_comm_t
*comm
;
5735 if (comm
->bCartesianPP
)
5737 for (i
= 1; i
< DIM
; i
++)
5739 bDiv
[i
] = ((cr
->npmenodes
*dd
->nc
[i
]) % (dd
->nnodes
) == 0);
5741 if (bDiv
[YY
] || bDiv
[ZZ
])
5743 comm
->bCartesianPP_PME
= TRUE
;
5744 /* If we have 2D PME decomposition, which is always in x+y,
5745 * we stack the PME only nodes in z.
5746 * Otherwise we choose the direction that provides the thinnest slab
5747 * of PME only nodes as this will have the least effect
5748 * on the PP communication.
5749 * But for the PME communication the opposite might be better.
5751 if (bDiv
[ZZ
] && (comm
->npmenodes_y
> 1 ||
5753 dd
->nc
[YY
] > dd
->nc
[ZZ
]))
5755 comm
->cartpmedim
= ZZ
;
5759 comm
->cartpmedim
= YY
;
5761 comm
->ntot
[comm
->cartpmedim
]
5762 += (cr
->npmenodes
*dd
->nc
[comm
->cartpmedim
])/dd
->nnodes
;
5766 fprintf(fplog
, "Number of PME-only ranks (%d) is not a multiple of nx*ny (%d*%d) or nx*nz (%d*%d)\n", cr
->npmenodes
, dd
->nc
[XX
], dd
->nc
[YY
], dd
->nc
[XX
], dd
->nc
[ZZ
]);
5768 "Will not use a Cartesian communicator for PP <-> PME\n\n");
5773 if (comm
->bCartesianPP_PME
)
5780 fprintf(fplog
, "Will use a Cartesian communicator for PP <-> PME: %d x %d x %d\n", comm
->ntot
[XX
], comm
->ntot
[YY
], comm
->ntot
[ZZ
]);
5783 for (i
= 0; i
< DIM
; i
++)
5787 MPI_Cart_create(cr
->mpi_comm_mysim
, DIM
, comm
->ntot
, periods
, reorder
,
5789 MPI_Comm_rank(comm_cart
, &rank
);
5790 if (MASTER(cr
) && rank
!= 0)
5792 gmx_fatal(FARGS
, "MPI rank 0 was renumbered by MPI_Cart_create, we do not allow this");
5795 /* With this assigment we loose the link to the original communicator
5796 * which will usually be MPI_COMM_WORLD, unless have multisim.
5798 cr
->mpi_comm_mysim
= comm_cart
;
5799 cr
->sim_nodeid
= rank
;
5801 MPI_Cart_coords(cr
->mpi_comm_mysim
, cr
->sim_nodeid
, DIM
, dd
->ci
);
5805 fprintf(fplog
, "Cartesian rank %d, coordinates %d %d %d\n\n",
5806 cr
->sim_nodeid
, dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
5809 if (dd
->ci
[comm
->cartpmedim
] < dd
->nc
[comm
->cartpmedim
])
5813 if (cr
->npmenodes
== 0 ||
5814 dd
->ci
[comm
->cartpmedim
] >= dd
->nc
[comm
->cartpmedim
])
5816 cr
->duty
= DUTY_PME
;
5819 /* Split the sim communicator into PP and PME only nodes */
5820 MPI_Comm_split(cr
->mpi_comm_mysim
,
5822 dd_index(comm
->ntot
, dd
->ci
),
5823 &cr
->mpi_comm_mygroup
);
5827 switch (dd_rank_order
)
5829 case ddrankorderPP_PME
:
5832 fprintf(fplog
, "Order of the ranks: PP first, PME last\n");
5835 case ddrankorderINTERLEAVE
:
5836 /* Interleave the PP-only and PME-only ranks */
5839 fprintf(fplog
, "Interleaving PP and PME ranks\n");
5841 comm
->pmenodes
= dd_interleaved_pme_ranks(dd
);
5843 case ddrankorderCARTESIAN
:
5846 gmx_fatal(FARGS
, "Unknown dd_rank_order=%d", dd_rank_order
);
5849 if (dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
) == -1)
5851 cr
->duty
= DUTY_PME
;
5858 /* Split the sim communicator into PP and PME only nodes */
5859 MPI_Comm_split(cr
->mpi_comm_mysim
,
5862 &cr
->mpi_comm_mygroup
);
5863 MPI_Comm_rank(cr
->mpi_comm_mygroup
, &cr
->nodeid
);
5869 fprintf(fplog
, "This rank does only %s work.\n\n",
5870 (cr
->duty
& DUTY_PP
) ? "particle-particle" : "PME-mesh");
5874 /*! \brief Generates the MPI communicators for domain decomposition */
5875 static void make_dd_communicators(FILE *fplog
, t_commrec
*cr
,
5876 gmx_domdec_t
*dd
, int dd_rank_order
)
5878 gmx_domdec_comm_t
*comm
;
5883 copy_ivec(dd
->nc
, comm
->ntot
);
5885 comm
->bCartesianPP
= (dd_rank_order
== ddrankorderCARTESIAN
);
5886 comm
->bCartesianPP_PME
= FALSE
;
5888 /* Reorder the nodes by default. This might change the MPI ranks.
5889 * Real reordering is only supported on very few architectures,
5890 * Blue Gene is one of them.
5892 CartReorder
= (getenv("GMX_NO_CART_REORDER") == nullptr);
5894 if (cr
->npmenodes
> 0)
5896 /* Split the communicator into a PP and PME part */
5897 split_communicator(fplog
, cr
, dd
, dd_rank_order
, CartReorder
);
5898 if (comm
->bCartesianPP_PME
)
5900 /* We (possibly) reordered the nodes in split_communicator,
5901 * so it is no longer required in make_pp_communicator.
5903 CartReorder
= FALSE
;
5908 /* All nodes do PP and PME */
5910 /* We do not require separate communicators */
5911 cr
->mpi_comm_mygroup
= cr
->mpi_comm_mysim
;
5915 if (cr
->duty
& DUTY_PP
)
5917 /* Copy or make a new PP communicator */
5918 make_pp_communicator(fplog
, dd
, cr
, CartReorder
);
5922 receive_ddindex2simnodeid(dd
, cr
);
5925 if (!(cr
->duty
& DUTY_PME
))
5927 /* Set up the commnuication to our PME node */
5928 dd
->pme_nodeid
= dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
);
5929 dd
->pme_receive_vir_ener
= receive_vir_ener(dd
, cr
);
5932 fprintf(debug
, "My pme_nodeid %d receive ener %d\n",
5933 dd
->pme_nodeid
, dd
->pme_receive_vir_ener
);
5938 dd
->pme_nodeid
= -1;
5943 dd
->ma
= init_gmx_domdec_master_t(dd
,
5945 comm
->cgs_gl
.index
[comm
->cgs_gl
.nr
]);
5949 static real
*get_slb_frac(FILE *fplog
, const char *dir
, int nc
, const char *size_string
)
5951 real
*slb_frac
, tot
;
5956 if (nc
> 1 && size_string
!= nullptr)
5960 fprintf(fplog
, "Using static load balancing for the %s direction\n",
5965 for (i
= 0; i
< nc
; i
++)
5968 sscanf(size_string
, "%20lf%n", &dbl
, &n
);
5971 gmx_fatal(FARGS
, "Incorrect or not enough DD cell size entries for direction %s: '%s'", dir
, size_string
);
5980 fprintf(fplog
, "Relative cell sizes:");
5982 for (i
= 0; i
< nc
; i
++)
5987 fprintf(fplog
, " %5.3f", slb_frac
[i
]);
5992 fprintf(fplog
, "\n");
5999 static int multi_body_bondeds_count(const gmx_mtop_t
*mtop
)
6002 gmx_mtop_ilistloop_t iloop
;
6006 iloop
= gmx_mtop_ilistloop_init(mtop
);
6007 while (gmx_mtop_ilistloop_next(iloop
, &il
, &nmol
))
6009 for (ftype
= 0; ftype
< F_NRE
; ftype
++)
6011 if ((interaction_function
[ftype
].flags
& IF_BOND
) &&
6014 n
+= nmol
*il
[ftype
].nr
/(1 + NRAL(ftype
));
6022 static int dd_getenv(FILE *fplog
, const char *env_var
, int def
)
6028 val
= getenv(env_var
);
6031 if (sscanf(val
, "%20d", &nst
) <= 0)
6037 fprintf(fplog
, "Found env.var. %s = %s, using value %d\n",
6045 static void dd_warning(t_commrec
*cr
, FILE *fplog
, const char *warn_string
)
6049 fprintf(stderr
, "\n%s\n", warn_string
);
6053 fprintf(fplog
, "\n%s\n", warn_string
);
6057 static void check_dd_restrictions(t_commrec
*cr
, const gmx_domdec_t
*dd
,
6058 const t_inputrec
*ir
, FILE *fplog
)
6060 if (ir
->ePBC
== epbcSCREW
&&
6061 (dd
->nc
[XX
] == 1 || dd
->nc
[YY
] > 1 || dd
->nc
[ZZ
] > 1))
6063 gmx_fatal(FARGS
, "With pbc=%s can only do domain decomposition in the x-direction", epbc_names
[ir
->ePBC
]);
6066 if (ir
->ns_type
== ensSIMPLE
)
6068 gmx_fatal(FARGS
, "Domain decomposition does not support simple neighbor searching, use grid searching or run with one MPI rank");
6071 if (ir
->nstlist
== 0)
6073 gmx_fatal(FARGS
, "Domain decomposition does not work with nstlist=0");
6076 if (ir
->comm_mode
== ecmANGULAR
&& ir
->ePBC
!= epbcNONE
)
6078 dd_warning(cr
, fplog
, "comm-mode angular will give incorrect results when the comm group partially crosses a periodic boundary");
6082 static real
average_cellsize_min(gmx_domdec_t
*dd
, gmx_ddbox_t
*ddbox
)
6087 r
= ddbox
->box_size
[XX
];
6088 for (di
= 0; di
< dd
->ndim
; di
++)
6091 /* Check using the initial average cell size */
6092 r
= std::min(r
, ddbox
->box_size
[d
]*ddbox
->skew_fac
[d
]/dd
->nc
[d
]);
6098 static int check_dlb_support(FILE *fplog
, t_commrec
*cr
,
6099 const char *dlb_opt
, gmx_bool bRecordLoad
,
6100 unsigned long Flags
, const t_inputrec
*ir
)
6107 case 'a': dlbState
= edlbsOffCanTurnOn
; break;
6108 case 'n': dlbState
= edlbsOffForever
; break;
6109 case 'y': dlbState
= edlbsOnForever
; break;
6110 default: gmx_incons("Unknown dlb_opt");
6113 if (Flags
& MD_RERUN
)
6115 return edlbsOffForever
;
6118 if (!EI_DYNAMICS(ir
->eI
))
6120 if (dlbState
== edlbsOnForever
)
6122 sprintf(buf
, "NOTE: dynamic load balancing is only supported with dynamics, not with integrator '%s'\n", EI(ir
->eI
));
6123 dd_warning(cr
, fplog
, buf
);
6126 return edlbsOffForever
;
6131 dd_warning(cr
, fplog
, "NOTE: Cycle counters unsupported or not enabled in kernel. Cannot use dynamic load balancing.\n");
6132 return edlbsOffForever
;
6135 if (Flags
& MD_REPRODUCIBLE
)
6139 case edlbsOffForever
:
6141 case edlbsOffCanTurnOn
:
6142 case edlbsOnCanTurnOff
:
6143 dd_warning(cr
, fplog
, "NOTE: reproducibility requested, will not use dynamic load balancing\n");
6144 dlbState
= edlbsOffForever
;
6146 case edlbsOnForever
:
6147 dd_warning(cr
, fplog
, "WARNING: reproducibility requested with dynamic load balancing, the simulation will NOT be binary reproducible\n");
6150 gmx_fatal(FARGS
, "Death horror: undefined case (%d) for load balancing choice", dlbState
);
6158 static void set_dd_dim(FILE *fplog
, gmx_domdec_t
*dd
)
6163 if (getenv("GMX_DD_ORDER_ZYX") != nullptr)
6165 /* Decomposition order z,y,x */
6168 fprintf(fplog
, "Using domain decomposition order z, y, x\n");
6170 for (dim
= DIM
-1; dim
>= 0; dim
--)
6172 if (dd
->nc
[dim
] > 1)
6174 dd
->dim
[dd
->ndim
++] = dim
;
6180 /* Decomposition order x,y,z */
6181 for (dim
= 0; dim
< DIM
; dim
++)
6183 if (dd
->nc
[dim
] > 1)
6185 dd
->dim
[dd
->ndim
++] = dim
;
6191 static gmx_domdec_comm_t
*init_dd_comm()
6193 gmx_domdec_comm_t
*comm
;
6197 snew(comm
->cggl_flag
, DIM
*2);
6198 snew(comm
->cgcm_state
, DIM
*2);
6199 for (i
= 0; i
< DIM
*2; i
++)
6201 comm
->cggl_flag_nalloc
[i
] = 0;
6202 comm
->cgcm_state_nalloc
[i
] = 0;
6205 comm
->nalloc_int
= 0;
6206 comm
->buf_int
= nullptr;
6208 vec_rvec_init(&comm
->vbuf
);
6210 comm
->n_load_have
= 0;
6211 comm
->n_load_collect
= 0;
6213 for (i
= 0; i
< ddnatNR
-ddnatZONE
; i
++)
6215 comm
->sum_nat
[i
] = 0;
6219 comm
->load_step
= 0;
6222 clear_ivec(comm
->load_lim
);
6229 /*! \brief Set the cell size and interaction limits, as well as the DD grid */
6230 static void set_dd_limits_and_grid(FILE *fplog
, t_commrec
*cr
, gmx_domdec_t
*dd
,
6231 unsigned long Flags
,
6232 ivec nc
, int nPmeRanks
,
6233 real comm_distance_min
, real rconstr
,
6234 const char *dlb_opt
, real dlb_scale
,
6235 const char *sizex
, const char *sizey
, const char *sizez
,
6236 const gmx_mtop_t
*mtop
,
6237 const t_inputrec
*ir
,
6238 matrix box
, const rvec
*x
,
6240 int *npme_x
, int *npme_y
)
6243 real r_bonded_limit
= -1;
6244 const real tenPercentMargin
= 1.1;
6245 gmx_domdec_comm_t
*comm
= dd
->comm
;
6247 snew(comm
->cggl_flag
, DIM
*2);
6248 snew(comm
->cgcm_state
, DIM
*2);
6250 dd
->npbcdim
= ePBC2npbcdim(ir
->ePBC
);
6251 dd
->bScrewPBC
= (ir
->ePBC
== epbcSCREW
);
6253 dd
->pme_recv_f_alloc
= 0;
6254 dd
->pme_recv_f_buf
= nullptr;
6256 /* Initialize to GPU share count to 0, might change later */
6257 comm
->nrank_gpu_shared
= 0;
6259 comm
->dlbState
= check_dlb_support(fplog
, cr
, dlb_opt
, comm
->bRecordLoad
, Flags
, ir
);
6260 dd_dlb_set_should_check_whether_to_turn_dlb_on(dd
, TRUE
);
6261 /* To consider turning DLB on after 2*nstlist steps we need to check
6262 * at partitioning count 3. Thus we need to increase the first count by 2.
6264 comm
->ddPartioningCountFirstDlbOff
+= 2;
6268 fprintf(fplog
, "Dynamic load balancing: %s\n",
6269 edlbs_names
[comm
->dlbState
]);
6271 comm
->bPMELoadBalDLBLimits
= FALSE
;
6273 /* Allocate the charge group/atom sorting struct */
6274 snew(comm
->sort
, 1);
6276 comm
->bCGs
= (ncg_mtop(mtop
) < mtop
->natoms
);
6278 comm
->bInterCGBondeds
= ((ncg_mtop(mtop
) > mtop
->mols
.nr
) ||
6279 mtop
->bIntermolecularInteractions
);
6280 if (comm
->bInterCGBondeds
)
6282 comm
->bInterCGMultiBody
= (multi_body_bondeds_count(mtop
) > 0);
6286 comm
->bInterCGMultiBody
= FALSE
;
6289 dd
->bInterCGcons
= inter_charge_group_constraints(mtop
);
6290 dd
->bInterCGsettles
= inter_charge_group_settles(mtop
);
6294 /* Set the cut-off to some very large value,
6295 * so we don't need if statements everywhere in the code.
6296 * We use sqrt, since the cut-off is squared in some places.
6298 comm
->cutoff
= GMX_CUTOFF_INF
;
6302 comm
->cutoff
= ir
->rlist
;
6304 comm
->cutoff_mbody
= 0;
6306 comm
->cellsize_limit
= 0;
6307 comm
->bBondComm
= FALSE
;
6309 /* Atoms should be able to move by up to half the list buffer size (if > 0)
6310 * within nstlist steps. Since boundaries are allowed to displace by half
6311 * a cell size, DD cells should be at least the size of the list buffer.
6313 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
,
6314 ir
->rlist
- std::max(ir
->rvdw
, ir
->rcoulomb
));
6316 if (comm
->bInterCGBondeds
)
6318 if (comm_distance_min
> 0)
6320 comm
->cutoff_mbody
= comm_distance_min
;
6321 if (Flags
& MD_DDBONDCOMM
)
6323 comm
->bBondComm
= (comm
->cutoff_mbody
> comm
->cutoff
);
6327 comm
->cutoff
= std::max(comm
->cutoff
, comm
->cutoff_mbody
);
6329 r_bonded_limit
= comm
->cutoff_mbody
;
6331 else if (ir
->bPeriodicMols
)
6333 /* Can not easily determine the required cut-off */
6334 dd_warning(cr
, fplog
, "NOTE: Periodic molecules are present in this system. Because of this, the domain decomposition algorithm cannot easily determine the minimum cell size that it requires for treating bonded interactions. Instead, domain decomposition will assume that half the non-bonded cut-off will be a suitable lower bound.\n");
6335 comm
->cutoff_mbody
= comm
->cutoff
/2;
6336 r_bonded_limit
= comm
->cutoff_mbody
;
6344 dd_bonded_cg_distance(fplog
, mtop
, ir
, x
, box
,
6345 Flags
& MD_DDBONDCHECK
, &r_2b
, &r_mb
);
6347 gmx_bcast(sizeof(r_2b
), &r_2b
, cr
);
6348 gmx_bcast(sizeof(r_mb
), &r_mb
, cr
);
6350 /* We use an initial margin of 10% for the minimum cell size,
6351 * except when we are just below the non-bonded cut-off.
6353 if (Flags
& MD_DDBONDCOMM
)
6355 if (std::max(r_2b
, r_mb
) > comm
->cutoff
)
6357 r_bonded
= std::max(r_2b
, r_mb
);
6358 r_bonded_limit
= tenPercentMargin
*r_bonded
;
6359 comm
->bBondComm
= TRUE
;
6364 r_bonded_limit
= std::min(tenPercentMargin
*r_bonded
, comm
->cutoff
);
6366 /* We determine cutoff_mbody later */
6370 /* No special bonded communication,
6371 * simply increase the DD cut-off.
6373 r_bonded_limit
= tenPercentMargin
*std::max(r_2b
, r_mb
);
6374 comm
->cutoff_mbody
= r_bonded_limit
;
6375 comm
->cutoff
= std::max(comm
->cutoff
, comm
->cutoff_mbody
);
6381 "Minimum cell size due to bonded interactions: %.3f nm\n",
6384 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
, r_bonded_limit
);
6387 if (dd
->bInterCGcons
&& rconstr
<= 0)
6389 /* There is a cell size limit due to the constraints (P-LINCS) */
6390 rconstr
= constr_r_max(fplog
, mtop
, ir
);
6394 "Estimated maximum distance required for P-LINCS: %.3f nm\n",
6396 if (rconstr
> comm
->cellsize_limit
)
6398 fprintf(fplog
, "This distance will limit the DD cell size, you can override this with -rcon\n");
6402 else if (rconstr
> 0 && fplog
)
6404 /* Here we do not check for dd->bInterCGcons,
6405 * because one can also set a cell size limit for virtual sites only
6406 * and at this point we don't know yet if there are intercg v-sites.
6409 "User supplied maximum distance required for P-LINCS: %.3f nm\n",
6412 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
, rconstr
);
6414 comm
->cgs_gl
= gmx_mtop_global_cgs(mtop
);
6418 copy_ivec(nc
, dd
->nc
);
6419 set_dd_dim(fplog
, dd
);
6420 set_ddbox_cr(cr
, &dd
->nc
, ir
, box
, &comm
->cgs_gl
, x
, ddbox
);
6424 cr
->npmenodes
= nPmeRanks
;
6428 /* When the DD grid is set explicitly and -npme is set to auto,
6429 * don't use PME ranks. We check later if the DD grid is
6430 * compatible with the total number of ranks.
6435 real acs
= average_cellsize_min(dd
, ddbox
);
6436 if (acs
< comm
->cellsize_limit
)
6440 fprintf(fplog
, "ERROR: The initial cell size (%f) is smaller than the cell size limit (%f)\n", acs
, comm
->cellsize_limit
);
6442 gmx_fatal_collective(FARGS
, cr
->mpi_comm_mysim
, MASTER(cr
),
6443 "The initial cell size (%f) is smaller than the cell size limit (%f), change options -dd, -rdd or -rcon, see the log file for details",
6444 acs
, comm
->cellsize_limit
);
6449 set_ddbox_cr(cr
, nullptr, ir
, box
, &comm
->cgs_gl
, x
, ddbox
);
6451 /* We need to choose the optimal DD grid and possibly PME nodes */
6453 dd_choose_grid(fplog
, cr
, dd
, ir
, mtop
, box
, ddbox
,
6455 comm
->dlbState
!= edlbsOffForever
, dlb_scale
,
6456 comm
->cellsize_limit
, comm
->cutoff
,
6457 comm
->bInterCGBondeds
);
6459 if (dd
->nc
[XX
] == 0)
6462 gmx_bool bC
= (dd
->bInterCGcons
&& rconstr
> r_bonded_limit
);
6463 sprintf(buf
, "Change the number of ranks or mdrun option %s%s%s",
6464 !bC
? "-rdd" : "-rcon",
6465 comm
->dlbState
!= edlbsOffForever
? " or -dds" : "",
6466 bC
? " or your LINCS settings" : "");
6468 gmx_fatal_collective(FARGS
, cr
->mpi_comm_mysim
, MASTER(cr
),
6469 "There is no domain decomposition for %d ranks that is compatible with the given box and a minimum cell size of %g nm\n"
6471 "Look in the log file for details on the domain decomposition",
6472 cr
->nnodes
-cr
->npmenodes
, limit
, buf
);
6474 set_dd_dim(fplog
, dd
);
6480 "Domain decomposition grid %d x %d x %d, separate PME ranks %d\n",
6481 dd
->nc
[XX
], dd
->nc
[YY
], dd
->nc
[ZZ
], cr
->npmenodes
);
6484 dd
->nnodes
= dd
->nc
[XX
]*dd
->nc
[YY
]*dd
->nc
[ZZ
];
6485 if (cr
->nnodes
- dd
->nnodes
!= cr
->npmenodes
)
6487 gmx_fatal_collective(FARGS
, cr
->mpi_comm_mysim
, MASTER(cr
),
6488 "The size of the domain decomposition grid (%d) does not match the number of ranks (%d). The total number of ranks is %d",
6489 dd
->nnodes
, cr
->nnodes
- cr
->npmenodes
, cr
->nnodes
);
6491 if (cr
->npmenodes
> dd
->nnodes
)
6493 gmx_fatal_collective(FARGS
, cr
->mpi_comm_mysim
, MASTER(cr
),
6494 "The number of separate PME ranks (%d) is larger than the number of PP ranks (%d), this is not supported.", cr
->npmenodes
, dd
->nnodes
);
6496 if (cr
->npmenodes
> 0)
6498 comm
->npmenodes
= cr
->npmenodes
;
6502 comm
->npmenodes
= dd
->nnodes
;
6505 if (EEL_PME(ir
->coulombtype
) || EVDW_PME(ir
->vdwtype
))
6507 /* The following choices should match those
6508 * in comm_cost_est in domdec_setup.c.
6509 * Note that here the checks have to take into account
6510 * that the decomposition might occur in a different order than xyz
6511 * (for instance through the env.var. GMX_DD_ORDER_ZYX),
6512 * in which case they will not match those in comm_cost_est,
6513 * but since that is mainly for testing purposes that's fine.
6515 if (dd
->ndim
>= 2 && dd
->dim
[0] == XX
&& dd
->dim
[1] == YY
&&
6516 comm
->npmenodes
> dd
->nc
[XX
] && comm
->npmenodes
% dd
->nc
[XX
] == 0 &&
6517 getenv("GMX_PMEONEDD") == nullptr)
6519 comm
->npmedecompdim
= 2;
6520 comm
->npmenodes_x
= dd
->nc
[XX
];
6521 comm
->npmenodes_y
= comm
->npmenodes
/comm
->npmenodes_x
;
6525 /* In case nc is 1 in both x and y we could still choose to
6526 * decompose pme in y instead of x, but we use x for simplicity.
6528 comm
->npmedecompdim
= 1;
6529 if (dd
->dim
[0] == YY
)
6531 comm
->npmenodes_x
= 1;
6532 comm
->npmenodes_y
= comm
->npmenodes
;
6536 comm
->npmenodes_x
= comm
->npmenodes
;
6537 comm
->npmenodes_y
= 1;
6542 fprintf(fplog
, "PME domain decomposition: %d x %d x %d\n",
6543 comm
->npmenodes_x
, comm
->npmenodes_y
, 1);
6548 comm
->npmedecompdim
= 0;
6549 comm
->npmenodes_x
= 0;
6550 comm
->npmenodes_y
= 0;
6553 /* Technically we don't need both of these,
6554 * but it simplifies code not having to recalculate it.
6556 *npme_x
= comm
->npmenodes_x
;
6557 *npme_y
= comm
->npmenodes_y
;
6559 snew(comm
->slb_frac
, DIM
);
6560 if (comm
->dlbState
== edlbsOffForever
)
6562 comm
->slb_frac
[XX
] = get_slb_frac(fplog
, "x", dd
->nc
[XX
], sizex
);
6563 comm
->slb_frac
[YY
] = get_slb_frac(fplog
, "y", dd
->nc
[YY
], sizey
);
6564 comm
->slb_frac
[ZZ
] = get_slb_frac(fplog
, "z", dd
->nc
[ZZ
], sizez
);
6567 if (comm
->bInterCGBondeds
&& comm
->cutoff_mbody
== 0)
6569 if (comm
->bBondComm
|| comm
->dlbState
!= edlbsOffForever
)
6571 /* Set the bonded communication distance to halfway
6572 * the minimum and the maximum,
6573 * since the extra communication cost is nearly zero.
6575 real acs
= average_cellsize_min(dd
, ddbox
);
6576 comm
->cutoff_mbody
= 0.5*(r_bonded
+ acs
);
6577 if (comm
->dlbState
!= edlbsOffForever
)
6579 /* Check if this does not limit the scaling */
6580 comm
->cutoff_mbody
= std::min(comm
->cutoff_mbody
, dlb_scale
*acs
);
6582 if (!comm
->bBondComm
)
6584 /* Without bBondComm do not go beyond the n.b. cut-off */
6585 comm
->cutoff_mbody
= std::min(comm
->cutoff_mbody
, comm
->cutoff
);
6586 if (comm
->cellsize_limit
>= comm
->cutoff
)
6588 /* We don't loose a lot of efficieny
6589 * when increasing it to the n.b. cut-off.
6590 * It can even be slightly faster, because we need
6591 * less checks for the communication setup.
6593 comm
->cutoff_mbody
= comm
->cutoff
;
6596 /* Check if we did not end up below our original limit */
6597 comm
->cutoff_mbody
= std::max(comm
->cutoff_mbody
, r_bonded_limit
);
6599 if (comm
->cutoff_mbody
> comm
->cellsize_limit
)
6601 comm
->cellsize_limit
= comm
->cutoff_mbody
;
6604 /* Without DLB and cutoff_mbody<cutoff, cutoff_mbody is dynamic */
6609 fprintf(debug
, "Bonded atom communication beyond the cut-off: %d\n"
6610 "cellsize limit %f\n",
6611 comm
->bBondComm
, comm
->cellsize_limit
);
6616 check_dd_restrictions(cr
, dd
, ir
, fplog
);
6620 static void set_dlb_limits(gmx_domdec_t
*dd
)
6625 for (d
= 0; d
< dd
->ndim
; d
++)
6627 dd
->comm
->cd
[d
].np
= dd
->comm
->cd
[d
].np_dlb
;
6628 dd
->comm
->cellsize_min
[dd
->dim
[d
]] =
6629 dd
->comm
->cellsize_min_dlb
[dd
->dim
[d
]];
6634 static void turn_on_dlb(FILE *fplog
, t_commrec
*cr
, gmx_int64_t step
)
6637 gmx_domdec_comm_t
*comm
;
6644 cellsize_min
= comm
->cellsize_min
[dd
->dim
[0]];
6645 for (d
= 1; d
< dd
->ndim
; d
++)
6647 cellsize_min
= std::min(cellsize_min
, comm
->cellsize_min
[dd
->dim
[d
]]);
6650 if (cellsize_min
< comm
->cellsize_limit
*1.05)
6653 sprintf(buf
, "step %" GMX_PRId64
" Measured %.1f %% performance load due to load imbalance, but the minimum cell size is smaller than 1.05 times the cell size limit. Will no longer try dynamic load balancing.\n", step
, dd_force_imb_perf_loss(dd
)*100);
6655 /* Change DLB from "auto" to "no". */
6656 comm
->dlbState
= edlbsOffForever
;
6662 sprintf(buf
, "step %" GMX_PRId64
" Turning on dynamic load balancing, because the performance loss due to load imbalance is %.1f %%.\n", step
, dd_force_imb_perf_loss(dd
)*100);
6663 dd_warning(cr
, fplog
, buf
);
6664 comm
->dlbState
= edlbsOnCanTurnOff
;
6666 /* Store the non-DLB performance, so we can check if DLB actually
6667 * improves performance.
6669 GMX_RELEASE_ASSERT(comm
->cycl_n
[ddCyclStep
] > 0, "When we turned on DLB, we should have measured cycles");
6670 comm
->cyclesPerStepBeforeDLB
= comm
->cycl
[ddCyclStep
]/comm
->cycl_n
[ddCyclStep
];
6674 /* We can set the required cell size info here,
6675 * so we do not need to communicate this.
6676 * The grid is completely uniform.
6678 for (d
= 0; d
< dd
->ndim
; d
++)
6682 comm
->load
[d
].sum_m
= comm
->load
[d
].sum
;
6684 nc
= dd
->nc
[dd
->dim
[d
]];
6685 for (i
= 0; i
< nc
; i
++)
6687 comm
->root
[d
]->cell_f
[i
] = i
/(real
)nc
;
6690 comm
->root
[d
]->cell_f_max0
[i
] = i
/(real
)nc
;
6691 comm
->root
[d
]->cell_f_min1
[i
] = (i
+1)/(real
)nc
;
6694 comm
->root
[d
]->cell_f
[nc
] = 1.0;
6699 static void turn_off_dlb(FILE *fplog
, t_commrec
*cr
, gmx_int64_t step
)
6701 gmx_domdec_t
*dd
= cr
->dd
;
6704 sprintf(buf
, "step %" GMX_PRId64
" Turning off dynamic load balancing, because it is degrading performance.\n", step
);
6705 dd_warning(cr
, fplog
, buf
);
6706 dd
->comm
->dlbState
= edlbsOffCanTurnOn
;
6707 dd
->comm
->haveTurnedOffDlb
= true;
6708 dd
->comm
->ddPartioningCountFirstDlbOff
= dd
->ddp_count
;
6711 static void turn_off_dlb_forever(FILE *fplog
, t_commrec
*cr
, gmx_int64_t step
)
6713 GMX_RELEASE_ASSERT(cr
->dd
->comm
->dlbState
== edlbsOffCanTurnOn
, "Can only turn off DLB forever when it was in the can-turn-on state");
6715 sprintf(buf
, "step %" GMX_PRId64
" Will no longer try dynamic load balancing, as it degraded performance.\n", step
);
6716 dd_warning(cr
, fplog
, buf
);
6717 cr
->dd
->comm
->dlbState
= edlbsOffForever
;
6720 static char *init_bLocalCG(const gmx_mtop_t
*mtop
)
6725 ncg
= ncg_mtop(mtop
);
6726 snew(bLocalCG
, ncg
);
6727 for (cg
= 0; cg
< ncg
; cg
++)
6729 bLocalCG
[cg
] = FALSE
;
6735 void dd_init_bondeds(FILE *fplog
,
6737 const gmx_mtop_t
*mtop
,
6738 const gmx_vsite_t
*vsite
,
6739 const t_inputrec
*ir
,
6740 gmx_bool bBCheck
, cginfo_mb_t
*cginfo_mb
)
6742 gmx_domdec_comm_t
*comm
;
6744 dd_make_reverse_top(fplog
, dd
, mtop
, vsite
, ir
, bBCheck
);
6748 if (comm
->bBondComm
)
6750 /* Communicate atoms beyond the cut-off for bonded interactions */
6753 comm
->cglink
= make_charge_group_links(mtop
, dd
, cginfo_mb
);
6755 comm
->bLocalCG
= init_bLocalCG(mtop
);
6759 /* Only communicate atoms based on cut-off */
6760 comm
->cglink
= nullptr;
6761 comm
->bLocalCG
= nullptr;
6765 static void print_dd_settings(FILE *fplog
, gmx_domdec_t
*dd
,
6766 const gmx_mtop_t
*mtop
, const t_inputrec
*ir
,
6767 gmx_bool bDynLoadBal
, real dlb_scale
,
6768 const gmx_ddbox_t
*ddbox
)
6770 gmx_domdec_comm_t
*comm
;
6776 if (fplog
== nullptr)
6785 fprintf(fplog
, "The maximum number of communication pulses is:");
6786 for (d
= 0; d
< dd
->ndim
; d
++)
6788 fprintf(fplog
, " %c %d", dim2char(dd
->dim
[d
]), comm
->cd
[d
].np_dlb
);
6790 fprintf(fplog
, "\n");
6791 fprintf(fplog
, "The minimum size for domain decomposition cells is %.3f nm\n", comm
->cellsize_limit
);
6792 fprintf(fplog
, "The requested allowed shrink of DD cells (option -dds) is: %.2f\n", dlb_scale
);
6793 fprintf(fplog
, "The allowed shrink of domain decomposition cells is:");
6794 for (d
= 0; d
< DIM
; d
++)
6798 if (d
>= ddbox
->npbcdim
&& dd
->nc
[d
] == 2)
6805 comm
->cellsize_min_dlb
[d
]/
6806 (ddbox
->box_size
[d
]*ddbox
->skew_fac
[d
]/dd
->nc
[d
]);
6808 fprintf(fplog
, " %c %.2f", dim2char(d
), shrink
);
6811 fprintf(fplog
, "\n");
6815 set_dd_cell_sizes_slb(dd
, ddbox
, setcellsizeslbPULSE_ONLY
, np
);
6816 fprintf(fplog
, "The initial number of communication pulses is:");
6817 for (d
= 0; d
< dd
->ndim
; d
++)
6819 fprintf(fplog
, " %c %d", dim2char(dd
->dim
[d
]), np
[dd
->dim
[d
]]);
6821 fprintf(fplog
, "\n");
6822 fprintf(fplog
, "The initial domain decomposition cell size is:");
6823 for (d
= 0; d
< DIM
; d
++)
6827 fprintf(fplog
, " %c %.2f nm",
6828 dim2char(d
), dd
->comm
->cellsize_min
[d
]);
6831 fprintf(fplog
, "\n\n");
6834 gmx_bool bInterCGVsites
= count_intercg_vsites(mtop
);
6836 if (comm
->bInterCGBondeds
||
6838 dd
->bInterCGcons
|| dd
->bInterCGsettles
)
6840 fprintf(fplog
, "The maximum allowed distance for charge groups involved in interactions is:\n");
6841 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6842 "non-bonded interactions", "", comm
->cutoff
);
6846 limit
= dd
->comm
->cellsize_limit
;
6850 if (dynamic_dd_box(ddbox
, ir
))
6852 fprintf(fplog
, "(the following are initial values, they could change due to box deformation)\n");
6854 limit
= dd
->comm
->cellsize_min
[XX
];
6855 for (d
= 1; d
< DIM
; d
++)
6857 limit
= std::min(limit
, dd
->comm
->cellsize_min
[d
]);
6861 if (comm
->bInterCGBondeds
)
6863 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6864 "two-body bonded interactions", "(-rdd)",
6865 std::max(comm
->cutoff
, comm
->cutoff_mbody
));
6866 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6867 "multi-body bonded interactions", "(-rdd)",
6868 (comm
->bBondComm
|| dlbIsOn(dd
->comm
)) ? comm
->cutoff_mbody
: std::min(comm
->cutoff
, limit
));
6872 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6873 "virtual site constructions", "(-rcon)", limit
);
6875 if (dd
->bInterCGcons
|| dd
->bInterCGsettles
)
6877 sprintf(buf
, "atoms separated by up to %d constraints",
6879 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6880 buf
, "(-rcon)", limit
);
6882 fprintf(fplog
, "\n");
6888 static void set_cell_limits_dlb(gmx_domdec_t
*dd
,
6890 const t_inputrec
*ir
,
6891 const gmx_ddbox_t
*ddbox
)
6893 gmx_domdec_comm_t
*comm
;
6894 int d
, dim
, npulse
, npulse_d_max
, npulse_d
;
6899 bNoCutOff
= (ir
->rvdw
== 0 || ir
->rcoulomb
== 0);
6901 /* Determine the maximum number of comm. pulses in one dimension */
6903 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
, comm
->cutoff_mbody
);
6905 /* Determine the maximum required number of grid pulses */
6906 if (comm
->cellsize_limit
>= comm
->cutoff
)
6908 /* Only a single pulse is required */
6911 else if (!bNoCutOff
&& comm
->cellsize_limit
> 0)
6913 /* We round down slightly here to avoid overhead due to the latency
6914 * of extra communication calls when the cut-off
6915 * would be only slightly longer than the cell size.
6916 * Later cellsize_limit is redetermined,
6917 * so we can not miss interactions due to this rounding.
6919 npulse
= (int)(0.96 + comm
->cutoff
/comm
->cellsize_limit
);
6923 /* There is no cell size limit */
6924 npulse
= std::max(dd
->nc
[XX
]-1, std::max(dd
->nc
[YY
]-1, dd
->nc
[ZZ
]-1));
6927 if (!bNoCutOff
&& npulse
> 1)
6929 /* See if we can do with less pulses, based on dlb_scale */
6931 for (d
= 0; d
< dd
->ndim
; d
++)
6934 npulse_d
= (int)(1 + dd
->nc
[dim
]*comm
->cutoff
6935 /(ddbox
->box_size
[dim
]*ddbox
->skew_fac
[dim
]*dlb_scale
));
6936 npulse_d_max
= std::max(npulse_d_max
, npulse_d
);
6938 npulse
= std::min(npulse
, npulse_d_max
);
6941 /* This env var can override npulse */
6942 d
= dd_getenv(debug
, "GMX_DD_NPULSE", 0);
6949 comm
->bVacDLBNoLimit
= (ir
->ePBC
== epbcNONE
);
6950 for (d
= 0; d
< dd
->ndim
; d
++)
6952 comm
->cd
[d
].np_dlb
= std::min(npulse
, dd
->nc
[dd
->dim
[d
]]-1);
6953 comm
->cd
[d
].np_nalloc
= comm
->cd
[d
].np_dlb
;
6954 snew(comm
->cd
[d
].ind
, comm
->cd
[d
].np_nalloc
);
6955 comm
->maxpulse
= std::max(comm
->maxpulse
, comm
->cd
[d
].np_dlb
);
6956 if (comm
->cd
[d
].np_dlb
< dd
->nc
[dd
->dim
[d
]]-1)
6958 comm
->bVacDLBNoLimit
= FALSE
;
6962 /* cellsize_limit is set for LINCS in init_domain_decomposition */
6963 if (!comm
->bVacDLBNoLimit
)
6965 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
,
6966 comm
->cutoff
/comm
->maxpulse
);
6968 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
, comm
->cutoff_mbody
);
6969 /* Set the minimum cell size for each DD dimension */
6970 for (d
= 0; d
< dd
->ndim
; d
++)
6972 if (comm
->bVacDLBNoLimit
||
6973 comm
->cd
[d
].np_dlb
*comm
->cellsize_limit
>= comm
->cutoff
)
6975 comm
->cellsize_min_dlb
[dd
->dim
[d
]] = comm
->cellsize_limit
;
6979 comm
->cellsize_min_dlb
[dd
->dim
[d
]] =
6980 comm
->cutoff
/comm
->cd
[d
].np_dlb
;
6983 if (comm
->cutoff_mbody
<= 0)
6985 comm
->cutoff_mbody
= std::min(comm
->cutoff
, comm
->cellsize_limit
);
6993 gmx_bool
dd_bonded_molpbc(const gmx_domdec_t
*dd
, int ePBC
)
6995 /* If each molecule is a single charge group
6996 * or we use domain decomposition for each periodic dimension,
6997 * we do not need to take pbc into account for the bonded interactions.
6999 return (ePBC
!= epbcNONE
&& dd
->comm
->bInterCGBondeds
&&
7002 (dd
->nc
[ZZ
] > 1 || ePBC
== epbcXY
)));
7005 /*! \brief Sets grid size limits and PP-PME setup, prints settings to log */
7006 static void set_ddgrid_parameters(FILE *fplog
, gmx_domdec_t
*dd
, real dlb_scale
,
7007 const gmx_mtop_t
*mtop
, const t_inputrec
*ir
,
7008 const gmx_ddbox_t
*ddbox
)
7010 gmx_domdec_comm_t
*comm
;
7016 if (EEL_PME(ir
->coulombtype
) || EVDW_PME(ir
->vdwtype
))
7018 init_ddpme(dd
, &comm
->ddpme
[0], 0);
7019 if (comm
->npmedecompdim
>= 2)
7021 init_ddpme(dd
, &comm
->ddpme
[1], 1);
7026 comm
->npmenodes
= 0;
7027 if (dd
->pme_nodeid
>= 0)
7029 gmx_fatal_collective(FARGS
, dd
->mpi_comm_all
, DDMASTER(dd
),
7030 "Can not have separate PME ranks without PME electrostatics");
7036 fprintf(debug
, "The DD cut-off is %f\n", comm
->cutoff
);
7038 if (comm
->dlbState
!= edlbsOffForever
)
7040 set_cell_limits_dlb(dd
, dlb_scale
, ir
, ddbox
);
7043 print_dd_settings(fplog
, dd
, mtop
, ir
, dlbIsOn(comm
), dlb_scale
, ddbox
);
7044 if (comm
->dlbState
== edlbsOffCanTurnOn
)
7048 fprintf(fplog
, "When dynamic load balancing gets turned on, these settings will change to:\n");
7050 print_dd_settings(fplog
, dd
, mtop
, ir
, TRUE
, dlb_scale
, ddbox
);
7053 if (ir
->ePBC
== epbcNONE
)
7055 vol_frac
= 1 - 1/(double)dd
->nnodes
;
7060 (1 + comm_box_frac(dd
->nc
, comm
->cutoff
, ddbox
))/(double)dd
->nnodes
;
7064 fprintf(debug
, "Volume fraction for all DD zones: %f\n", vol_frac
);
7066 natoms_tot
= comm
->cgs_gl
.index
[comm
->cgs_gl
.nr
];
7068 dd
->ga2la
= ga2la_init(natoms_tot
, static_cast<int>(vol_frac
*natoms_tot
));
7071 /*! \brief Set some important DD parameters that can be modified by env.vars */
7072 static void set_dd_envvar_options(FILE *fplog
, gmx_domdec_t
*dd
, int rank_mysim
)
7074 gmx_domdec_comm_t
*comm
= dd
->comm
;
7076 dd
->bSendRecv2
= dd_getenv(fplog
, "GMX_DD_USE_SENDRECV2", 0);
7077 comm
->dlb_scale_lim
= dd_getenv(fplog
, "GMX_DLB_MAX_BOX_SCALING", 10);
7078 comm
->eFlop
= dd_getenv(fplog
, "GMX_DLB_BASED_ON_FLOPS", 0);
7079 int recload
= dd_getenv(fplog
, "GMX_DD_RECORD_LOAD", 1);
7080 comm
->nstDDDump
= dd_getenv(fplog
, "GMX_DD_NST_DUMP", 0);
7081 comm
->nstDDDumpGrid
= dd_getenv(fplog
, "GMX_DD_NST_DUMP_GRID", 0);
7082 comm
->DD_debug
= dd_getenv(fplog
, "GMX_DD_DEBUG", 0);
7084 if (dd
->bSendRecv2
&& fplog
)
7086 fprintf(fplog
, "Will use two sequential MPI_Sendrecv calls instead of two simultaneous non-blocking MPI_Irecv and MPI_Isend pairs for constraint and vsite communication\n");
7093 fprintf(fplog
, "Will load balance based on FLOP count\n");
7095 if (comm
->eFlop
> 1)
7097 srand(1 + rank_mysim
);
7099 comm
->bRecordLoad
= TRUE
;
7103 comm
->bRecordLoad
= (wallcycle_have_counter() && recload
> 0);
7107 gmx_domdec_t
*init_domain_decomposition(FILE *fplog
, t_commrec
*cr
,
7108 unsigned long Flags
,
7109 ivec nc
, int nPmeRanks
,
7111 real comm_distance_min
, real rconstr
,
7112 const char *dlb_opt
, real dlb_scale
,
7113 const char *sizex
, const char *sizey
, const char *sizez
,
7114 const gmx_mtop_t
*mtop
,
7115 const t_inputrec
*ir
,
7116 matrix box
, rvec
*x
,
7118 int *npme_x
, int *npme_y
)
7125 "\nInitializing Domain Decomposition on %d ranks\n", cr
->nnodes
);
7130 dd
->comm
= init_dd_comm();
7132 set_dd_envvar_options(fplog
, dd
, cr
->nodeid
);
7134 set_dd_limits_and_grid(fplog
, cr
, dd
, Flags
,
7136 comm_distance_min
, rconstr
,
7138 sizex
, sizey
, sizez
,
7144 make_dd_communicators(fplog
, cr
, dd
, dd_rank_order
);
7146 if (cr
->duty
& DUTY_PP
)
7148 set_ddgrid_parameters(fplog
, dd
, dlb_scale
, mtop
, ir
, ddbox
);
7150 setup_neighbor_relations(dd
);
7153 /* Set overallocation to avoid frequent reallocation of arrays */
7154 set_over_alloc_dd(TRUE
);
7156 /* Initialize DD paritioning counters */
7157 dd
->comm
->partition_step
= INT_MIN
;
7160 /* We currently don't know the number of threads yet, we set this later */
7163 clear_dd_cycle_counts(dd
);
7168 static gmx_bool
test_dd_cutoff(t_commrec
*cr
,
7169 t_state
*state
, const t_inputrec
*ir
,
7180 set_ddbox(dd
, FALSE
, cr
, ir
, state
->box
,
7181 TRUE
, &dd
->comm
->cgs_gl
, as_rvec_array(state
->x
.data()), &ddbox
);
7185 for (d
= 0; d
< dd
->ndim
; d
++)
7189 inv_cell_size
= DD_CELL_MARGIN
*dd
->nc
[dim
]/ddbox
.box_size
[dim
];
7190 if (dynamic_dd_box(&ddbox
, ir
))
7192 inv_cell_size
*= DD_PRES_SCALE_MARGIN
;
7195 np
= 1 + (int)(cutoff_req
*inv_cell_size
*ddbox
.skew_fac
[dim
]);
7197 if (dd
->comm
->dlbState
!= edlbsOffForever
&& dim
< ddbox
.npbcdim
&&
7198 dd
->comm
->cd
[d
].np_dlb
> 0)
7200 if (np
> dd
->comm
->cd
[d
].np_dlb
)
7205 /* If a current local cell size is smaller than the requested
7206 * cut-off, we could still fix it, but this gets very complicated.
7207 * Without fixing here, we might actually need more checks.
7209 if ((dd
->comm
->cell_x1
[dim
] - dd
->comm
->cell_x0
[dim
])*ddbox
.skew_fac
[dim
]*dd
->comm
->cd
[d
].np_dlb
< cutoff_req
)
7216 if (dd
->comm
->dlbState
!= edlbsOffForever
)
7218 /* If DLB is not active yet, we don't need to check the grid jumps.
7219 * Actually we shouldn't, because then the grid jump data is not set.
7221 if (dlbIsOn(dd
->comm
) &&
7222 check_grid_jump(0, dd
, cutoff_req
, &ddbox
, FALSE
))
7227 gmx_sumi(1, &LocallyLimited
, cr
);
7229 if (LocallyLimited
> 0)
7238 gmx_bool
change_dd_cutoff(t_commrec
*cr
, t_state
*state
, const t_inputrec
*ir
,
7241 gmx_bool bCutoffAllowed
;
7243 bCutoffAllowed
= test_dd_cutoff(cr
, state
, ir
, cutoff_req
);
7247 cr
->dd
->comm
->cutoff
= cutoff_req
;
7250 return bCutoffAllowed
;
7253 void set_dd_dlb_max_cutoff(t_commrec
*cr
, real cutoff
)
7255 gmx_domdec_comm_t
*comm
;
7257 comm
= cr
->dd
->comm
;
7259 /* Turn on the DLB limiting (might have been on already) */
7260 comm
->bPMELoadBalDLBLimits
= TRUE
;
7262 /* Change the cut-off limit */
7263 comm
->PMELoadBal_max_cutoff
= cutoff
;
7267 fprintf(debug
, "PME load balancing set a limit to the DLB staggering such that a %f cut-off will continue to fit\n",
7268 comm
->PMELoadBal_max_cutoff
);
7272 /* Sets whether we should later check the load imbalance data, so that
7273 * we can trigger dynamic load balancing if enough imbalance has
7276 * Used after PME load balancing unlocks DLB, so that the check
7277 * whether DLB will be useful can happen immediately.
7279 static void dd_dlb_set_should_check_whether_to_turn_dlb_on(gmx_domdec_t
*dd
, gmx_bool bValue
)
7281 if (dd
->comm
->dlbState
== edlbsOffCanTurnOn
)
7283 dd
->comm
->bCheckWhetherToTurnDlbOn
= bValue
;
7287 /* Store the DD partitioning count, so we can ignore cycle counts
7288 * over the next nstlist steps, which are often slower.
7290 dd
->comm
->ddPartioningCountFirstDlbOff
= dd
->ddp_count
;
7295 /* Returns if we should check whether there has been enough load
7296 * imbalance to trigger dynamic load balancing.
7298 static gmx_bool
dd_dlb_get_should_check_whether_to_turn_dlb_on(gmx_domdec_t
*dd
)
7300 if (dd
->comm
->dlbState
!= edlbsOffCanTurnOn
)
7305 if (dd
->ddp_count
<= dd
->comm
->ddPartioningCountFirstDlbOff
)
7307 /* We ignore the first nstlist steps at the start of the run
7308 * or after PME load balancing or after turning DLB off, since
7309 * these often have extra allocation or cache miss overhead.
7314 /* We should check whether we should use DLB directly after
7316 if (dd
->comm
->bCheckWhetherToTurnDlbOn
)
7318 /* This flag was set when the PME load-balancing routines
7319 unlocked DLB, and should now be cleared. */
7320 dd_dlb_set_should_check_whether_to_turn_dlb_on(dd
, FALSE
);
7323 /* We check whether we should use DLB every c_checkTurnDlbOnInterval
7324 * partitionings (we do not do this every partioning, so that we
7325 * avoid excessive communication). */
7326 if (dd
->comm
->n_load_have
% c_checkTurnDlbOnInterval
== c_checkTurnDlbOnInterval
- 1)
7334 gmx_bool
dd_dlb_is_on(const gmx_domdec_t
*dd
)
7336 return dlbIsOn(dd
->comm
);
7339 gmx_bool
dd_dlb_is_locked(const gmx_domdec_t
*dd
)
7341 return (dd
->comm
->dlbState
== edlbsOffTemporarilyLocked
);
7344 void dd_dlb_lock(gmx_domdec_t
*dd
)
7346 /* We can only lock the DLB when it is set to auto, otherwise don't do anything */
7347 if (dd
->comm
->dlbState
== edlbsOffCanTurnOn
)
7349 dd
->comm
->dlbState
= edlbsOffTemporarilyLocked
;
7353 void dd_dlb_unlock(gmx_domdec_t
*dd
)
7355 /* We can only lock the DLB when it is set to auto, otherwise don't do anything */
7356 if (dd
->comm
->dlbState
== edlbsOffTemporarilyLocked
)
7358 dd
->comm
->dlbState
= edlbsOffCanTurnOn
;
7359 dd_dlb_set_should_check_whether_to_turn_dlb_on(dd
, TRUE
);
7363 static void merge_cg_buffers(int ncell
,
7364 gmx_domdec_comm_dim_t
*cd
, int pulse
,
7366 int *index_gl
, int *recv_i
,
7367 rvec
*cg_cm
, rvec
*recv_vr
,
7369 cginfo_mb_t
*cginfo_mb
, int *cginfo
)
7371 gmx_domdec_ind_t
*ind
, *ind_p
;
7372 int p
, cell
, c
, cg
, cg0
, cg1
, cg_gl
, nat
;
7373 int shift
, shift_at
;
7375 ind
= &cd
->ind
[pulse
];
7377 /* First correct the already stored data */
7378 shift
= ind
->nrecv
[ncell
];
7379 for (cell
= ncell
-1; cell
>= 0; cell
--)
7381 shift
-= ind
->nrecv
[cell
];
7384 /* Move the cg's present from previous grid pulses */
7385 cg0
= ncg_cell
[ncell
+cell
];
7386 cg1
= ncg_cell
[ncell
+cell
+1];
7387 cgindex
[cg1
+shift
] = cgindex
[cg1
];
7388 for (cg
= cg1
-1; cg
>= cg0
; cg
--)
7390 index_gl
[cg
+shift
] = index_gl
[cg
];
7391 copy_rvec(cg_cm
[cg
], cg_cm
[cg
+shift
]);
7392 cgindex
[cg
+shift
] = cgindex
[cg
];
7393 cginfo
[cg
+shift
] = cginfo
[cg
];
7395 /* Correct the already stored send indices for the shift */
7396 for (p
= 1; p
<= pulse
; p
++)
7398 ind_p
= &cd
->ind
[p
];
7400 for (c
= 0; c
< cell
; c
++)
7402 cg0
+= ind_p
->nsend
[c
];
7404 cg1
= cg0
+ ind_p
->nsend
[cell
];
7405 for (cg
= cg0
; cg
< cg1
; cg
++)
7407 ind_p
->index
[cg
] += shift
;
7413 /* Merge in the communicated buffers */
7417 for (cell
= 0; cell
< ncell
; cell
++)
7419 cg1
= ncg_cell
[ncell
+cell
+1] + shift
;
7422 /* Correct the old cg indices */
7423 for (cg
= ncg_cell
[ncell
+cell
]; cg
< cg1
; cg
++)
7425 cgindex
[cg
+1] += shift_at
;
7428 for (cg
= 0; cg
< ind
->nrecv
[cell
]; cg
++)
7430 /* Copy this charge group from the buffer */
7431 index_gl
[cg1
] = recv_i
[cg0
];
7432 copy_rvec(recv_vr
[cg0
], cg_cm
[cg1
]);
7433 /* Add it to the cgindex */
7434 cg_gl
= index_gl
[cg1
];
7435 cginfo
[cg1
] = ddcginfo(cginfo_mb
, cg_gl
);
7436 nat
= GET_CGINFO_NATOMS(cginfo
[cg1
]);
7437 cgindex
[cg1
+1] = cgindex
[cg1
] + nat
;
7442 shift
+= ind
->nrecv
[cell
];
7443 ncg_cell
[ncell
+cell
+1] = cg1
;
7447 static void make_cell2at_index(gmx_domdec_comm_dim_t
*cd
,
7448 int nzone
, int cg0
, const int *cgindex
)
7452 /* Store the atom block boundaries for easy copying of communication buffers
7455 for (zone
= 0; zone
< nzone
; zone
++)
7457 for (p
= 0; p
< cd
->np
; p
++)
7459 cd
->ind
[p
].cell2at0
[zone
] = cgindex
[cg
];
7460 cg
+= cd
->ind
[p
].nrecv
[zone
];
7461 cd
->ind
[p
].cell2at1
[zone
] = cgindex
[cg
];
7466 static gmx_bool
missing_link(t_blocka
*link
, int cg_gl
, char *bLocalCG
)
7472 for (i
= link
->index
[cg_gl
]; i
< link
->index
[cg_gl
+1]; i
++)
7474 if (!bLocalCG
[link
->a
[i
]])
7483 /* Domain corners for communication, a maximum of 4 i-zones see a j domain */
7485 real c
[DIM
][4]; /* the corners for the non-bonded communication */
7486 real cr0
; /* corner for rounding */
7487 real cr1
[4]; /* corners for rounding */
7488 real bc
[DIM
]; /* corners for bounded communication */
7489 real bcr1
; /* corner for rounding for bonded communication */
7492 /* Determine the corners of the domain(s) we are communicating with */
7494 set_dd_corners(const gmx_domdec_t
*dd
,
7495 int dim0
, int dim1
, int dim2
,
7499 const gmx_domdec_comm_t
*comm
;
7500 const gmx_domdec_zones_t
*zones
;
7505 zones
= &comm
->zones
;
7507 /* Keep the compiler happy */
7511 /* The first dimension is equal for all cells */
7512 c
->c
[0][0] = comm
->cell_x0
[dim0
];
7515 c
->bc
[0] = c
->c
[0][0];
7520 /* This cell row is only seen from the first row */
7521 c
->c
[1][0] = comm
->cell_x0
[dim1
];
7522 /* All rows can see this row */
7523 c
->c
[1][1] = comm
->cell_x0
[dim1
];
7524 if (dlbIsOn(dd
->comm
))
7526 c
->c
[1][1] = std::max(comm
->cell_x0
[dim1
], comm
->zone_d1
[1].mch0
);
7529 /* For the multi-body distance we need the maximum */
7530 c
->bc
[1] = std::max(comm
->cell_x0
[dim1
], comm
->zone_d1
[1].p1_0
);
7533 /* Set the upper-right corner for rounding */
7534 c
->cr0
= comm
->cell_x1
[dim0
];
7539 for (j
= 0; j
< 4; j
++)
7541 c
->c
[2][j
] = comm
->cell_x0
[dim2
];
7543 if (dlbIsOn(dd
->comm
))
7545 /* Use the maximum of the i-cells that see a j-cell */
7546 for (i
= 0; i
< zones
->nizone
; i
++)
7548 for (j
= zones
->izone
[i
].j0
; j
< zones
->izone
[i
].j1
; j
++)
7553 std::max(c
->c
[2][j
-4],
7554 comm
->zone_d2
[zones
->shift
[i
][dim0
]][zones
->shift
[i
][dim1
]].mch0
);
7560 /* For the multi-body distance we need the maximum */
7561 c
->bc
[2] = comm
->cell_x0
[dim2
];
7562 for (i
= 0; i
< 2; i
++)
7564 for (j
= 0; j
< 2; j
++)
7566 c
->bc
[2] = std::max(c
->bc
[2], comm
->zone_d2
[i
][j
].p1_0
);
7572 /* Set the upper-right corner for rounding */
7573 /* Cell (0,0,0) and cell (1,0,0) can see cell 4 (0,1,1)
7574 * Only cell (0,0,0) can see cell 7 (1,1,1)
7576 c
->cr1
[0] = comm
->cell_x1
[dim1
];
7577 c
->cr1
[3] = comm
->cell_x1
[dim1
];
7578 if (dlbIsOn(dd
->comm
))
7580 c
->cr1
[0] = std::max(comm
->cell_x1
[dim1
], comm
->zone_d1
[1].mch1
);
7583 /* For the multi-body distance we need the maximum */
7584 c
->bcr1
= std::max(comm
->cell_x1
[dim1
], comm
->zone_d1
[1].p1_1
);
7591 /* Determine which cg's we need to send in this pulse from this zone */
7593 get_zone_pulse_cgs(gmx_domdec_t
*dd
,
7594 int zonei
, int zone
,
7596 const int *index_gl
,
7598 int dim
, int dim_ind
,
7599 int dim0
, int dim1
, int dim2
,
7600 real r_comm2
, real r_bcomm2
,
7604 real skew_fac2_d
, real skew_fac_01
,
7605 rvec
*v_d
, rvec
*v_0
, rvec
*v_1
,
7606 const dd_corners_t
*c
,
7608 gmx_bool bDistBonded
,
7614 gmx_domdec_ind_t
*ind
,
7615 int **ibuf
, int *ibuf_nalloc
,
7621 gmx_domdec_comm_t
*comm
;
7623 gmx_bool bDistMB_pulse
;
7625 real r2
, rb2
, r
, tric_sh
;
7628 int nsend_z
, nsend
, nat
;
7632 bScrew
= (dd
->bScrewPBC
&& dim
== XX
);
7634 bDistMB_pulse
= (bDistMB
&& bDistBonded
);
7640 for (cg
= cg0
; cg
< cg1
; cg
++)
7644 if (tric_dist
[dim_ind
] == 0)
7646 /* Rectangular direction, easy */
7647 r
= cg_cm
[cg
][dim
] - c
->c
[dim_ind
][zone
];
7654 r
= cg_cm
[cg
][dim
] - c
->bc
[dim_ind
];
7660 /* Rounding gives at most a 16% reduction
7661 * in communicated atoms
7663 if (dim_ind
>= 1 && (zonei
== 1 || zonei
== 2))
7665 r
= cg_cm
[cg
][dim0
] - c
->cr0
;
7666 /* This is the first dimension, so always r >= 0 */
7673 if (dim_ind
== 2 && (zonei
== 2 || zonei
== 3))
7675 r
= cg_cm
[cg
][dim1
] - c
->cr1
[zone
];
7682 r
= cg_cm
[cg
][dim1
] - c
->bcr1
;
7692 /* Triclinic direction, more complicated */
7695 /* Rounding, conservative as the skew_fac multiplication
7696 * will slightly underestimate the distance.
7698 if (dim_ind
>= 1 && (zonei
== 1 || zonei
== 2))
7700 rn
[dim0
] = cg_cm
[cg
][dim0
] - c
->cr0
;
7701 for (i
= dim0
+1; i
< DIM
; i
++)
7703 rn
[dim0
] -= cg_cm
[cg
][i
]*v_0
[i
][dim0
];
7705 r2
= rn
[dim0
]*rn
[dim0
]*sf2_round
[dim0
];
7708 rb
[dim0
] = rn
[dim0
];
7711 /* Take care that the cell planes along dim0 might not
7712 * be orthogonal to those along dim1 and dim2.
7714 for (i
= 1; i
<= dim_ind
; i
++)
7717 if (normal
[dim0
][dimd
] > 0)
7719 rn
[dimd
] -= rn
[dim0
]*normal
[dim0
][dimd
];
7722 rb
[dimd
] -= rb
[dim0
]*normal
[dim0
][dimd
];
7727 if (dim_ind
== 2 && (zonei
== 2 || zonei
== 3))
7729 rn
[dim1
] += cg_cm
[cg
][dim1
] - c
->cr1
[zone
];
7731 for (i
= dim1
+1; i
< DIM
; i
++)
7733 tric_sh
-= cg_cm
[cg
][i
]*v_1
[i
][dim1
];
7735 rn
[dim1
] += tric_sh
;
7738 r2
+= rn
[dim1
]*rn
[dim1
]*sf2_round
[dim1
];
7739 /* Take care of coupling of the distances
7740 * to the planes along dim0 and dim1 through dim2.
7742 r2
-= rn
[dim0
]*rn
[dim1
]*skew_fac_01
;
7743 /* Take care that the cell planes along dim1
7744 * might not be orthogonal to that along dim2.
7746 if (normal
[dim1
][dim2
] > 0)
7748 rn
[dim2
] -= rn
[dim1
]*normal
[dim1
][dim2
];
7754 cg_cm
[cg
][dim1
] - c
->bcr1
+ tric_sh
;
7757 rb2
+= rb
[dim1
]*rb
[dim1
]*sf2_round
[dim1
];
7758 /* Take care of coupling of the distances
7759 * to the planes along dim0 and dim1 through dim2.
7761 rb2
-= rb
[dim0
]*rb
[dim1
]*skew_fac_01
;
7762 /* Take care that the cell planes along dim1
7763 * might not be orthogonal to that along dim2.
7765 if (normal
[dim1
][dim2
] > 0)
7767 rb
[dim2
] -= rb
[dim1
]*normal
[dim1
][dim2
];
7772 /* The distance along the communication direction */
7773 rn
[dim
] += cg_cm
[cg
][dim
] - c
->c
[dim_ind
][zone
];
7775 for (i
= dim
+1; i
< DIM
; i
++)
7777 tric_sh
-= cg_cm
[cg
][i
]*v_d
[i
][dim
];
7782 r2
+= rn
[dim
]*rn
[dim
]*skew_fac2_d
;
7783 /* Take care of coupling of the distances
7784 * to the planes along dim0 and dim1 through dim2.
7786 if (dim_ind
== 1 && zonei
== 1)
7788 r2
-= rn
[dim0
]*rn
[dim
]*skew_fac_01
;
7794 rb
[dim
] += cg_cm
[cg
][dim
] - c
->bc
[dim_ind
] + tric_sh
;
7797 rb2
+= rb
[dim
]*rb
[dim
]*skew_fac2_d
;
7798 /* Take care of coupling of the distances
7799 * to the planes along dim0 and dim1 through dim2.
7801 if (dim_ind
== 1 && zonei
== 1)
7803 rb2
-= rb
[dim0
]*rb
[dim
]*skew_fac_01
;
7811 ((bDistMB
&& rb2
< r_bcomm2
) ||
7812 (bDist2B
&& r2
< r_bcomm2
)) &&
7814 (GET_CGINFO_BOND_INTER(cginfo
[cg
]) &&
7815 missing_link(comm
->cglink
, index_gl
[cg
],
7818 /* Make an index to the local charge groups */
7819 if (nsend
+1 > ind
->nalloc
)
7821 ind
->nalloc
= over_alloc_large(nsend
+1);
7822 srenew(ind
->index
, ind
->nalloc
);
7824 if (nsend
+1 > *ibuf_nalloc
)
7826 *ibuf_nalloc
= over_alloc_large(nsend
+1);
7827 srenew(*ibuf
, *ibuf_nalloc
);
7829 ind
->index
[nsend
] = cg
;
7830 (*ibuf
)[nsend
] = index_gl
[cg
];
7832 vec_rvec_check_alloc(vbuf
, nsend
+1);
7834 if (dd
->ci
[dim
] == 0)
7836 /* Correct cg_cm for pbc */
7837 rvec_add(cg_cm
[cg
], box
[dim
], vbuf
->v
[nsend
]);
7840 vbuf
->v
[nsend
][YY
] = box
[YY
][YY
] - vbuf
->v
[nsend
][YY
];
7841 vbuf
->v
[nsend
][ZZ
] = box
[ZZ
][ZZ
] - vbuf
->v
[nsend
][ZZ
];
7846 copy_rvec(cg_cm
[cg
], vbuf
->v
[nsend
]);
7849 nat
+= cgindex
[cg
+1] - cgindex
[cg
];
7855 *nsend_z_ptr
= nsend_z
;
7858 static void setup_dd_communication(gmx_domdec_t
*dd
,
7859 matrix box
, gmx_ddbox_t
*ddbox
,
7861 t_state
*state
, PaddedRVecVector
*f
)
7863 int dim_ind
, dim
, dim0
, dim1
, dim2
, dimd
, p
, nat_tot
;
7864 int nzone
, nzone_send
, zone
, zonei
, cg0
, cg1
;
7865 int c
, i
, cg
, cg_gl
, nrcg
;
7866 int *zone_cg_range
, pos_cg
, *index_gl
, *cgindex
, *recv_i
;
7867 gmx_domdec_comm_t
*comm
;
7868 gmx_domdec_zones_t
*zones
;
7869 gmx_domdec_comm_dim_t
*cd
;
7870 gmx_domdec_ind_t
*ind
;
7871 cginfo_mb_t
*cginfo_mb
;
7872 gmx_bool bBondComm
, bDist2B
, bDistMB
, bDistBonded
;
7873 real r_comm2
, r_bcomm2
;
7874 dd_corners_t corners
;
7876 rvec
*cg_cm
, *normal
, *v_d
, *v_0
= nullptr, *v_1
= nullptr, *recv_vr
;
7877 real skew_fac2_d
, skew_fac_01
;
7884 fprintf(debug
, "Setting up DD communication\n");
7891 /* Initialize the thread data.
7892 * This can not be done in init_domain_decomposition,
7893 * as the numbers of threads is determined later.
7895 comm
->nth
= gmx_omp_nthreads_get(emntDomdec
);
7898 snew(comm
->dth
, comm
->nth
);
7902 switch (fr
->cutoff_scheme
)
7908 cg_cm
= as_rvec_array(state
->x
.data());
7911 gmx_incons("unimplemented");
7915 for (dim_ind
= 0; dim_ind
< dd
->ndim
; dim_ind
++)
7917 /* Check if we need to use triclinic distances */
7918 tric_dist
[dim_ind
] = 0;
7919 for (i
= 0; i
<= dim_ind
; i
++)
7921 if (ddbox
->tric_dir
[dd
->dim
[i
]])
7923 tric_dist
[dim_ind
] = 1;
7928 bBondComm
= comm
->bBondComm
;
7930 /* Do we need to determine extra distances for multi-body bondeds? */
7931 bDistMB
= (comm
->bInterCGMultiBody
&& dlbIsOn(dd
->comm
) && dd
->ndim
> 1);
7933 /* Do we need to determine extra distances for only two-body bondeds? */
7934 bDist2B
= (bBondComm
&& !bDistMB
);
7936 r_comm2
= gmx::square(comm
->cutoff
);
7937 r_bcomm2
= gmx::square(comm
->cutoff_mbody
);
7941 fprintf(debug
, "bBondComm %d, r_bc %f\n", bBondComm
, std::sqrt(r_bcomm2
));
7944 zones
= &comm
->zones
;
7947 dim1
= (dd
->ndim
>= 2 ? dd
->dim
[1] : -1);
7948 dim2
= (dd
->ndim
>= 3 ? dd
->dim
[2] : -1);
7950 set_dd_corners(dd
, dim0
, dim1
, dim2
, bDistMB
, &corners
);
7952 /* Triclinic stuff */
7953 normal
= ddbox
->normal
;
7957 v_0
= ddbox
->v
[dim0
];
7958 if (ddbox
->tric_dir
[dim0
] && ddbox
->tric_dir
[dim1
])
7960 /* Determine the coupling coefficient for the distances
7961 * to the cell planes along dim0 and dim1 through dim2.
7962 * This is required for correct rounding.
7965 ddbox
->v
[dim0
][dim1
+1][dim0
]*ddbox
->v
[dim1
][dim1
+1][dim1
];
7968 fprintf(debug
, "\nskew_fac_01 %f\n", skew_fac_01
);
7974 v_1
= ddbox
->v
[dim1
];
7977 zone_cg_range
= zones
->cg_range
;
7978 index_gl
= dd
->index_gl
;
7979 cgindex
= dd
->cgindex
;
7980 cginfo_mb
= fr
->cginfo_mb
;
7982 zone_cg_range
[0] = 0;
7983 zone_cg_range
[1] = dd
->ncg_home
;
7984 comm
->zone_ncg1
[0] = dd
->ncg_home
;
7985 pos_cg
= dd
->ncg_home
;
7987 nat_tot
= dd
->nat_home
;
7989 for (dim_ind
= 0; dim_ind
< dd
->ndim
; dim_ind
++)
7991 dim
= dd
->dim
[dim_ind
];
7992 cd
= &comm
->cd
[dim_ind
];
7994 if (dim
>= ddbox
->npbcdim
&& dd
->ci
[dim
] == 0)
7996 /* No pbc in this dimension, the first node should not comm. */
8004 v_d
= ddbox
->v
[dim
];
8005 skew_fac2_d
= gmx::square(ddbox
->skew_fac
[dim
]);
8007 cd
->bInPlace
= TRUE
;
8008 for (p
= 0; p
< cd
->np
; p
++)
8010 /* Only atoms communicated in the first pulse are used
8011 * for multi-body bonded interactions or for bBondComm.
8013 bDistBonded
= ((bDistMB
|| bDist2B
) && p
== 0);
8018 for (zone
= 0; zone
< nzone_send
; zone
++)
8020 if (tric_dist
[dim_ind
] && dim_ind
> 0)
8022 /* Determine slightly more optimized skew_fac's
8024 * This reduces the number of communicated atoms
8025 * by about 10% for 3D DD of rhombic dodecahedra.
8027 for (dimd
= 0; dimd
< dim
; dimd
++)
8029 sf2_round
[dimd
] = 1;
8030 if (ddbox
->tric_dir
[dimd
])
8032 for (i
= dd
->dim
[dimd
]+1; i
< DIM
; i
++)
8034 /* If we are shifted in dimension i
8035 * and the cell plane is tilted forward
8036 * in dimension i, skip this coupling.
8038 if (!(zones
->shift
[nzone
+zone
][i
] &&
8039 ddbox
->v
[dimd
][i
][dimd
] >= 0))
8042 gmx::square(ddbox
->v
[dimd
][i
][dimd
]);
8045 sf2_round
[dimd
] = 1/sf2_round
[dimd
];
8050 zonei
= zone_perm
[dim_ind
][zone
];
8053 /* Here we permutate the zones to obtain a convenient order
8054 * for neighbor searching
8056 cg0
= zone_cg_range
[zonei
];
8057 cg1
= zone_cg_range
[zonei
+1];
8061 /* Look only at the cg's received in the previous grid pulse
8063 cg1
= zone_cg_range
[nzone
+zone
+1];
8064 cg0
= cg1
- cd
->ind
[p
-1].nrecv
[zone
];
8067 #pragma omp parallel for num_threads(comm->nth) schedule(static)
8068 for (th
= 0; th
< comm
->nth
; th
++)
8072 gmx_domdec_ind_t
*ind_p
;
8073 int **ibuf_p
, *ibuf_nalloc_p
;
8075 int *nsend_p
, *nat_p
;
8081 /* Thread 0 writes in the comm buffers */
8083 ibuf_p
= &comm
->buf_int
;
8084 ibuf_nalloc_p
= &comm
->nalloc_int
;
8085 vbuf_p
= &comm
->vbuf
;
8088 nsend_zone_p
= &ind
->nsend
[zone
];
8092 /* Other threads write into temp buffers */
8093 ind_p
= &comm
->dth
[th
].ind
;
8094 ibuf_p
= &comm
->dth
[th
].ibuf
;
8095 ibuf_nalloc_p
= &comm
->dth
[th
].ibuf_nalloc
;
8096 vbuf_p
= &comm
->dth
[th
].vbuf
;
8097 nsend_p
= &comm
->dth
[th
].nsend
;
8098 nat_p
= &comm
->dth
[th
].nat
;
8099 nsend_zone_p
= &comm
->dth
[th
].nsend_zone
;
8101 comm
->dth
[th
].nsend
= 0;
8102 comm
->dth
[th
].nat
= 0;
8103 comm
->dth
[th
].nsend_zone
= 0;
8113 cg0_th
= cg0
+ ((cg1
- cg0
)* th
)/comm
->nth
;
8114 cg1_th
= cg0
+ ((cg1
- cg0
)*(th
+1))/comm
->nth
;
8117 /* Get the cg's for this pulse in this zone */
8118 get_zone_pulse_cgs(dd
, zonei
, zone
, cg0_th
, cg1_th
,
8120 dim
, dim_ind
, dim0
, dim1
, dim2
,
8123 normal
, skew_fac2_d
, skew_fac_01
,
8124 v_d
, v_0
, v_1
, &corners
, sf2_round
,
8125 bDistBonded
, bBondComm
,
8129 ibuf_p
, ibuf_nalloc_p
,
8134 GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR
;
8137 /* Append data of threads>=1 to the communication buffers */
8138 for (th
= 1; th
< comm
->nth
; th
++)
8140 dd_comm_setup_work_t
*dth
;
8143 dth
= &comm
->dth
[th
];
8145 ns1
= nsend
+ dth
->nsend_zone
;
8146 if (ns1
> ind
->nalloc
)
8148 ind
->nalloc
= over_alloc_dd(ns1
);
8149 srenew(ind
->index
, ind
->nalloc
);
8151 if (ns1
> comm
->nalloc_int
)
8153 comm
->nalloc_int
= over_alloc_dd(ns1
);
8154 srenew(comm
->buf_int
, comm
->nalloc_int
);
8156 if (ns1
> comm
->vbuf
.nalloc
)
8158 comm
->vbuf
.nalloc
= over_alloc_dd(ns1
);
8159 srenew(comm
->vbuf
.v
, comm
->vbuf
.nalloc
);
8162 for (i
= 0; i
< dth
->nsend_zone
; i
++)
8164 ind
->index
[nsend
] = dth
->ind
.index
[i
];
8165 comm
->buf_int
[nsend
] = dth
->ibuf
[i
];
8166 copy_rvec(dth
->vbuf
.v
[i
],
8167 comm
->vbuf
.v
[nsend
]);
8171 ind
->nsend
[zone
] += dth
->nsend_zone
;
8174 /* Clear the counts in case we do not have pbc */
8175 for (zone
= nzone_send
; zone
< nzone
; zone
++)
8177 ind
->nsend
[zone
] = 0;
8179 ind
->nsend
[nzone
] = nsend
;
8180 ind
->nsend
[nzone
+1] = nat
;
8181 /* Communicate the number of cg's and atoms to receive */
8182 dd_sendrecv_int(dd
, dim_ind
, dddirBackward
,
8183 ind
->nsend
, nzone
+2,
8184 ind
->nrecv
, nzone
+2);
8186 /* The rvec buffer is also required for atom buffers of size nsend
8187 * in dd_move_x and dd_move_f.
8189 vec_rvec_check_alloc(&comm
->vbuf
, ind
->nsend
[nzone
+1]);
8193 /* We can receive in place if only the last zone is not empty */
8194 for (zone
= 0; zone
< nzone
-1; zone
++)
8196 if (ind
->nrecv
[zone
] > 0)
8198 cd
->bInPlace
= FALSE
;
8203 /* The int buffer is only required here for the cg indices */
8204 if (ind
->nrecv
[nzone
] > comm
->nalloc_int2
)
8206 comm
->nalloc_int2
= over_alloc_dd(ind
->nrecv
[nzone
]);
8207 srenew(comm
->buf_int2
, comm
->nalloc_int2
);
8209 /* The rvec buffer is also required for atom buffers
8210 * of size nrecv in dd_move_x and dd_move_f.
8212 i
= std::max(cd
->ind
[0].nrecv
[nzone
+1], ind
->nrecv
[nzone
+1]);
8213 vec_rvec_check_alloc(&comm
->vbuf2
, i
);
8217 /* Make space for the global cg indices */
8218 if (pos_cg
+ ind
->nrecv
[nzone
] > dd
->cg_nalloc
8219 || dd
->cg_nalloc
== 0)
8221 dd
->cg_nalloc
= over_alloc_dd(pos_cg
+ ind
->nrecv
[nzone
]);
8222 srenew(index_gl
, dd
->cg_nalloc
);
8223 srenew(cgindex
, dd
->cg_nalloc
+1);
8225 /* Communicate the global cg indices */
8228 recv_i
= index_gl
+ pos_cg
;
8232 recv_i
= comm
->buf_int2
;
8234 dd_sendrecv_int(dd
, dim_ind
, dddirBackward
,
8235 comm
->buf_int
, nsend
,
8236 recv_i
, ind
->nrecv
[nzone
]);
8238 /* Make space for cg_cm */
8239 dd_check_alloc_ncg(fr
, state
, f
, pos_cg
+ ind
->nrecv
[nzone
]);
8240 if (fr
->cutoff_scheme
== ecutsGROUP
)
8246 cg_cm
= as_rvec_array(state
->x
.data());
8248 /* Communicate cg_cm */
8251 recv_vr
= cg_cm
+ pos_cg
;
8255 recv_vr
= comm
->vbuf2
.v
;
8257 dd_sendrecv_rvec(dd
, dim_ind
, dddirBackward
,
8258 comm
->vbuf
.v
, nsend
,
8259 recv_vr
, ind
->nrecv
[nzone
]);
8261 /* Make the charge group index */
8264 zone
= (p
== 0 ? 0 : nzone
- 1);
8265 while (zone
< nzone
)
8267 for (cg
= 0; cg
< ind
->nrecv
[zone
]; cg
++)
8269 cg_gl
= index_gl
[pos_cg
];
8270 fr
->cginfo
[pos_cg
] = ddcginfo(cginfo_mb
, cg_gl
);
8271 nrcg
= GET_CGINFO_NATOMS(fr
->cginfo
[pos_cg
]);
8272 cgindex
[pos_cg
+1] = cgindex
[pos_cg
] + nrcg
;
8275 /* Update the charge group presence,
8276 * so we can use it in the next pass of the loop.
8278 comm
->bLocalCG
[cg_gl
] = TRUE
;
8284 comm
->zone_ncg1
[nzone
+zone
] = ind
->nrecv
[zone
];
8287 zone_cg_range
[nzone
+zone
] = pos_cg
;
8292 /* This part of the code is never executed with bBondComm. */
8293 merge_cg_buffers(nzone
, cd
, p
, zone_cg_range
,
8294 index_gl
, recv_i
, cg_cm
, recv_vr
,
8295 cgindex
, fr
->cginfo_mb
, fr
->cginfo
);
8296 pos_cg
+= ind
->nrecv
[nzone
];
8298 nat_tot
+= ind
->nrecv
[nzone
+1];
8302 /* Store the atom block for easy copying of communication buffers */
8303 make_cell2at_index(cd
, nzone
, zone_cg_range
[nzone
], cgindex
);
8307 dd
->index_gl
= index_gl
;
8308 dd
->cgindex
= cgindex
;
8310 dd
->ncg_tot
= zone_cg_range
[zones
->n
];
8311 dd
->nat_tot
= nat_tot
;
8312 comm
->nat
[ddnatHOME
] = dd
->nat_home
;
8313 for (i
= ddnatZONE
; i
< ddnatNR
; i
++)
8315 comm
->nat
[i
] = dd
->nat_tot
;
8320 /* We don't need to update cginfo, since that was alrady done above.
8321 * So we pass NULL for the forcerec.
8323 dd_set_cginfo(dd
->index_gl
, dd
->ncg_home
, dd
->ncg_tot
,
8324 nullptr, comm
->bLocalCG
);
8329 fprintf(debug
, "Finished setting up DD communication, zones:");
8330 for (c
= 0; c
< zones
->n
; c
++)
8332 fprintf(debug
, " %d", zones
->cg_range
[c
+1]-zones
->cg_range
[c
]);
8334 fprintf(debug
, "\n");
8338 static void set_cg_boundaries(gmx_domdec_zones_t
*zones
)
8342 for (c
= 0; c
< zones
->nizone
; c
++)
8344 zones
->izone
[c
].cg1
= zones
->cg_range
[c
+1];
8345 zones
->izone
[c
].jcg0
= zones
->cg_range
[zones
->izone
[c
].j0
];
8346 zones
->izone
[c
].jcg1
= zones
->cg_range
[zones
->izone
[c
].j1
];
8350 static void set_zones_size(gmx_domdec_t
*dd
,
8351 matrix box
, const gmx_ddbox_t
*ddbox
,
8352 int zone_start
, int zone_end
)
8354 gmx_domdec_comm_t
*comm
;
8355 gmx_domdec_zones_t
*zones
;
8364 zones
= &comm
->zones
;
8366 /* Do we need to determine extra distances for multi-body bondeds? */
8367 bDistMB
= (comm
->bInterCGMultiBody
&& dlbIsOn(dd
->comm
) && dd
->ndim
> 1);
8369 for (z
= zone_start
; z
< zone_end
; z
++)
8371 /* Copy cell limits to zone limits.
8372 * Valid for non-DD dims and non-shifted dims.
8374 copy_rvec(comm
->cell_x0
, zones
->size
[z
].x0
);
8375 copy_rvec(comm
->cell_x1
, zones
->size
[z
].x1
);
8378 for (d
= 0; d
< dd
->ndim
; d
++)
8382 for (z
= 0; z
< zones
->n
; z
++)
8384 /* With a staggered grid we have different sizes
8385 * for non-shifted dimensions.
8387 if (dlbIsOn(dd
->comm
) && zones
->shift
[z
][dim
] == 0)
8391 zones
->size
[z
].x0
[dim
] = comm
->zone_d1
[zones
->shift
[z
][dd
->dim
[d
-1]]].min0
;
8392 zones
->size
[z
].x1
[dim
] = comm
->zone_d1
[zones
->shift
[z
][dd
->dim
[d
-1]]].max1
;
8396 zones
->size
[z
].x0
[dim
] = comm
->zone_d2
[zones
->shift
[z
][dd
->dim
[d
-2]]][zones
->shift
[z
][dd
->dim
[d
-1]]].min0
;
8397 zones
->size
[z
].x1
[dim
] = comm
->zone_d2
[zones
->shift
[z
][dd
->dim
[d
-2]]][zones
->shift
[z
][dd
->dim
[d
-1]]].max1
;
8403 rcmbs
= comm
->cutoff_mbody
;
8404 if (ddbox
->tric_dir
[dim
])
8406 rcs
/= ddbox
->skew_fac
[dim
];
8407 rcmbs
/= ddbox
->skew_fac
[dim
];
8410 /* Set the lower limit for the shifted zone dimensions */
8411 for (z
= zone_start
; z
< zone_end
; z
++)
8413 if (zones
->shift
[z
][dim
] > 0)
8416 if (!dlbIsOn(dd
->comm
) || d
== 0)
8418 zones
->size
[z
].x0
[dim
] = comm
->cell_x1
[dim
];
8419 zones
->size
[z
].x1
[dim
] = comm
->cell_x1
[dim
] + rcs
;
8423 /* Here we take the lower limit of the zone from
8424 * the lowest domain of the zone below.
8428 zones
->size
[z
].x0
[dim
] =
8429 comm
->zone_d1
[zones
->shift
[z
][dd
->dim
[d
-1]]].min1
;
8435 zones
->size
[z
].x0
[dim
] =
8436 zones
->size
[zone_perm
[2][z
-4]].x0
[dim
];
8440 zones
->size
[z
].x0
[dim
] =
8441 comm
->zone_d2
[zones
->shift
[z
][dd
->dim
[d
-2]]][zones
->shift
[z
][dd
->dim
[d
-1]]].min1
;
8444 /* A temporary limit, is updated below */
8445 zones
->size
[z
].x1
[dim
] = zones
->size
[z
].x0
[dim
];
8449 for (zi
= 0; zi
< zones
->nizone
; zi
++)
8451 if (zones
->shift
[zi
][dim
] == 0)
8453 /* This takes the whole zone into account.
8454 * With multiple pulses this will lead
8455 * to a larger zone then strictly necessary.
8457 zones
->size
[z
].x1
[dim
] = std::max(zones
->size
[z
].x1
[dim
],
8458 zones
->size
[zi
].x1
[dim
]+rcmbs
);
8466 /* Loop over the i-zones to set the upper limit of each
8469 for (zi
= 0; zi
< zones
->nizone
; zi
++)
8471 if (zones
->shift
[zi
][dim
] == 0)
8473 for (z
= zones
->izone
[zi
].j0
; z
< zones
->izone
[zi
].j1
; z
++)
8475 if (zones
->shift
[z
][dim
] > 0)
8477 zones
->size
[z
].x1
[dim
] = std::max(zones
->size
[z
].x1
[dim
],
8478 zones
->size
[zi
].x1
[dim
]+rcs
);
8485 for (z
= zone_start
; z
< zone_end
; z
++)
8487 /* Initialization only required to keep the compiler happy */
8488 rvec corner_min
= {0, 0, 0}, corner_max
= {0, 0, 0}, corner
;
8491 /* To determine the bounding box for a zone we need to find
8492 * the extreme corners of 4, 2 or 1 corners.
8494 nc
= 1 << (ddbox
->nboundeddim
- 1);
8496 for (c
= 0; c
< nc
; c
++)
8498 /* Set up a zone corner at x=0, ignoring trilinic couplings */
8502 corner
[YY
] = zones
->size
[z
].x0
[YY
];
8506 corner
[YY
] = zones
->size
[z
].x1
[YY
];
8510 corner
[ZZ
] = zones
->size
[z
].x0
[ZZ
];
8514 corner
[ZZ
] = zones
->size
[z
].x1
[ZZ
];
8516 if (dd
->ndim
== 1 && dd
->dim
[0] < ZZ
&& ZZ
< dd
->npbcdim
&&
8517 box
[ZZ
][1 - dd
->dim
[0]] != 0)
8519 /* With 1D domain decomposition the cg's are not in
8520 * the triclinic box, but triclinic x-y and rectangular y/x-z.
8521 * Shift the corner of the z-vector back to along the box
8522 * vector of dimension d, so it will later end up at 0 along d.
8523 * This can affect the location of this corner along dd->dim[0]
8524 * through the matrix operation below if box[d][dd->dim[0]]!=0.
8526 int d
= 1 - dd
->dim
[0];
8528 corner
[d
] -= corner
[ZZ
]*box
[ZZ
][d
]/box
[ZZ
][ZZ
];
8530 /* Apply the triclinic couplings */
8531 assert(ddbox
->npbcdim
<= DIM
);
8532 for (i
= YY
; i
< ddbox
->npbcdim
; i
++)
8534 for (j
= XX
; j
< i
; j
++)
8536 corner
[j
] += corner
[i
]*box
[i
][j
]/box
[i
][i
];
8541 copy_rvec(corner
, corner_min
);
8542 copy_rvec(corner
, corner_max
);
8546 for (i
= 0; i
< DIM
; i
++)
8548 corner_min
[i
] = std::min(corner_min
[i
], corner
[i
]);
8549 corner_max
[i
] = std::max(corner_max
[i
], corner
[i
]);
8553 /* Copy the extreme cornes without offset along x */
8554 for (i
= 0; i
< DIM
; i
++)
8556 zones
->size
[z
].bb_x0
[i
] = corner_min
[i
];
8557 zones
->size
[z
].bb_x1
[i
] = corner_max
[i
];
8559 /* Add the offset along x */
8560 zones
->size
[z
].bb_x0
[XX
] += zones
->size
[z
].x0
[XX
];
8561 zones
->size
[z
].bb_x1
[XX
] += zones
->size
[z
].x1
[XX
];
8564 if (zone_start
== 0)
8567 for (dim
= 0; dim
< DIM
; dim
++)
8569 vol
*= zones
->size
[0].x1
[dim
] - zones
->size
[0].x0
[dim
];
8571 zones
->dens_zone0
= (zones
->cg_range
[1] - zones
->cg_range
[0])/vol
;
8576 for (z
= zone_start
; z
< zone_end
; z
++)
8578 fprintf(debug
, "zone %d %6.3f - %6.3f %6.3f - %6.3f %6.3f - %6.3f\n",
8580 zones
->size
[z
].x0
[XX
], zones
->size
[z
].x1
[XX
],
8581 zones
->size
[z
].x0
[YY
], zones
->size
[z
].x1
[YY
],
8582 zones
->size
[z
].x0
[ZZ
], zones
->size
[z
].x1
[ZZ
]);
8583 fprintf(debug
, "zone %d bb %6.3f - %6.3f %6.3f - %6.3f %6.3f - %6.3f\n",
8585 zones
->size
[z
].bb_x0
[XX
], zones
->size
[z
].bb_x1
[XX
],
8586 zones
->size
[z
].bb_x0
[YY
], zones
->size
[z
].bb_x1
[YY
],
8587 zones
->size
[z
].bb_x0
[ZZ
], zones
->size
[z
].bb_x1
[ZZ
]);
8592 static int comp_cgsort(const void *a
, const void *b
)
8596 gmx_cgsort_t
*cga
, *cgb
;
8597 cga
= (gmx_cgsort_t
*)a
;
8598 cgb
= (gmx_cgsort_t
*)b
;
8600 comp
= cga
->nsc
- cgb
->nsc
;
8603 comp
= cga
->ind_gl
- cgb
->ind_gl
;
8609 static void order_int_cg(int n
, const gmx_cgsort_t
*sort
,
8614 /* Order the data */
8615 for (i
= 0; i
< n
; i
++)
8617 buf
[i
] = a
[sort
[i
].ind
];
8620 /* Copy back to the original array */
8621 for (i
= 0; i
< n
; i
++)
8627 static void order_vec_cg(int n
, const gmx_cgsort_t
*sort
,
8632 /* Order the data */
8633 for (i
= 0; i
< n
; i
++)
8635 copy_rvec(v
[sort
[i
].ind
], buf
[i
]);
8638 /* Copy back to the original array */
8639 for (i
= 0; i
< n
; i
++)
8641 copy_rvec(buf
[i
], v
[i
]);
8645 static void order_vec_atom(int ncg
, const int *cgindex
, const gmx_cgsort_t
*sort
,
8648 int a
, atot
, cg
, cg0
, cg1
, i
;
8650 if (cgindex
== nullptr)
8652 /* Avoid the useless loop of the atoms within a cg */
8653 order_vec_cg(ncg
, sort
, v
, buf
);
8658 /* Order the data */
8660 for (cg
= 0; cg
< ncg
; cg
++)
8662 cg0
= cgindex
[sort
[cg
].ind
];
8663 cg1
= cgindex
[sort
[cg
].ind
+1];
8664 for (i
= cg0
; i
< cg1
; i
++)
8666 copy_rvec(v
[i
], buf
[a
]);
8672 /* Copy back to the original array */
8673 for (a
= 0; a
< atot
; a
++)
8675 copy_rvec(buf
[a
], v
[a
]);
8679 static void ordered_sort(int nsort2
, gmx_cgsort_t
*sort2
,
8680 int nsort_new
, gmx_cgsort_t
*sort_new
,
8681 gmx_cgsort_t
*sort1
)
8685 /* The new indices are not very ordered, so we qsort them */
8686 gmx_qsort_threadsafe(sort_new
, nsort_new
, sizeof(sort_new
[0]), comp_cgsort
);
8688 /* sort2 is already ordered, so now we can merge the two arrays */
8692 while (i2
< nsort2
|| i_new
< nsort_new
)
8696 sort1
[i1
++] = sort_new
[i_new
++];
8698 else if (i_new
== nsort_new
)
8700 sort1
[i1
++] = sort2
[i2
++];
8702 else if (sort2
[i2
].nsc
< sort_new
[i_new
].nsc
||
8703 (sort2
[i2
].nsc
== sort_new
[i_new
].nsc
&&
8704 sort2
[i2
].ind_gl
< sort_new
[i_new
].ind_gl
))
8706 sort1
[i1
++] = sort2
[i2
++];
8710 sort1
[i1
++] = sort_new
[i_new
++];
8715 static int dd_sort_order(gmx_domdec_t
*dd
, t_forcerec
*fr
, int ncg_home_old
)
8717 gmx_domdec_sort_t
*sort
;
8718 gmx_cgsort_t
*cgsort
, *sort_i
;
8719 int ncg_new
, nsort2
, nsort_new
, i
, *a
, moved
;
8721 sort
= dd
->comm
->sort
;
8723 a
= fr
->ns
->grid
->cell_index
;
8725 moved
= NSGRID_SIGNAL_MOVED_FAC
*fr
->ns
->grid
->ncells
;
8727 if (ncg_home_old
>= 0)
8729 /* The charge groups that remained in the same ns grid cell
8730 * are completely ordered. So we can sort efficiently by sorting
8731 * the charge groups that did move into the stationary list.
8736 for (i
= 0; i
< dd
->ncg_home
; i
++)
8738 /* Check if this cg did not move to another node */
8741 if (i
>= ncg_home_old
|| a
[i
] != sort
->sort
[i
].nsc
)
8743 /* This cg is new on this node or moved ns grid cell */
8744 if (nsort_new
>= sort
->sort_new_nalloc
)
8746 sort
->sort_new_nalloc
= over_alloc_dd(nsort_new
+1);
8747 srenew(sort
->sort_new
, sort
->sort_new_nalloc
);
8749 sort_i
= &(sort
->sort_new
[nsort_new
++]);
8753 /* This cg did not move */
8754 sort_i
= &(sort
->sort2
[nsort2
++]);
8756 /* Sort on the ns grid cell indices
8757 * and the global topology index.
8758 * index_gl is irrelevant with cell ns,
8759 * but we set it here anyhow to avoid a conditional.
8762 sort_i
->ind_gl
= dd
->index_gl
[i
];
8769 fprintf(debug
, "ordered sort cgs: stationary %d moved %d\n",
8772 /* Sort efficiently */
8773 ordered_sort(nsort2
, sort
->sort2
, nsort_new
, sort
->sort_new
,
8778 cgsort
= sort
->sort
;
8780 for (i
= 0; i
< dd
->ncg_home
; i
++)
8782 /* Sort on the ns grid cell indices
8783 * and the global topology index
8785 cgsort
[i
].nsc
= a
[i
];
8786 cgsort
[i
].ind_gl
= dd
->index_gl
[i
];
8788 if (cgsort
[i
].nsc
< moved
)
8795 fprintf(debug
, "qsort cgs: %d new home %d\n", dd
->ncg_home
, ncg_new
);
8797 /* Determine the order of the charge groups using qsort */
8798 gmx_qsort_threadsafe(cgsort
, dd
->ncg_home
, sizeof(cgsort
[0]), comp_cgsort
);
8804 static int dd_sort_order_nbnxn(gmx_domdec_t
*dd
, t_forcerec
*fr
)
8810 sort
= dd
->comm
->sort
->sort
;
8812 nbnxn_get_atomorder(fr
->nbv
->nbs
, &a
, &na
);
8815 for (i
= 0; i
< na
; i
++)
8819 sort
[ncg_new
].ind
= a
[i
];
8827 static void dd_sort_state(gmx_domdec_t
*dd
, rvec
*cgcm
, t_forcerec
*fr
, t_state
*state
,
8830 gmx_domdec_sort_t
*sort
;
8831 gmx_cgsort_t
*cgsort
;
8833 int ncg_new
, i
, *ibuf
, cgsize
;
8836 sort
= dd
->comm
->sort
;
8838 if (dd
->ncg_home
> sort
->sort_nalloc
)
8840 sort
->sort_nalloc
= over_alloc_dd(dd
->ncg_home
);
8841 srenew(sort
->sort
, sort
->sort_nalloc
);
8842 srenew(sort
->sort2
, sort
->sort_nalloc
);
8844 cgsort
= sort
->sort
;
8846 switch (fr
->cutoff_scheme
)
8849 ncg_new
= dd_sort_order(dd
, fr
, ncg_home_old
);
8852 ncg_new
= dd_sort_order_nbnxn(dd
, fr
);
8855 gmx_incons("unimplemented");
8859 /* We alloc with the old size, since cgindex is still old */
8860 vec_rvec_check_alloc(&dd
->comm
->vbuf
, dd
->cgindex
[dd
->ncg_home
]);
8861 vbuf
= dd
->comm
->vbuf
.v
;
8865 cgindex
= dd
->cgindex
;
8872 /* Remove the charge groups which are no longer at home here */
8873 dd
->ncg_home
= ncg_new
;
8876 fprintf(debug
, "Set the new home charge group count to %d\n",
8880 /* Reorder the state */
8881 if (state
->flags
& (1 << estX
))
8883 order_vec_atom(dd
->ncg_home
, cgindex
, cgsort
, as_rvec_array(state
->x
.data()), vbuf
);
8885 if (state
->flags
& (1 << estV
))
8887 order_vec_atom(dd
->ncg_home
, cgindex
, cgsort
, as_rvec_array(state
->v
.data()), vbuf
);
8889 if (state
->flags
& (1 << estCGP
))
8891 order_vec_atom(dd
->ncg_home
, cgindex
, cgsort
, as_rvec_array(state
->cg_p
.data()), vbuf
);
8894 if (fr
->cutoff_scheme
== ecutsGROUP
)
8897 order_vec_cg(dd
->ncg_home
, cgsort
, cgcm
, vbuf
);
8900 if (dd
->ncg_home
+1 > sort
->ibuf_nalloc
)
8902 sort
->ibuf_nalloc
= over_alloc_dd(dd
->ncg_home
+1);
8903 srenew(sort
->ibuf
, sort
->ibuf_nalloc
);
8906 /* Reorder the global cg index */
8907 order_int_cg(dd
->ncg_home
, cgsort
, dd
->index_gl
, ibuf
);
8908 /* Reorder the cginfo */
8909 order_int_cg(dd
->ncg_home
, cgsort
, fr
->cginfo
, ibuf
);
8910 /* Rebuild the local cg index */
8914 for (i
= 0; i
< dd
->ncg_home
; i
++)
8916 cgsize
= dd
->cgindex
[cgsort
[i
].ind
+1] - dd
->cgindex
[cgsort
[i
].ind
];
8917 ibuf
[i
+1] = ibuf
[i
] + cgsize
;
8919 for (i
= 0; i
< dd
->ncg_home
+1; i
++)
8921 dd
->cgindex
[i
] = ibuf
[i
];
8926 for (i
= 0; i
< dd
->ncg_home
+1; i
++)
8931 /* Set the home atom number */
8932 dd
->nat_home
= dd
->cgindex
[dd
->ncg_home
];
8934 if (fr
->cutoff_scheme
== ecutsVERLET
)
8936 /* The atoms are now exactly in grid order, update the grid order */
8937 nbnxn_set_atomorder(fr
->nbv
->nbs
);
8941 /* Copy the sorted ns cell indices back to the ns grid struct */
8942 for (i
= 0; i
< dd
->ncg_home
; i
++)
8944 fr
->ns
->grid
->cell_index
[i
] = cgsort
[i
].nsc
;
8946 fr
->ns
->grid
->nr
= dd
->ncg_home
;
8950 static void add_dd_statistics(gmx_domdec_t
*dd
)
8952 gmx_domdec_comm_t
*comm
;
8957 for (ddnat
= ddnatZONE
; ddnat
< ddnatNR
; ddnat
++)
8959 comm
->sum_nat
[ddnat
-ddnatZONE
] +=
8960 comm
->nat
[ddnat
] - comm
->nat
[ddnat
-1];
8965 void reset_dd_statistics_counters(gmx_domdec_t
*dd
)
8967 gmx_domdec_comm_t
*comm
;
8972 /* Reset all the statistics and counters for total run counting */
8973 for (ddnat
= ddnatZONE
; ddnat
< ddnatNR
; ddnat
++)
8975 comm
->sum_nat
[ddnat
-ddnatZONE
] = 0;
8979 comm
->load_step
= 0;
8982 clear_ivec(comm
->load_lim
);
8987 void print_dd_statistics(t_commrec
*cr
, t_inputrec
*ir
, FILE *fplog
)
8989 gmx_domdec_comm_t
*comm
;
8993 comm
= cr
->dd
->comm
;
8995 gmx_sumd(ddnatNR
-ddnatZONE
, comm
->sum_nat
, cr
);
8997 if (fplog
== nullptr)
9002 fprintf(fplog
, "\n D O M A I N D E C O M P O S I T I O N S T A T I S T I C S\n\n");
9004 for (ddnat
= ddnatZONE
; ddnat
< ddnatNR
; ddnat
++)
9006 av
= comm
->sum_nat
[ddnat
-ddnatZONE
]/comm
->ndecomp
;
9011 " av. #atoms communicated per step for force: %d x %.1f\n",
9015 if (cr
->dd
->vsite_comm
)
9018 " av. #atoms communicated per step for vsites: %d x %.1f\n",
9019 (EEL_PME(ir
->coulombtype
) || ir
->coulombtype
== eelEWALD
) ? 3 : 2,
9024 if (cr
->dd
->constraint_comm
)
9027 " av. #atoms communicated per step for LINCS: %d x %.1f\n",
9028 1 + ir
->nLincsIter
, av
);
9032 gmx_incons(" Unknown type for DD statistics");
9035 fprintf(fplog
, "\n");
9037 if (comm
->bRecordLoad
&& EI_DYNAMICS(ir
->eI
))
9039 print_dd_load_av(fplog
, cr
->dd
);
9043 void dd_partition_system(FILE *fplog
,
9046 gmx_bool bMasterState
,
9048 t_state
*state_global
,
9049 const gmx_mtop_t
*top_global
,
9050 const t_inputrec
*ir
,
9051 t_state
*state_local
,
9052 PaddedRVecVector
*f
,
9054 gmx_localtop_t
*top_local
,
9057 gmx_constr_t constr
,
9059 gmx_wallcycle_t wcycle
,
9063 gmx_domdec_comm_t
*comm
;
9064 gmx_ddbox_t ddbox
= {0};
9066 gmx_int64_t step_pcoupl
;
9067 rvec cell_ns_x0
, cell_ns_x1
;
9068 int i
, n
, ncgindex_set
, ncg_home_old
= -1, ncg_moved
, nat_f_novirsum
;
9069 gmx_bool bBoxChanged
, bNStGlobalComm
, bDoDLB
, bCheckWhetherToTurnDlbOn
, bLogLoad
;
9070 gmx_bool bRedist
, bSortCG
, bResortAll
;
9071 ivec ncells_old
= {0, 0, 0}, ncells_new
= {0, 0, 0}, np
;
9075 wallcycle_start(wcycle
, ewcDOMDEC
);
9080 bBoxChanged
= (bMasterState
|| inputrecDeform(ir
));
9081 if (ir
->epc
!= epcNO
)
9083 /* With nstpcouple > 1 pressure coupling happens.
9084 * one step after calculating the pressure.
9085 * Box scaling happens at the end of the MD step,
9086 * after the DD partitioning.
9087 * We therefore have to do DLB in the first partitioning
9088 * after an MD step where P-coupling occurred.
9089 * We need to determine the last step in which p-coupling occurred.
9090 * MRS -- need to validate this for vv?
9095 step_pcoupl
= step
- 1;
9099 step_pcoupl
= ((step
- 1)/n
)*n
+ 1;
9101 if (step_pcoupl
>= comm
->partition_step
)
9107 bNStGlobalComm
= (step
% nstglobalcomm
== 0);
9115 /* Should we do dynamic load balacing this step?
9116 * Since it requires (possibly expensive) global communication,
9117 * we might want to do DLB less frequently.
9119 if (bBoxChanged
|| ir
->epc
!= epcNO
)
9121 bDoDLB
= bBoxChanged
;
9125 bDoDLB
= bNStGlobalComm
;
9129 /* Check if we have recorded loads on the nodes */
9130 if (comm
->bRecordLoad
&& dd_load_count(comm
) > 0)
9132 bCheckWhetherToTurnDlbOn
= dd_dlb_get_should_check_whether_to_turn_dlb_on(dd
);
9134 /* Print load every nstlog, first and last step to the log file */
9135 bLogLoad
= ((ir
->nstlog
> 0 && step
% ir
->nstlog
== 0) ||
9136 comm
->n_load_collect
== 0 ||
9138 (step
+ ir
->nstlist
> ir
->init_step
+ ir
->nsteps
)));
9140 /* Avoid extra communication due to verbose screen output
9141 * when nstglobalcomm is set.
9143 if (bDoDLB
|| bLogLoad
|| bCheckWhetherToTurnDlbOn
||
9144 (bVerbose
&& (ir
->nstlist
== 0 || nstglobalcomm
<= ir
->nstlist
)))
9146 get_load_distribution(dd
, wcycle
);
9151 dd_print_load(fplog
, dd
, step
-1);
9155 dd_print_load_verbose(dd
);
9158 comm
->n_load_collect
++;
9164 /* Add the measured cycles to the running average */
9165 const float averageFactor
= 0.1f
;
9166 comm
->cyclesPerStepDlbExpAverage
=
9167 (1 - averageFactor
)*comm
->cyclesPerStepDlbExpAverage
+
9168 averageFactor
*comm
->cycl
[ddCyclStep
]/comm
->cycl_n
[ddCyclStep
];
9170 if (comm
->dlbState
== edlbsOnCanTurnOff
&&
9171 dd
->comm
->n_load_have
% c_checkTurnDlbOffInterval
== c_checkTurnDlbOffInterval
- 1)
9173 gmx_bool turnOffDlb
;
9176 /* If the running averaged cycles with DLB are more
9177 * than before we turned on DLB, turn off DLB.
9178 * We will again run and check the cycles without DLB
9179 * and we can then decide if to turn off DLB forever.
9181 turnOffDlb
= (comm
->cyclesPerStepDlbExpAverage
>
9182 comm
->cyclesPerStepBeforeDLB
);
9184 dd_bcast(dd
, sizeof(turnOffDlb
), &turnOffDlb
);
9187 /* To turn off DLB, we need to redistribute the atoms */
9188 dd_collect_state(dd
, state_local
, state_global
);
9189 bMasterState
= TRUE
;
9190 turn_off_dlb(fplog
, cr
, step
);
9194 else if (bCheckWhetherToTurnDlbOn
)
9196 gmx_bool turnOffDlbForever
= FALSE
;
9197 gmx_bool turnOnDlb
= FALSE
;
9199 /* Since the timings are node dependent, the master decides */
9202 /* If we recently turned off DLB, we want to check if
9203 * performance is better without DLB. We want to do this
9204 * ASAP to minimize the chance that external factors
9205 * slowed down the DLB step are gone here and we
9206 * incorrectly conclude that DLB was causing the slowdown.
9207 * So we measure one nstlist block, no running average.
9209 if (comm
->haveTurnedOffDlb
&&
9210 comm
->cycl
[ddCyclStep
]/comm
->cycl_n
[ddCyclStep
] <
9211 comm
->cyclesPerStepDlbExpAverage
)
9213 /* After turning off DLB we ran nstlist steps in fewer
9214 * cycles than with DLB. This likely means that DLB
9215 * in not benefical, but this could be due to a one
9216 * time unlucky fluctuation, so we require two such
9217 * observations in close succession to turn off DLB
9220 if (comm
->dlbSlowerPartitioningCount
> 0 &&
9221 dd
->ddp_count
< comm
->dlbSlowerPartitioningCount
+ 10*c_checkTurnDlbOnInterval
)
9223 turnOffDlbForever
= TRUE
;
9225 comm
->haveTurnedOffDlb
= false;
9226 /* Register when we last measured DLB slowdown */
9227 comm
->dlbSlowerPartitioningCount
= dd
->ddp_count
;
9231 /* Here we check if the max PME rank load is more than 0.98
9232 * the max PP force load. If so, PP DLB will not help,
9233 * since we are (almost) limited by PME. Furthermore,
9234 * DLB will cause a significant extra x/f redistribution
9235 * cost on the PME ranks, which will then surely result
9236 * in lower total performance.
9238 if (cr
->npmenodes
> 0 &&
9239 dd_pme_f_ratio(dd
) > 1 - DD_PERF_LOSS_DLB_ON
)
9245 turnOnDlb
= (dd_force_imb_perf_loss(dd
) >= DD_PERF_LOSS_DLB_ON
);
9251 gmx_bool turnOffDlbForever
;
9255 turnOffDlbForever
, turnOnDlb
9257 dd_bcast(dd
, sizeof(bools
), &bools
);
9258 if (bools
.turnOffDlbForever
)
9260 turn_off_dlb_forever(fplog
, cr
, step
);
9262 else if (bools
.turnOnDlb
)
9264 turn_on_dlb(fplog
, cr
, step
);
9269 comm
->n_load_have
++;
9272 cgs_gl
= &comm
->cgs_gl
;
9277 /* Clear the old state */
9278 clear_dd_indices(dd
, 0, 0);
9281 set_ddbox(dd
, bMasterState
, cr
, ir
, state_global
->box
,
9282 TRUE
, cgs_gl
, as_rvec_array(state_global
->x
.data()), &ddbox
);
9284 get_cg_distribution(fplog
, dd
, cgs_gl
,
9285 state_global
->box
, &ddbox
, as_rvec_array(state_global
->x
.data()));
9287 dd_distribute_state(dd
, cgs_gl
,
9288 state_global
, state_local
, f
);
9290 dd_make_local_cgs(dd
, &top_local
->cgs
);
9292 /* Ensure that we have space for the new distribution */
9293 dd_check_alloc_ncg(fr
, state_local
, f
, dd
->ncg_home
);
9295 if (fr
->cutoff_scheme
== ecutsGROUP
)
9297 calc_cgcm(fplog
, 0, dd
->ncg_home
,
9298 &top_local
->cgs
, as_rvec_array(state_local
->x
.data()), fr
->cg_cm
);
9301 inc_nrnb(nrnb
, eNR_CGCM
, dd
->nat_home
);
9303 dd_set_cginfo(dd
->index_gl
, 0, dd
->ncg_home
, fr
, comm
->bLocalCG
);
9305 else if (state_local
->ddp_count
!= dd
->ddp_count
)
9307 if (state_local
->ddp_count
> dd
->ddp_count
)
9309 gmx_fatal(FARGS
, "Internal inconsistency state_local->ddp_count (%d) > dd->ddp_count (%d)", state_local
->ddp_count
, dd
->ddp_count
);
9312 if (state_local
->ddp_count_cg_gl
!= state_local
->ddp_count
)
9314 gmx_fatal(FARGS
, "Internal inconsistency state_local->ddp_count_cg_gl (%d) != state_local->ddp_count (%d)", state_local
->ddp_count_cg_gl
, state_local
->ddp_count
);
9317 /* Clear the old state */
9318 clear_dd_indices(dd
, 0, 0);
9320 /* Build the new indices */
9321 rebuild_cgindex(dd
, cgs_gl
->index
, state_local
);
9322 make_dd_indices(dd
, cgs_gl
->index
, 0);
9323 ncgindex_set
= dd
->ncg_home
;
9325 if (fr
->cutoff_scheme
== ecutsGROUP
)
9327 /* Redetermine the cg COMs */
9328 calc_cgcm(fplog
, 0, dd
->ncg_home
,
9329 &top_local
->cgs
, as_rvec_array(state_local
->x
.data()), fr
->cg_cm
);
9332 inc_nrnb(nrnb
, eNR_CGCM
, dd
->nat_home
);
9334 dd_set_cginfo(dd
->index_gl
, 0, dd
->ncg_home
, fr
, comm
->bLocalCG
);
9336 set_ddbox(dd
, bMasterState
, cr
, ir
, state_local
->box
,
9337 TRUE
, &top_local
->cgs
, as_rvec_array(state_local
->x
.data()), &ddbox
);
9339 bRedist
= dlbIsOn(comm
);
9343 /* We have the full state, only redistribute the cgs */
9345 /* Clear the non-home indices */
9346 clear_dd_indices(dd
, dd
->ncg_home
, dd
->nat_home
);
9349 /* Avoid global communication for dim's without pbc and -gcom */
9350 if (!bNStGlobalComm
)
9352 copy_rvec(comm
->box0
, ddbox
.box0
);
9353 copy_rvec(comm
->box_size
, ddbox
.box_size
);
9355 set_ddbox(dd
, bMasterState
, cr
, ir
, state_local
->box
,
9356 bNStGlobalComm
, &top_local
->cgs
, as_rvec_array(state_local
->x
.data()), &ddbox
);
9361 /* For dim's without pbc and -gcom */
9362 copy_rvec(ddbox
.box0
, comm
->box0
);
9363 copy_rvec(ddbox
.box_size
, comm
->box_size
);
9365 set_dd_cell_sizes(dd
, &ddbox
, dynamic_dd_box(&ddbox
, ir
), bMasterState
, bDoDLB
,
9368 if (comm
->nstDDDumpGrid
> 0 && step
% comm
->nstDDDumpGrid
== 0)
9370 write_dd_grid_pdb("dd_grid", step
, dd
, state_local
->box
, &ddbox
);
9373 /* Check if we should sort the charge groups */
9374 bSortCG
= (bMasterState
|| bRedist
);
9376 ncg_home_old
= dd
->ncg_home
;
9381 wallcycle_sub_start(wcycle
, ewcsDD_REDIST
);
9383 dd_redistribute_cg(fplog
, step
, dd
, ddbox
.tric_dir
,
9385 !bSortCG
, nrnb
, &ncgindex_set
, &ncg_moved
);
9387 wallcycle_sub_stop(wcycle
, ewcsDD_REDIST
);
9390 get_nsgrid_boundaries(ddbox
.nboundeddim
, state_local
->box
,
9392 &comm
->cell_x0
, &comm
->cell_x1
,
9393 dd
->ncg_home
, fr
->cg_cm
,
9394 cell_ns_x0
, cell_ns_x1
, &grid_density
);
9398 comm_dd_ns_cell_sizes(dd
, &ddbox
, cell_ns_x0
, cell_ns_x1
, step
);
9401 switch (fr
->cutoff_scheme
)
9404 copy_ivec(fr
->ns
->grid
->n
, ncells_old
);
9405 grid_first(fplog
, fr
->ns
->grid
, dd
, &ddbox
,
9406 state_local
->box
, cell_ns_x0
, cell_ns_x1
,
9407 fr
->rlist
, grid_density
);
9410 nbnxn_get_ncells(fr
->nbv
->nbs
, &ncells_old
[XX
], &ncells_old
[YY
]);
9413 gmx_incons("unimplemented");
9415 /* We need to store tric_dir for dd_get_ns_ranges called from ns.c */
9416 copy_ivec(ddbox
.tric_dir
, comm
->tric_dir
);
9420 wallcycle_sub_start(wcycle
, ewcsDD_GRID
);
9422 /* Sort the state on charge group position.
9423 * This enables exact restarts from this step.
9424 * It also improves performance by about 15% with larger numbers
9425 * of atoms per node.
9428 /* Fill the ns grid with the home cell,
9429 * so we can sort with the indices.
9431 set_zones_ncg_home(dd
);
9433 switch (fr
->cutoff_scheme
)
9436 set_zones_size(dd
, state_local
->box
, &ddbox
, 0, 1);
9438 nbnxn_put_on_grid(fr
->nbv
->nbs
, fr
->ePBC
, state_local
->box
,
9440 comm
->zones
.size
[0].bb_x0
,
9441 comm
->zones
.size
[0].bb_x1
,
9443 comm
->zones
.dens_zone0
,
9445 as_rvec_array(state_local
->x
.data()),
9446 ncg_moved
, bRedist
? comm
->moved
: nullptr,
9447 fr
->nbv
->grp
[eintLocal
].kernel_type
,
9448 fr
->nbv
->grp
[eintLocal
].nbat
);
9450 nbnxn_get_ncells(fr
->nbv
->nbs
, &ncells_new
[XX
], &ncells_new
[YY
]);
9453 fill_grid(&comm
->zones
, fr
->ns
->grid
, dd
->ncg_home
,
9454 0, dd
->ncg_home
, fr
->cg_cm
);
9456 copy_ivec(fr
->ns
->grid
->n
, ncells_new
);
9459 gmx_incons("unimplemented");
9462 bResortAll
= bMasterState
;
9464 /* Check if we can user the old order and ns grid cell indices
9465 * of the charge groups to sort the charge groups efficiently.
9467 if (ncells_new
[XX
] != ncells_old
[XX
] ||
9468 ncells_new
[YY
] != ncells_old
[YY
] ||
9469 ncells_new
[ZZ
] != ncells_old
[ZZ
])
9476 fprintf(debug
, "Step %s, sorting the %d home charge groups\n",
9477 gmx_step_str(step
, sbuf
), dd
->ncg_home
);
9479 dd_sort_state(dd
, fr
->cg_cm
, fr
, state_local
,
9480 bResortAll
? -1 : ncg_home_old
);
9482 /* After sorting and compacting we set the correct size */
9483 dd_resize_state(state_local
, f
, dd
->nat_home
);
9485 /* Rebuild all the indices */
9486 ga2la_clear(dd
->ga2la
);
9489 wallcycle_sub_stop(wcycle
, ewcsDD_GRID
);
9492 wallcycle_sub_start(wcycle
, ewcsDD_SETUPCOMM
);
9494 /* Setup up the communication and communicate the coordinates */
9495 setup_dd_communication(dd
, state_local
->box
, &ddbox
, fr
, state_local
, f
);
9497 /* Set the indices */
9498 make_dd_indices(dd
, cgs_gl
->index
, ncgindex_set
);
9500 /* Set the charge group boundaries for neighbor searching */
9501 set_cg_boundaries(&comm
->zones
);
9503 if (fr
->cutoff_scheme
== ecutsVERLET
)
9505 set_zones_size(dd
, state_local
->box
, &ddbox
,
9506 bSortCG
? 1 : 0, comm
->zones
.n
);
9509 wallcycle_sub_stop(wcycle
, ewcsDD_SETUPCOMM
);
9512 write_dd_pdb("dd_home",step,"dump",top_global,cr,
9513 -1,as_rvec_array(state_local->x.data()),state_local->box);
9516 wallcycle_sub_start(wcycle
, ewcsDD_MAKETOP
);
9518 /* Extract a local topology from the global topology */
9519 for (i
= 0; i
< dd
->ndim
; i
++)
9521 np
[dd
->dim
[i
]] = comm
->cd
[i
].np
;
9523 dd_make_local_top(dd
, &comm
->zones
, dd
->npbcdim
, state_local
->box
,
9524 comm
->cellsize_min
, np
,
9526 fr
->cutoff_scheme
== ecutsGROUP
? fr
->cg_cm
: as_rvec_array(state_local
->x
.data()),
9527 vsite
, top_global
, top_local
);
9529 wallcycle_sub_stop(wcycle
, ewcsDD_MAKETOP
);
9531 wallcycle_sub_start(wcycle
, ewcsDD_MAKECONSTR
);
9533 /* Set up the special atom communication */
9534 n
= comm
->nat
[ddnatZONE
];
9535 for (i
= ddnatZONE
+1; i
< ddnatNR
; i
++)
9540 if (vsite
&& vsite
->n_intercg_vsite
)
9542 n
= dd_make_local_vsites(dd
, n
, top_local
->idef
.il
);
9546 if (dd
->bInterCGcons
|| dd
->bInterCGsettles
)
9548 /* Only for inter-cg constraints we need special code */
9549 n
= dd_make_local_constraints(dd
, n
, top_global
, fr
->cginfo
,
9550 constr
, ir
->nProjOrder
,
9551 top_local
->idef
.il
);
9555 gmx_incons("Unknown special atom type setup");
9560 wallcycle_sub_stop(wcycle
, ewcsDD_MAKECONSTR
);
9562 wallcycle_sub_start(wcycle
, ewcsDD_TOPOTHER
);
9564 /* Make space for the extra coordinates for virtual site
9565 * or constraint communication.
9567 state_local
->natoms
= comm
->nat
[ddnatNR
-1];
9569 dd_resize_state(state_local
, f
, state_local
->natoms
);
9571 if (fr
->bF_NoVirSum
)
9573 if (vsite
&& vsite
->n_intercg_vsite
)
9575 nat_f_novirsum
= comm
->nat
[ddnatVSITE
];
9579 if (EEL_FULL(ir
->coulombtype
) && dd
->n_intercg_excl
> 0)
9581 nat_f_novirsum
= dd
->nat_tot
;
9585 nat_f_novirsum
= dd
->nat_home
;
9594 /* Set the number of atoms required for the force calculation.
9595 * Forces need to be constrained when doing energy
9596 * minimization. For simple simulations we could avoid some
9597 * allocation, zeroing and copying, but this is probably not worth
9598 * the complications and checking.
9600 forcerec_set_ranges(fr
, dd
->ncg_home
, dd
->ncg_tot
,
9601 dd
->nat_tot
, comm
->nat
[ddnatCON
], nat_f_novirsum
);
9603 /* Update atom data for mdatoms and several algorithms */
9604 mdAlgorithmsSetupAtomData(cr
, ir
, top_global
, top_local
, fr
,
9605 nullptr, mdatoms
, vsite
, nullptr);
9607 if (ir
->implicit_solvent
)
9609 make_local_gb(cr
, fr
->born
, ir
->gb_algorithm
);
9612 if (!(cr
->duty
& DUTY_PME
))
9614 /* Send the charges and/or c6/sigmas to our PME only node */
9615 gmx_pme_send_parameters(cr
,
9617 mdatoms
->nChargePerturbed
, mdatoms
->nTypePerturbed
,
9618 mdatoms
->chargeA
, mdatoms
->chargeB
,
9619 mdatoms
->sqrt_c6A
, mdatoms
->sqrt_c6B
,
9620 mdatoms
->sigmaA
, mdatoms
->sigmaB
,
9621 dd_pme_maxshift_x(dd
), dd_pme_maxshift_y(dd
));
9626 set_constraints(constr
, top_local
, ir
, mdatoms
, cr
);
9631 /* Update the local pull groups */
9632 dd_make_local_pull_groups(cr
, ir
->pull_work
, mdatoms
);
9637 /* Update the local rotation groups */
9638 dd_make_local_rotation_groups(dd
, ir
->rot
);
9641 if (ir
->eSwapCoords
!= eswapNO
)
9643 /* Update the local groups needed for ion swapping */
9644 dd_make_local_swap_groups(dd
, ir
->swap
);
9647 /* Update the local atoms to be communicated via the IMD protocol if bIMD is TRUE. */
9648 dd_make_local_IMD_atoms(ir
->bIMD
, dd
, ir
->imd
);
9650 add_dd_statistics(dd
);
9652 /* Make sure we only count the cycles for this DD partitioning */
9653 clear_dd_cycle_counts(dd
);
9655 /* Because the order of the atoms might have changed since
9656 * the last vsite construction, we need to communicate the constructing
9657 * atom coordinates again (for spreading the forces this MD step).
9659 dd_move_x_vsites(dd
, state_local
->box
, as_rvec_array(state_local
->x
.data()));
9661 wallcycle_sub_stop(wcycle
, ewcsDD_TOPOTHER
);
9663 if (comm
->nstDDDump
> 0 && step
% comm
->nstDDDump
== 0)
9665 dd_move_x(dd
, state_local
->box
, as_rvec_array(state_local
->x
.data()));
9666 write_dd_pdb("dd_dump", step
, "dump", top_global
, cr
,
9667 -1, as_rvec_array(state_local
->x
.data()), state_local
->box
);
9670 /* Store the partitioning step */
9671 comm
->partition_step
= step
;
9673 /* Increase the DD partitioning counter */
9675 /* The state currently matches this DD partitioning count, store it */
9676 state_local
->ddp_count
= dd
->ddp_count
;
9679 /* The DD master node knows the complete cg distribution,
9680 * store the count so we can possibly skip the cg info communication.
9682 comm
->master_cg_ddp_count
= (bSortCG
? 0 : dd
->ddp_count
);
9685 if (comm
->DD_debug
> 0)
9687 /* Set the env var GMX_DD_DEBUG if you suspect corrupted indices */
9688 check_index_consistency(dd
, top_global
->natoms
, ncg_mtop(top_global
),
9689 "after partitioning");
9692 wallcycle_stop(wcycle
, ewcDOMDEC
);