2 * This file is part of the GROMACS molecular simulation package.
4 * Copyright (c) 2005,2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016,2017, by the GROMACS development team, led by
5 * Mark Abraham, David van der Spoel, Berk Hess, and Erik Lindahl,
6 * and including many others, as listed in the AUTHORS file in the
7 * top-level source directory and at http://www.gromacs.org.
9 * GROMACS is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU Lesser General Public License
11 * as published by the Free Software Foundation; either version 2.1
12 * of the License, or (at your option) any later version.
14 * GROMACS is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 * Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public
20 * License along with GROMACS; if not, see
21 * http://www.gnu.org/licenses, or write to the Free Software Foundation,
22 * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
24 * If you want to redistribute modifications to GROMACS, please
25 * consider that scientific software is very special. Version
26 * control is crucial - bugs must be traceable. We will be happy to
27 * consider code for inclusion in the official distribution, but
28 * derived work must not be called official GROMACS. Details are found
29 * in the README & COPYING files - if they are missing, get the
30 * official version at http://www.gromacs.org.
32 * To help us fund GROMACS development, we humbly ask that you cite
33 * the research papers on the package. Check out http://www.gromacs.org.
51 #include "gromacs/domdec/domdec_network.h"
52 #include "gromacs/domdec/ga2la.h"
53 #include "gromacs/ewald/pme.h"
54 #include "gromacs/fileio/gmxfio.h"
55 #include "gromacs/fileio/pdbio.h"
56 #include "gromacs/gmxlib/chargegroup.h"
57 #include "gromacs/gmxlib/network.h"
58 #include "gromacs/gmxlib/nrnb.h"
59 #include "gromacs/gpu_utils/gpu_utils.h"
60 #include "gromacs/hardware/hw_info.h"
61 #include "gromacs/imd/imd.h"
62 #include "gromacs/listed-forces/manage-threading.h"
63 #include "gromacs/math/functions.h"
64 #include "gromacs/math/vec.h"
65 #include "gromacs/math/vectypes.h"
66 #include "gromacs/mdlib/constr.h"
67 #include "gromacs/mdlib/force.h"
68 #include "gromacs/mdlib/forcerec.h"
69 #include "gromacs/mdlib/genborn.h"
70 #include "gromacs/mdlib/gmx_omp_nthreads.h"
71 #include "gromacs/mdlib/mdatoms.h"
72 #include "gromacs/mdlib/mdrun.h"
73 #include "gromacs/mdlib/mdsetup.h"
74 #include "gromacs/mdlib/nb_verlet.h"
75 #include "gromacs/mdlib/nbnxn_grid.h"
76 #include "gromacs/mdlib/nsgrid.h"
77 #include "gromacs/mdlib/vsite.h"
78 #include "gromacs/mdtypes/commrec.h"
79 #include "gromacs/mdtypes/df_history.h"
80 #include "gromacs/mdtypes/forcerec.h"
81 #include "gromacs/mdtypes/inputrec.h"
82 #include "gromacs/mdtypes/md_enums.h"
83 #include "gromacs/mdtypes/mdatom.h"
84 #include "gromacs/mdtypes/nblist.h"
85 #include "gromacs/mdtypes/state.h"
86 #include "gromacs/pbcutil/ishift.h"
87 #include "gromacs/pbcutil/pbc.h"
88 #include "gromacs/pulling/pull.h"
89 #include "gromacs/pulling/pull_rotation.h"
90 #include "gromacs/swap/swapcoords.h"
91 #include "gromacs/timing/wallcycle.h"
92 #include "gromacs/topology/block.h"
93 #include "gromacs/topology/idef.h"
94 #include "gromacs/topology/ifunc.h"
95 #include "gromacs/topology/mtop_lookup.h"
96 #include "gromacs/topology/mtop_util.h"
97 #include "gromacs/topology/topology.h"
98 #include "gromacs/utility/basedefinitions.h"
99 #include "gromacs/utility/basenetwork.h"
100 #include "gromacs/utility/cstringutil.h"
101 #include "gromacs/utility/exceptions.h"
102 #include "gromacs/utility/fatalerror.h"
103 #include "gromacs/utility/gmxmpi.h"
104 #include "gromacs/utility/qsort_threadsafe.h"
105 #include "gromacs/utility/real.h"
106 #include "gromacs/utility/smalloc.h"
107 #include "gromacs/utility/stringutil.h"
109 #include "domdec_constraints.h"
110 #include "domdec_internal.h"
111 #include "domdec_vsite.h"
113 #define DDRANK(dd, rank) (rank)
114 #define DDMASTERRANK(dd) (dd->masterrank)
116 struct gmx_domdec_master_t
118 /* The cell boundaries */
120 /* The global charge group division */
121 int *ncg
; /* Number of home charge groups for each node */
122 int *index
; /* Index of nnodes+1 into cg */
123 int *cg
; /* Global charge group index */
124 int *nat
; /* Number of home atoms for each node. */
125 int *ibuf
; /* Buffer for communication */
126 rvec
*vbuf
; /* Buffer for state scattering and gathering */
129 #define DD_NLOAD_MAX 9
131 const char *edlbs_names
[edlbsNR
] = { "off", "auto", "locked", "on", "on" };
133 /* The size per charge group of the cggl_flag buffer in gmx_domdec_comm_t */
136 /* The flags for the cggl_flag buffer in gmx_domdec_comm_t */
137 #define DD_FLAG_NRCG 65535
138 #define DD_FLAG_FW(d) (1<<(16+(d)*2))
139 #define DD_FLAG_BW(d) (1<<(16+(d)*2+1))
141 /* The DD zone order */
142 static const ivec dd_zo
[DD_MAXZONE
] =
143 {{0, 0, 0}, {1, 0, 0}, {1, 1, 0}, {0, 1, 0}, {0, 1, 1}, {0, 0, 1}, {1, 0, 1}, {1, 1, 1}};
145 /* The non-bonded zone-pair setup for domain decomposition
146 * The first number is the i-zone, the second number the first j-zone seen by
147 * this i-zone, the third number the last+1 j-zone seen by this i-zone.
148 * As is, this is for 3D decomposition, where there are 4 i-zones.
149 * With 2D decomposition use only the first 2 i-zones and a last+1 j-zone of 4.
150 * With 1D decomposition use only the first i-zone and a last+1 j-zone of 2.
153 ddNonbondedZonePairRanges
[DD_MAXIZONE
][3] = {{0, 0, 8},
158 /* Factors used to avoid problems due to rounding issues */
159 #define DD_CELL_MARGIN 1.0001
160 #define DD_CELL_MARGIN2 1.00005
161 /* Factor to account for pressure scaling during nstlist steps */
162 #define DD_PRES_SCALE_MARGIN 1.02
164 /* Turn on DLB when the load imbalance causes this amount of total loss.
165 * There is a bit of overhead with DLB and it's difficult to achieve
166 * a load imbalance of less than 2% with DLB.
168 #define DD_PERF_LOSS_DLB_ON 0.02
170 /* Warn about imbalance due to PP or PP/PME load imbalance at this loss */
171 #define DD_PERF_LOSS_WARN 0.05
173 #define DD_CELL_F_SIZE(dd, di) ((dd)->nc[(dd)->dim[(di)]]+1+(di)*2+1+(di))
175 /* Use separate MPI send and receive commands
176 * when nnodes <= GMX_DD_NNODES_SENDRECV.
177 * This saves memory (and some copying for small nnodes).
178 * For high parallelization scatter and gather calls are used.
180 #define GMX_DD_NNODES_SENDRECV 4
183 /* We check if to turn on DLB at the first and every 100 DD partitionings.
184 * With large imbalance DLB will turn on at the first step, so we can
185 * make the interval so large that the MPI overhead of the check is negligible.
187 static const int c_checkTurnDlbOnInterval
= 100;
188 /* We need to check if DLB results in worse performance and then turn it off.
189 * We check this more often then for turning DLB on, because the DLB can scale
190 * the domains very rapidly, so if unlucky the load imbalance can go up quickly
191 * and furthermore, we are already synchronizing often with DLB, so
192 * the overhead of the MPI Bcast is not that high.
194 static const int c_checkTurnDlbOffInterval
= 20;
196 /* Forward declaration */
197 static void dd_dlb_set_should_check_whether_to_turn_dlb_on(gmx_domdec_t
*dd
, gmx_bool bValue
);
201 #define dd_index(n,i) ((((i)[ZZ]*(n)[YY] + (i)[YY])*(n)[XX]) + (i)[XX])
203 static void index2xyz(ivec nc,int ind,ivec xyz)
205 xyz[XX] = ind % nc[XX];
206 xyz[YY] = (ind / nc[XX]) % nc[YY];
207 xyz[ZZ] = ind / (nc[YY]*nc[XX]);
211 /* This order is required to minimize the coordinate communication in PME
212 * which uses decomposition in the x direction.
214 #define dd_index(n, i) ((((i)[XX]*(n)[YY] + (i)[YY])*(n)[ZZ]) + (i)[ZZ])
216 static void ddindex2xyz(ivec nc
, int ind
, ivec xyz
)
218 xyz
[XX
] = ind
/ (nc
[YY
]*nc
[ZZ
]);
219 xyz
[YY
] = (ind
/ nc
[ZZ
]) % nc
[YY
];
220 xyz
[ZZ
] = ind
% nc
[ZZ
];
223 static int ddcoord2ddnodeid(gmx_domdec_t
*dd
, ivec c
)
228 ddindex
= dd_index(dd
->nc
, c
);
229 if (dd
->comm
->bCartesianPP_PME
)
231 ddnodeid
= dd
->comm
->ddindex2ddnodeid
[ddindex
];
233 else if (dd
->comm
->bCartesianPP
)
236 MPI_Cart_rank(dd
->mpi_comm_all
, c
, &ddnodeid
);
247 static gmx_bool
dynamic_dd_box(const gmx_ddbox_t
*ddbox
, const t_inputrec
*ir
)
249 return (ddbox
->nboundeddim
< DIM
|| inputrecDynamicBox(ir
));
252 int ddglatnr(const gmx_domdec_t
*dd
, int i
)
262 if (i
>= dd
->comm
->nat
[ddnatNR
-1])
264 gmx_fatal(FARGS
, "glatnr called with %d, which is larger than the local number of atoms (%d)", i
, dd
->comm
->nat
[ddnatNR
-1]);
266 atnr
= dd
->gatindex
[i
] + 1;
272 t_block
*dd_charge_groups_global(gmx_domdec_t
*dd
)
274 return &dd
->comm
->cgs_gl
;
277 /*! \brief Returns true if the DLB state indicates that the balancer is on. */
278 static bool isDlbOn(const gmx_domdec_comm_t
*comm
)
280 return (comm
->dlbState
== edlbsOnCanTurnOff
||
281 comm
->dlbState
== edlbsOnUser
);
283 /*! \brief Returns true if the DLB state indicates that the balancer is off/disabled.
285 static bool isDlbDisabled(const gmx_domdec_comm_t
*comm
)
287 return (comm
->dlbState
== edlbsOffUser
||
288 comm
->dlbState
== edlbsOffForever
);
291 static void vec_rvec_init(vec_rvec_t
*v
)
297 static void vec_rvec_check_alloc(vec_rvec_t
*v
, int n
)
301 v
->nalloc
= over_alloc_dd(n
);
302 srenew(v
->v
, v
->nalloc
);
306 void dd_store_state(gmx_domdec_t
*dd
, t_state
*state
)
310 if (state
->ddp_count
!= dd
->ddp_count
)
312 gmx_incons("The MD state does not match the domain decomposition state");
315 state
->cg_gl
.resize(dd
->ncg_home
);
316 for (i
= 0; i
< dd
->ncg_home
; i
++)
318 state
->cg_gl
[i
] = dd
->index_gl
[i
];
321 state
->ddp_count_cg_gl
= dd
->ddp_count
;
324 gmx_domdec_zones_t
*domdec_zones(gmx_domdec_t
*dd
)
326 return &dd
->comm
->zones
;
329 void dd_get_ns_ranges(const gmx_domdec_t
*dd
, int icg
,
330 int *jcg0
, int *jcg1
, ivec shift0
, ivec shift1
)
332 gmx_domdec_zones_t
*zones
;
335 zones
= &dd
->comm
->zones
;
338 while (icg
>= zones
->izone
[izone
].cg1
)
347 else if (izone
< zones
->nizone
)
349 *jcg0
= zones
->izone
[izone
].jcg0
;
353 gmx_fatal(FARGS
, "DD icg %d out of range: izone (%d) >= nizone (%d)",
354 icg
, izone
, zones
->nizone
);
357 *jcg1
= zones
->izone
[izone
].jcg1
;
359 for (d
= 0; d
< dd
->ndim
; d
++)
362 shift0
[dim
] = zones
->izone
[izone
].shift0
[dim
];
363 shift1
[dim
] = zones
->izone
[izone
].shift1
[dim
];
364 if (dd
->comm
->tric_dir
[dim
] || (isDlbOn(dd
->comm
) && d
> 0))
366 /* A conservative approach, this can be optimized */
373 int dd_natoms_mdatoms(const gmx_domdec_t
*dd
)
375 /* We currently set mdatoms entries for all atoms:
376 * local + non-local + communicated for vsite + constraints
379 return dd
->comm
->nat
[ddnatNR
- 1];
382 int dd_natoms_vsite(const gmx_domdec_t
*dd
)
384 return dd
->comm
->nat
[ddnatVSITE
];
387 void dd_get_constraint_range(const gmx_domdec_t
*dd
, int *at_start
, int *at_end
)
389 *at_start
= dd
->comm
->nat
[ddnatCON
-1];
390 *at_end
= dd
->comm
->nat
[ddnatCON
];
393 void dd_move_x(gmx_domdec_t
*dd
, matrix box
, rvec x
[])
395 int nzone
, nat_tot
, n
, d
, p
, i
, j
, at0
, at1
, zone
;
396 int *index
, *cgindex
;
397 gmx_domdec_comm_t
*comm
;
398 gmx_domdec_comm_dim_t
*cd
;
399 gmx_domdec_ind_t
*ind
;
400 rvec shift
= {0, 0, 0}, *buf
, *rbuf
;
401 gmx_bool bPBC
, bScrew
;
405 cgindex
= dd
->cgindex
;
410 nat_tot
= dd
->nat_home
;
411 for (d
= 0; d
< dd
->ndim
; d
++)
413 bPBC
= (dd
->ci
[dd
->dim
[d
]] == 0);
414 bScrew
= (bPBC
&& dd
->bScrewPBC
&& dd
->dim
[d
] == XX
);
417 copy_rvec(box
[dd
->dim
[d
]], shift
);
420 for (p
= 0; p
< cd
->np
; p
++)
427 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
429 at0
= cgindex
[index
[i
]];
430 at1
= cgindex
[index
[i
]+1];
431 for (j
= at0
; j
< at1
; j
++)
433 copy_rvec(x
[j
], buf
[n
]);
440 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
442 at0
= cgindex
[index
[i
]];
443 at1
= cgindex
[index
[i
]+1];
444 for (j
= at0
; j
< at1
; j
++)
446 /* We need to shift the coordinates */
447 rvec_add(x
[j
], shift
, buf
[n
]);
454 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
456 at0
= cgindex
[index
[i
]];
457 at1
= cgindex
[index
[i
]+1];
458 for (j
= at0
; j
< at1
; j
++)
461 buf
[n
][XX
] = x
[j
][XX
] + shift
[XX
];
463 * This operation requires a special shift force
464 * treatment, which is performed in calc_vir.
466 buf
[n
][YY
] = box
[YY
][YY
] - x
[j
][YY
];
467 buf
[n
][ZZ
] = box
[ZZ
][ZZ
] - x
[j
][ZZ
];
479 rbuf
= comm
->vbuf2
.v
;
481 /* Send and receive the coordinates */
482 dd_sendrecv_rvec(dd
, d
, dddirBackward
,
483 buf
, ind
->nsend
[nzone
+1],
484 rbuf
, ind
->nrecv
[nzone
+1]);
488 for (zone
= 0; zone
< nzone
; zone
++)
490 for (i
= ind
->cell2at0
[zone
]; i
< ind
->cell2at1
[zone
]; i
++)
492 copy_rvec(rbuf
[j
], x
[i
]);
497 nat_tot
+= ind
->nrecv
[nzone
+1];
503 void dd_move_f(gmx_domdec_t
*dd
, rvec f
[], rvec
*fshift
)
505 int nzone
, nat_tot
, n
, d
, p
, i
, j
, at0
, at1
, zone
;
506 int *index
, *cgindex
;
507 gmx_domdec_comm_t
*comm
;
508 gmx_domdec_comm_dim_t
*cd
;
509 gmx_domdec_ind_t
*ind
;
513 gmx_bool bShiftForcesNeedPbc
, bScrew
;
517 cgindex
= dd
->cgindex
;
521 nzone
= comm
->zones
.n
/2;
522 nat_tot
= dd
->nat_tot
;
523 for (d
= dd
->ndim
-1; d
>= 0; d
--)
525 /* Only forces in domains near the PBC boundaries need to
526 consider PBC in the treatment of fshift */
527 bShiftForcesNeedPbc
= (dd
->ci
[dd
->dim
[d
]] == 0);
528 bScrew
= (bShiftForcesNeedPbc
&& dd
->bScrewPBC
&& dd
->dim
[d
] == XX
);
529 if (fshift
== nullptr && !bScrew
)
531 bShiftForcesNeedPbc
= FALSE
;
533 /* Determine which shift vector we need */
539 for (p
= cd
->np
-1; p
>= 0; p
--)
542 nat_tot
-= ind
->nrecv
[nzone
+1];
549 sbuf
= comm
->vbuf2
.v
;
551 for (zone
= 0; zone
< nzone
; zone
++)
553 for (i
= ind
->cell2at0
[zone
]; i
< ind
->cell2at1
[zone
]; i
++)
555 copy_rvec(f
[i
], sbuf
[j
]);
560 /* Communicate the forces */
561 dd_sendrecv_rvec(dd
, d
, dddirForward
,
562 sbuf
, ind
->nrecv
[nzone
+1],
563 buf
, ind
->nsend
[nzone
+1]);
565 /* Add the received forces */
567 if (!bShiftForcesNeedPbc
)
569 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
571 at0
= cgindex
[index
[i
]];
572 at1
= cgindex
[index
[i
]+1];
573 for (j
= at0
; j
< at1
; j
++)
575 rvec_inc(f
[j
], buf
[n
]);
582 /* fshift should always be defined if this function is
583 * called when bShiftForcesNeedPbc is true */
584 assert(NULL
!= fshift
);
585 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
587 at0
= cgindex
[index
[i
]];
588 at1
= cgindex
[index
[i
]+1];
589 for (j
= at0
; j
< at1
; j
++)
591 rvec_inc(f
[j
], buf
[n
]);
592 /* Add this force to the shift force */
593 rvec_inc(fshift
[is
], buf
[n
]);
600 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
602 at0
= cgindex
[index
[i
]];
603 at1
= cgindex
[index
[i
]+1];
604 for (j
= at0
; j
< at1
; j
++)
606 /* Rotate the force */
607 f
[j
][XX
] += buf
[n
][XX
];
608 f
[j
][YY
] -= buf
[n
][YY
];
609 f
[j
][ZZ
] -= buf
[n
][ZZ
];
612 /* Add this force to the shift force */
613 rvec_inc(fshift
[is
], buf
[n
]);
624 void dd_atom_spread_real(gmx_domdec_t
*dd
, real v
[])
626 int nzone
, nat_tot
, n
, d
, p
, i
, j
, at0
, at1
, zone
;
627 int *index
, *cgindex
;
628 gmx_domdec_comm_t
*comm
;
629 gmx_domdec_comm_dim_t
*cd
;
630 gmx_domdec_ind_t
*ind
;
635 cgindex
= dd
->cgindex
;
637 buf
= &comm
->vbuf
.v
[0][0];
640 nat_tot
= dd
->nat_home
;
641 for (d
= 0; d
< dd
->ndim
; d
++)
644 for (p
= 0; p
< cd
->np
; p
++)
649 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
651 at0
= cgindex
[index
[i
]];
652 at1
= cgindex
[index
[i
]+1];
653 for (j
= at0
; j
< at1
; j
++)
666 rbuf
= &comm
->vbuf2
.v
[0][0];
668 /* Send and receive the coordinates */
669 dd_sendrecv_real(dd
, d
, dddirBackward
,
670 buf
, ind
->nsend
[nzone
+1],
671 rbuf
, ind
->nrecv
[nzone
+1]);
675 for (zone
= 0; zone
< nzone
; zone
++)
677 for (i
= ind
->cell2at0
[zone
]; i
< ind
->cell2at1
[zone
]; i
++)
684 nat_tot
+= ind
->nrecv
[nzone
+1];
690 void dd_atom_sum_real(gmx_domdec_t
*dd
, real v
[])
692 int nzone
, nat_tot
, n
, d
, p
, i
, j
, at0
, at1
, zone
;
693 int *index
, *cgindex
;
694 gmx_domdec_comm_t
*comm
;
695 gmx_domdec_comm_dim_t
*cd
;
696 gmx_domdec_ind_t
*ind
;
701 cgindex
= dd
->cgindex
;
703 buf
= &comm
->vbuf
.v
[0][0];
705 nzone
= comm
->zones
.n
/2;
706 nat_tot
= dd
->nat_tot
;
707 for (d
= dd
->ndim
-1; d
>= 0; d
--)
710 for (p
= cd
->np
-1; p
>= 0; p
--)
713 nat_tot
-= ind
->nrecv
[nzone
+1];
720 sbuf
= &comm
->vbuf2
.v
[0][0];
722 for (zone
= 0; zone
< nzone
; zone
++)
724 for (i
= ind
->cell2at0
[zone
]; i
< ind
->cell2at1
[zone
]; i
++)
731 /* Communicate the forces */
732 dd_sendrecv_real(dd
, d
, dddirForward
,
733 sbuf
, ind
->nrecv
[nzone
+1],
734 buf
, ind
->nsend
[nzone
+1]);
736 /* Add the received forces */
738 for (i
= 0; i
< ind
->nsend
[nzone
]; i
++)
740 at0
= cgindex
[index
[i
]];
741 at1
= cgindex
[index
[i
]+1];
742 for (j
= at0
; j
< at1
; j
++)
753 static void print_ddzone(FILE *fp
, int d
, int i
, int j
, gmx_ddzone_t
*zone
)
755 fprintf(fp
, "zone d0 %d d1 %d d2 %d min0 %6.3f max1 %6.3f mch0 %6.3f mch1 %6.3f p1_0 %6.3f p1_1 %6.3f\n",
757 zone
->min0
, zone
->max1
,
758 zone
->mch0
, zone
->mch0
,
759 zone
->p1_0
, zone
->p1_1
);
763 #define DDZONECOMM_MAXZONE 5
764 #define DDZONECOMM_BUFSIZE 3
766 static void dd_sendrecv_ddzone(const gmx_domdec_t
*dd
,
767 int ddimind
, int direction
,
768 gmx_ddzone_t
*buf_s
, int n_s
,
769 gmx_ddzone_t
*buf_r
, int n_r
)
771 #define ZBS DDZONECOMM_BUFSIZE
772 rvec vbuf_s
[DDZONECOMM_MAXZONE
*ZBS
];
773 rvec vbuf_r
[DDZONECOMM_MAXZONE
*ZBS
];
776 for (i
= 0; i
< n_s
; i
++)
778 vbuf_s
[i
*ZBS
][0] = buf_s
[i
].min0
;
779 vbuf_s
[i
*ZBS
][1] = buf_s
[i
].max1
;
780 vbuf_s
[i
*ZBS
][2] = buf_s
[i
].min1
;
781 vbuf_s
[i
*ZBS
+1][0] = buf_s
[i
].mch0
;
782 vbuf_s
[i
*ZBS
+1][1] = buf_s
[i
].mch1
;
783 vbuf_s
[i
*ZBS
+1][2] = 0;
784 vbuf_s
[i
*ZBS
+2][0] = buf_s
[i
].p1_0
;
785 vbuf_s
[i
*ZBS
+2][1] = buf_s
[i
].p1_1
;
786 vbuf_s
[i
*ZBS
+2][2] = 0;
789 dd_sendrecv_rvec(dd
, ddimind
, direction
,
793 for (i
= 0; i
< n_r
; i
++)
795 buf_r
[i
].min0
= vbuf_r
[i
*ZBS
][0];
796 buf_r
[i
].max1
= vbuf_r
[i
*ZBS
][1];
797 buf_r
[i
].min1
= vbuf_r
[i
*ZBS
][2];
798 buf_r
[i
].mch0
= vbuf_r
[i
*ZBS
+1][0];
799 buf_r
[i
].mch1
= vbuf_r
[i
*ZBS
+1][1];
800 buf_r
[i
].p1_0
= vbuf_r
[i
*ZBS
+2][0];
801 buf_r
[i
].p1_1
= vbuf_r
[i
*ZBS
+2][1];
807 static void dd_move_cellx(gmx_domdec_t
*dd
, gmx_ddbox_t
*ddbox
,
808 rvec cell_ns_x0
, rvec cell_ns_x1
)
810 int d
, d1
, dim
, pos
, buf_size
, i
, j
, p
, npulse
, npulse_min
;
812 gmx_ddzone_t buf_s
[DDZONECOMM_MAXZONE
];
813 gmx_ddzone_t buf_r
[DDZONECOMM_MAXZONE
];
814 gmx_ddzone_t buf_e
[DDZONECOMM_MAXZONE
];
815 rvec extr_s
[2], extr_r
[2];
817 real dist_d
, c
= 0, det
;
818 gmx_domdec_comm_t
*comm
;
823 for (d
= 1; d
< dd
->ndim
; d
++)
826 zp
= (d
== 1) ? &comm
->zone_d1
[0] : &comm
->zone_d2
[0][0];
827 zp
->min0
= cell_ns_x0
[dim
];
828 zp
->max1
= cell_ns_x1
[dim
];
829 zp
->min1
= cell_ns_x1
[dim
];
830 zp
->mch0
= cell_ns_x0
[dim
];
831 zp
->mch1
= cell_ns_x1
[dim
];
832 zp
->p1_0
= cell_ns_x0
[dim
];
833 zp
->p1_1
= cell_ns_x1
[dim
];
836 for (d
= dd
->ndim
-2; d
>= 0; d
--)
839 bPBC
= (dim
< ddbox
->npbcdim
);
841 /* Use an rvec to store two reals */
842 extr_s
[d
][0] = comm
->cell_f0
[d
+1];
843 extr_s
[d
][1] = comm
->cell_f1
[d
+1];
844 extr_s
[d
][2] = comm
->cell_f1
[d
+1];
847 /* Store the extremes in the backward sending buffer,
848 * so the get updated separately from the forward communication.
850 for (d1
= d
; d1
< dd
->ndim
-1; d1
++)
852 /* We invert the order to be able to use the same loop for buf_e */
853 buf_s
[pos
].min0
= extr_s
[d1
][1];
854 buf_s
[pos
].max1
= extr_s
[d1
][0];
855 buf_s
[pos
].min1
= extr_s
[d1
][2];
858 /* Store the cell corner of the dimension we communicate along */
859 buf_s
[pos
].p1_0
= comm
->cell_x0
[dim
];
864 buf_s
[pos
] = (dd
->ndim
== 2) ? comm
->zone_d1
[0] : comm
->zone_d2
[0][0];
867 if (dd
->ndim
== 3 && d
== 0)
869 buf_s
[pos
] = comm
->zone_d2
[0][1];
871 buf_s
[pos
] = comm
->zone_d1
[0];
875 /* We only need to communicate the extremes
876 * in the forward direction
878 npulse
= comm
->cd
[d
].np
;
881 /* Take the minimum to avoid double communication */
882 npulse_min
= std::min(npulse
, dd
->nc
[dim
]-1-npulse
);
886 /* Without PBC we should really not communicate over
887 * the boundaries, but implementing that complicates
888 * the communication setup and therefore we simply
889 * do all communication, but ignore some data.
893 for (p
= 0; p
< npulse_min
; p
++)
895 /* Communicate the extremes forward */
896 bUse
= (bPBC
|| dd
->ci
[dim
] > 0);
898 dd_sendrecv_rvec(dd
, d
, dddirForward
,
899 extr_s
+d
, dd
->ndim
-d
-1,
900 extr_r
+d
, dd
->ndim
-d
-1);
904 for (d1
= d
; d1
< dd
->ndim
-1; d1
++)
906 extr_s
[d1
][0] = std::max(extr_s
[d1
][0], extr_r
[d1
][0]);
907 extr_s
[d1
][1] = std::min(extr_s
[d1
][1], extr_r
[d1
][1]);
908 extr_s
[d1
][2] = std::min(extr_s
[d1
][2], extr_r
[d1
][2]);
914 for (p
= 0; p
< npulse
; p
++)
916 /* Communicate all the zone information backward */
917 bUse
= (bPBC
|| dd
->ci
[dim
] < dd
->nc
[dim
] - 1);
919 dd_sendrecv_ddzone(dd
, d
, dddirBackward
,
926 for (d1
= d
+1; d1
< dd
->ndim
; d1
++)
928 /* Determine the decrease of maximum required
929 * communication height along d1 due to the distance along d,
930 * this avoids a lot of useless atom communication.
932 dist_d
= comm
->cell_x1
[dim
] - buf_r
[0].p1_0
;
934 if (ddbox
->tric_dir
[dim
])
936 /* c is the off-diagonal coupling between the cell planes
937 * along directions d and d1.
939 c
= ddbox
->v
[dim
][dd
->dim
[d1
]][dim
];
945 det
= (1 + c
*c
)*comm
->cutoff
*comm
->cutoff
- dist_d
*dist_d
;
948 dh
[d1
] = comm
->cutoff
- (c
*dist_d
+ std::sqrt(det
))/(1 + c
*c
);
952 /* A negative value signals out of range */
958 /* Accumulate the extremes over all pulses */
959 for (i
= 0; i
< buf_size
; i
++)
969 buf_e
[i
].min0
= std::min(buf_e
[i
].min0
, buf_r
[i
].min0
);
970 buf_e
[i
].max1
= std::max(buf_e
[i
].max1
, buf_r
[i
].max1
);
971 buf_e
[i
].min1
= std::min(buf_e
[i
].min1
, buf_r
[i
].min1
);
974 if (dd
->ndim
== 3 && d
== 0 && i
== buf_size
- 1)
982 if (bUse
&& dh
[d1
] >= 0)
984 buf_e
[i
].mch0
= std::max(buf_e
[i
].mch0
, buf_r
[i
].mch0
-dh
[d1
]);
985 buf_e
[i
].mch1
= std::max(buf_e
[i
].mch1
, buf_r
[i
].mch1
-dh
[d1
]);
988 /* Copy the received buffer to the send buffer,
989 * to pass the data through with the next pulse.
993 if (((bPBC
|| dd
->ci
[dim
]+npulse
< dd
->nc
[dim
]) && p
== npulse
-1) ||
994 (!bPBC
&& dd
->ci
[dim
]+1+p
== dd
->nc
[dim
]-1))
996 /* Store the extremes */
999 for (d1
= d
; d1
< dd
->ndim
-1; d1
++)
1001 extr_s
[d1
][1] = std::min(extr_s
[d1
][1], buf_e
[pos
].min0
);
1002 extr_s
[d1
][0] = std::max(extr_s
[d1
][0], buf_e
[pos
].max1
);
1003 extr_s
[d1
][2] = std::min(extr_s
[d1
][2], buf_e
[pos
].min1
);
1007 if (d
== 1 || (d
== 0 && dd
->ndim
== 3))
1009 for (i
= d
; i
< 2; i
++)
1011 comm
->zone_d2
[1-d
][i
] = buf_e
[pos
];
1017 comm
->zone_d1
[1] = buf_e
[pos
];
1027 for (i
= 0; i
< 2; i
++)
1031 print_ddzone(debug
, 1, i
, 0, &comm
->zone_d1
[i
]);
1033 cell_ns_x0
[dim
] = std::min(cell_ns_x0
[dim
], comm
->zone_d1
[i
].min0
);
1034 cell_ns_x1
[dim
] = std::max(cell_ns_x1
[dim
], comm
->zone_d1
[i
].max1
);
1040 for (i
= 0; i
< 2; i
++)
1042 for (j
= 0; j
< 2; j
++)
1046 print_ddzone(debug
, 2, i
, j
, &comm
->zone_d2
[i
][j
]);
1048 cell_ns_x0
[dim
] = std::min(cell_ns_x0
[dim
], comm
->zone_d2
[i
][j
].min0
);
1049 cell_ns_x1
[dim
] = std::max(cell_ns_x1
[dim
], comm
->zone_d2
[i
][j
].max1
);
1053 for (d
= 1; d
< dd
->ndim
; d
++)
1055 comm
->cell_f_max0
[d
] = extr_s
[d
-1][0];
1056 comm
->cell_f_min1
[d
] = extr_s
[d
-1][1];
1059 fprintf(debug
, "Cell fraction d %d, max0 %f, min1 %f\n",
1060 d
, comm
->cell_f_max0
[d
], comm
->cell_f_min1
[d
]);
1065 static void dd_collect_cg(gmx_domdec_t
*dd
,
1066 t_state
*state_local
)
1068 gmx_domdec_master_t
*ma
= nullptr;
1069 int buf2
[2], *ibuf
, i
, ncg_home
= 0, *cg
= nullptr, nat_home
= 0;
1071 if (state_local
->ddp_count
== dd
->comm
->master_cg_ddp_count
)
1073 /* The master has the correct distribution */
1077 if (state_local
->ddp_count
== dd
->ddp_count
)
1079 /* The local state and DD are in sync, use the DD indices */
1080 ncg_home
= dd
->ncg_home
;
1082 nat_home
= dd
->nat_home
;
1084 else if (state_local
->ddp_count_cg_gl
== state_local
->ddp_count
)
1086 /* The DD is out of sync with the local state, but we have stored
1087 * the cg indices with the local state, so we can use those.
1091 cgs_gl
= &dd
->comm
->cgs_gl
;
1093 ncg_home
= state_local
->cg_gl
.size();
1094 cg
= state_local
->cg_gl
.data();
1096 for (i
= 0; i
< ncg_home
; i
++)
1098 nat_home
+= cgs_gl
->index
[cg
[i
]+1] - cgs_gl
->index
[cg
[i
]];
1103 gmx_incons("Attempted to collect a vector for a state for which the charge group distribution is unknown");
1117 /* Collect the charge group and atom counts on the master */
1118 dd_gather(dd
, 2*sizeof(int), buf2
, ibuf
);
1123 for (i
= 0; i
< dd
->nnodes
; i
++)
1125 ma
->ncg
[i
] = ma
->ibuf
[2*i
];
1126 ma
->nat
[i
] = ma
->ibuf
[2*i
+1];
1127 ma
->index
[i
+1] = ma
->index
[i
] + ma
->ncg
[i
];
1130 /* Make byte counts and indices */
1131 for (i
= 0; i
< dd
->nnodes
; i
++)
1133 ma
->ibuf
[i
] = ma
->ncg
[i
]*sizeof(int);
1134 ma
->ibuf
[dd
->nnodes
+i
] = ma
->index
[i
]*sizeof(int);
1138 fprintf(debug
, "Initial charge group distribution: ");
1139 for (i
= 0; i
< dd
->nnodes
; i
++)
1141 fprintf(debug
, " %d", ma
->ncg
[i
]);
1143 fprintf(debug
, "\n");
1147 /* Collect the charge group indices on the master */
1149 ncg_home
*sizeof(int), cg
,
1150 DDMASTER(dd
) ? ma
->ibuf
: nullptr,
1151 DDMASTER(dd
) ? ma
->ibuf
+dd
->nnodes
: nullptr,
1152 DDMASTER(dd
) ? ma
->cg
: nullptr);
1154 dd
->comm
->master_cg_ddp_count
= state_local
->ddp_count
;
1157 static void dd_collect_vec_sendrecv(gmx_domdec_t
*dd
,
1158 const rvec
*lv
, rvec
*v
)
1160 gmx_domdec_master_t
*ma
;
1161 int n
, i
, c
, a
, nalloc
= 0;
1162 rvec
*buf
= nullptr;
1170 MPI_Send(const_cast<void *>(static_cast<const void *>(lv
)), dd
->nat_home
*sizeof(rvec
), MPI_BYTE
,
1171 DDMASTERRANK(dd
), dd
->rank
, dd
->mpi_comm_all
);
1176 /* Copy the master coordinates to the global array */
1177 cgs_gl
= &dd
->comm
->cgs_gl
;
1179 n
= DDMASTERRANK(dd
);
1181 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1183 for (c
= cgs_gl
->index
[ma
->cg
[i
]]; c
< cgs_gl
->index
[ma
->cg
[i
]+1]; c
++)
1185 copy_rvec(lv
[a
++], v
[c
]);
1189 for (n
= 0; n
< dd
->nnodes
; n
++)
1193 if (ma
->nat
[n
] > nalloc
)
1195 nalloc
= over_alloc_dd(ma
->nat
[n
]);
1196 srenew(buf
, nalloc
);
1199 MPI_Recv(buf
, ma
->nat
[n
]*sizeof(rvec
), MPI_BYTE
, DDRANK(dd
, n
),
1200 n
, dd
->mpi_comm_all
, MPI_STATUS_IGNORE
);
1203 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1205 for (c
= cgs_gl
->index
[ma
->cg
[i
]]; c
< cgs_gl
->index
[ma
->cg
[i
]+1]; c
++)
1207 copy_rvec(buf
[a
++], v
[c
]);
1216 static void get_commbuffer_counts(gmx_domdec_t
*dd
,
1217 int **counts
, int **disps
)
1219 gmx_domdec_master_t
*ma
;
1224 /* Make the rvec count and displacment arrays */
1226 *disps
= ma
->ibuf
+ dd
->nnodes
;
1227 for (n
= 0; n
< dd
->nnodes
; n
++)
1229 (*counts
)[n
] = ma
->nat
[n
]*sizeof(rvec
);
1230 (*disps
)[n
] = (n
== 0 ? 0 : (*disps
)[n
-1] + (*counts
)[n
-1]);
1234 static void dd_collect_vec_gatherv(gmx_domdec_t
*dd
,
1235 const rvec
*lv
, rvec
*v
)
1237 gmx_domdec_master_t
*ma
;
1238 int *rcounts
= nullptr, *disps
= nullptr;
1240 rvec
*buf
= nullptr;
1247 get_commbuffer_counts(dd
, &rcounts
, &disps
);
1252 dd_gatherv(dd
, dd
->nat_home
*sizeof(rvec
), lv
, rcounts
, disps
, buf
);
1256 cgs_gl
= &dd
->comm
->cgs_gl
;
1259 for (n
= 0; n
< dd
->nnodes
; n
++)
1261 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1263 for (c
= cgs_gl
->index
[ma
->cg
[i
]]; c
< cgs_gl
->index
[ma
->cg
[i
]+1]; c
++)
1265 copy_rvec(buf
[a
++], v
[c
]);
1272 void dd_collect_vec(gmx_domdec_t
*dd
,
1273 t_state
*state_local
,
1274 const PaddedRVecVector
*localVector
,
1277 dd_collect_cg(dd
, state_local
);
1279 const rvec
*lv
= as_rvec_array(localVector
->data());
1281 if (dd
->nnodes
<= GMX_DD_NNODES_SENDRECV
)
1283 dd_collect_vec_sendrecv(dd
, lv
, v
);
1287 dd_collect_vec_gatherv(dd
, lv
, v
);
1291 void dd_collect_vec(gmx_domdec_t
*dd
,
1292 t_state
*state_local
,
1293 const PaddedRVecVector
*localVector
,
1294 PaddedRVecVector
*vector
)
1296 dd_collect_vec(dd
, state_local
, localVector
, as_rvec_array(vector
->data()));
1300 void dd_collect_state(gmx_domdec_t
*dd
,
1301 t_state
*state_local
, t_state
*state
)
1303 int nh
= state
->nhchainlength
;
1307 for (int i
= 0; i
< efptNR
; i
++)
1309 state
->lambda
[i
] = state_local
->lambda
[i
];
1311 state
->fep_state
= state_local
->fep_state
;
1312 state
->veta
= state_local
->veta
;
1313 state
->vol0
= state_local
->vol0
;
1314 copy_mat(state_local
->box
, state
->box
);
1315 copy_mat(state_local
->boxv
, state
->boxv
);
1316 copy_mat(state_local
->svir_prev
, state
->svir_prev
);
1317 copy_mat(state_local
->fvir_prev
, state
->fvir_prev
);
1318 copy_mat(state_local
->pres_prev
, state
->pres_prev
);
1320 for (int i
= 0; i
< state_local
->ngtc
; i
++)
1322 for (int j
= 0; j
< nh
; j
++)
1324 state
->nosehoover_xi
[i
*nh
+j
] = state_local
->nosehoover_xi
[i
*nh
+j
];
1325 state
->nosehoover_vxi
[i
*nh
+j
] = state_local
->nosehoover_vxi
[i
*nh
+j
];
1327 state
->therm_integral
[i
] = state_local
->therm_integral
[i
];
1329 for (int i
= 0; i
< state_local
->nnhpres
; i
++)
1331 for (int j
= 0; j
< nh
; j
++)
1333 state
->nhpres_xi
[i
*nh
+j
] = state_local
->nhpres_xi
[i
*nh
+j
];
1334 state
->nhpres_vxi
[i
*nh
+j
] = state_local
->nhpres_vxi
[i
*nh
+j
];
1337 state
->baros_integral
= state_local
->baros_integral
;
1339 if (state_local
->flags
& (1 << estX
))
1341 dd_collect_vec(dd
, state_local
, &state_local
->x
, &state
->x
);
1343 if (state_local
->flags
& (1 << estV
))
1345 dd_collect_vec(dd
, state_local
, &state_local
->v
, &state
->v
);
1347 if (state_local
->flags
& (1 << estCGP
))
1349 dd_collect_vec(dd
, state_local
, &state_local
->cg_p
, &state
->cg_p
);
1353 static void dd_resize_state(t_state
*state
, PaddedRVecVector
*f
, int natoms
)
1357 fprintf(debug
, "Resizing state: currently %d, required %d\n", state
->natoms
, natoms
);
1360 state_change_natoms(state
, natoms
);
1364 /* We need to allocate one element extra, since we might use
1365 * (unaligned) 4-wide SIMD loads to access rvec entries.
1367 f
->resize(natoms
+ 1);
1371 static void dd_check_alloc_ncg(t_forcerec
*fr
,
1373 PaddedRVecVector
*f
,
1374 int numChargeGroups
)
1376 if (numChargeGroups
> fr
->cg_nalloc
)
1380 fprintf(debug
, "Reallocating forcerec: currently %d, required %d, allocating %d\n", fr
->cg_nalloc
, numChargeGroups
, over_alloc_dd(numChargeGroups
));
1382 fr
->cg_nalloc
= over_alloc_dd(numChargeGroups
);
1383 srenew(fr
->cginfo
, fr
->cg_nalloc
);
1384 if (fr
->cutoff_scheme
== ecutsGROUP
)
1386 srenew(fr
->cg_cm
, fr
->cg_nalloc
);
1389 if (fr
->cutoff_scheme
== ecutsVERLET
)
1391 /* We don't use charge groups, we use x in state to set up
1392 * the atom communication.
1394 dd_resize_state(state
, f
, numChargeGroups
);
1398 static void dd_distribute_vec_sendrecv(gmx_domdec_t
*dd
, t_block
*cgs
,
1401 gmx_domdec_master_t
*ma
;
1402 int n
, i
, c
, a
, nalloc
= 0;
1403 rvec
*buf
= nullptr;
1409 for (n
= 0; n
< dd
->nnodes
; n
++)
1413 if (ma
->nat
[n
] > nalloc
)
1415 nalloc
= over_alloc_dd(ma
->nat
[n
]);
1416 srenew(buf
, nalloc
);
1418 /* Use lv as a temporary buffer */
1420 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1422 for (c
= cgs
->index
[ma
->cg
[i
]]; c
< cgs
->index
[ma
->cg
[i
]+1]; c
++)
1424 copy_rvec(v
[c
], buf
[a
++]);
1427 if (a
!= ma
->nat
[n
])
1429 gmx_fatal(FARGS
, "Internal error a (%d) != nat (%d)",
1434 MPI_Send(buf
, ma
->nat
[n
]*sizeof(rvec
), MPI_BYTE
,
1435 DDRANK(dd
, n
), n
, dd
->mpi_comm_all
);
1440 n
= DDMASTERRANK(dd
);
1442 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1444 for (c
= cgs
->index
[ma
->cg
[i
]]; c
< cgs
->index
[ma
->cg
[i
]+1]; c
++)
1446 copy_rvec(v
[c
], lv
[a
++]);
1453 MPI_Recv(lv
, dd
->nat_home
*sizeof(rvec
), MPI_BYTE
, DDMASTERRANK(dd
),
1454 MPI_ANY_TAG
, dd
->mpi_comm_all
, MPI_STATUS_IGNORE
);
1459 static void dd_distribute_vec_scatterv(gmx_domdec_t
*dd
, t_block
*cgs
,
1462 gmx_domdec_master_t
*ma
;
1463 int *scounts
= nullptr, *disps
= nullptr;
1465 rvec
*buf
= nullptr;
1471 get_commbuffer_counts(dd
, &scounts
, &disps
);
1475 for (n
= 0; n
< dd
->nnodes
; n
++)
1477 for (i
= ma
->index
[n
]; i
< ma
->index
[n
+1]; i
++)
1479 for (c
= cgs
->index
[ma
->cg
[i
]]; c
< cgs
->index
[ma
->cg
[i
]+1]; c
++)
1481 copy_rvec(v
[c
], buf
[a
++]);
1487 dd_scatterv(dd
, scounts
, disps
, buf
, dd
->nat_home
*sizeof(rvec
), lv
);
1490 static void dd_distribute_vec(gmx_domdec_t
*dd
, t_block
*cgs
, rvec
*v
, rvec
*lv
)
1492 if (dd
->nnodes
<= GMX_DD_NNODES_SENDRECV
)
1494 dd_distribute_vec_sendrecv(dd
, cgs
, v
, lv
);
1498 dd_distribute_vec_scatterv(dd
, cgs
, v
, lv
);
1502 static void dd_distribute_dfhist(gmx_domdec_t
*dd
, df_history_t
*dfhist
)
1504 if (dfhist
== nullptr)
1509 dd_bcast(dd
, sizeof(int), &dfhist
->bEquil
);
1510 dd_bcast(dd
, sizeof(int), &dfhist
->nlambda
);
1511 dd_bcast(dd
, sizeof(real
), &dfhist
->wl_delta
);
1513 if (dfhist
->nlambda
> 0)
1515 int nlam
= dfhist
->nlambda
;
1516 dd_bcast(dd
, sizeof(int)*nlam
, dfhist
->n_at_lam
);
1517 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->wl_histo
);
1518 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->sum_weights
);
1519 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->sum_dg
);
1520 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->sum_minvar
);
1521 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->sum_variance
);
1523 for (int i
= 0; i
< nlam
; i
++)
1525 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->accum_p
[i
]);
1526 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->accum_m
[i
]);
1527 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->accum_p2
[i
]);
1528 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->accum_m2
[i
]);
1529 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->Tij
[i
]);
1530 dd_bcast(dd
, sizeof(real
)*nlam
, dfhist
->Tij_empirical
[i
]);
1535 static void dd_distribute_state(gmx_domdec_t
*dd
, t_block
*cgs
,
1536 t_state
*state
, t_state
*state_local
,
1537 PaddedRVecVector
*f
)
1539 int nh
= state
->nhchainlength
;
1543 for (int i
= 0; i
< efptNR
; i
++)
1545 state_local
->lambda
[i
] = state
->lambda
[i
];
1547 state_local
->fep_state
= state
->fep_state
;
1548 state_local
->veta
= state
->veta
;
1549 state_local
->vol0
= state
->vol0
;
1550 copy_mat(state
->box
, state_local
->box
);
1551 copy_mat(state
->box_rel
, state_local
->box_rel
);
1552 copy_mat(state
->boxv
, state_local
->boxv
);
1553 copy_mat(state
->svir_prev
, state_local
->svir_prev
);
1554 copy_mat(state
->fvir_prev
, state_local
->fvir_prev
);
1555 if (state
->dfhist
!= nullptr)
1557 copy_df_history(state_local
->dfhist
, state
->dfhist
);
1559 for (int i
= 0; i
< state_local
->ngtc
; i
++)
1561 for (int j
= 0; j
< nh
; j
++)
1563 state_local
->nosehoover_xi
[i
*nh
+j
] = state
->nosehoover_xi
[i
*nh
+j
];
1564 state_local
->nosehoover_vxi
[i
*nh
+j
] = state
->nosehoover_vxi
[i
*nh
+j
];
1566 state_local
->therm_integral
[i
] = state
->therm_integral
[i
];
1568 for (int i
= 0; i
< state_local
->nnhpres
; i
++)
1570 for (int j
= 0; j
< nh
; j
++)
1572 state_local
->nhpres_xi
[i
*nh
+j
] = state
->nhpres_xi
[i
*nh
+j
];
1573 state_local
->nhpres_vxi
[i
*nh
+j
] = state
->nhpres_vxi
[i
*nh
+j
];
1576 state_local
->baros_integral
= state
->baros_integral
;
1578 dd_bcast(dd
, ((efptNR
)*sizeof(real
)), state_local
->lambda
.data());
1579 dd_bcast(dd
, sizeof(int), &state_local
->fep_state
);
1580 dd_bcast(dd
, sizeof(real
), &state_local
->veta
);
1581 dd_bcast(dd
, sizeof(real
), &state_local
->vol0
);
1582 dd_bcast(dd
, sizeof(state_local
->box
), state_local
->box
);
1583 dd_bcast(dd
, sizeof(state_local
->box_rel
), state_local
->box_rel
);
1584 dd_bcast(dd
, sizeof(state_local
->boxv
), state_local
->boxv
);
1585 dd_bcast(dd
, sizeof(state_local
->svir_prev
), state_local
->svir_prev
);
1586 dd_bcast(dd
, sizeof(state_local
->fvir_prev
), state_local
->fvir_prev
);
1587 dd_bcast(dd
, ((state_local
->ngtc
*nh
)*sizeof(double)), state_local
->nosehoover_xi
.data());
1588 dd_bcast(dd
, ((state_local
->ngtc
*nh
)*sizeof(double)), state_local
->nosehoover_vxi
.data());
1589 dd_bcast(dd
, state_local
->ngtc
*sizeof(double), state_local
->therm_integral
.data());
1590 dd_bcast(dd
, ((state_local
->nnhpres
*nh
)*sizeof(double)), state_local
->nhpres_xi
.data());
1591 dd_bcast(dd
, ((state_local
->nnhpres
*nh
)*sizeof(double)), state_local
->nhpres_vxi
.data());
1593 /* communicate df_history -- required for restarting from checkpoint */
1594 dd_distribute_dfhist(dd
, state_local
->dfhist
);
1596 dd_resize_state(state_local
, f
, dd
->nat_home
);
1598 if (state_local
->flags
& (1 << estX
))
1600 dd_distribute_vec(dd
, cgs
, as_rvec_array(state
->x
.data()), as_rvec_array(state_local
->x
.data()));
1602 if (state_local
->flags
& (1 << estV
))
1604 dd_distribute_vec(dd
, cgs
, as_rvec_array(state
->v
.data()), as_rvec_array(state_local
->v
.data()));
1606 if (state_local
->flags
& (1 << estCGP
))
1608 dd_distribute_vec(dd
, cgs
, as_rvec_array(state
->cg_p
.data()), as_rvec_array(state_local
->cg_p
.data()));
1612 static char dim2char(int dim
)
1618 case XX
: c
= 'X'; break;
1619 case YY
: c
= 'Y'; break;
1620 case ZZ
: c
= 'Z'; break;
1621 default: gmx_fatal(FARGS
, "Unknown dim %d", dim
);
1627 static void write_dd_grid_pdb(const char *fn
, gmx_int64_t step
,
1628 gmx_domdec_t
*dd
, matrix box
, gmx_ddbox_t
*ddbox
)
1630 rvec grid_s
[2], *grid_r
= nullptr, cx
, r
;
1631 char fname
[STRLEN
], buf
[22];
1633 int a
, i
, d
, z
, y
, x
;
1637 copy_rvec(dd
->comm
->cell_x0
, grid_s
[0]);
1638 copy_rvec(dd
->comm
->cell_x1
, grid_s
[1]);
1642 snew(grid_r
, 2*dd
->nnodes
);
1645 dd_gather(dd
, 2*sizeof(rvec
), grid_s
, DDMASTER(dd
) ? grid_r
: nullptr);
1649 for (d
= 0; d
< DIM
; d
++)
1651 for (i
= 0; i
< DIM
; i
++)
1659 if (d
< ddbox
->npbcdim
&& dd
->nc
[d
] > 1)
1661 tric
[d
][i
] = box
[i
][d
]/box
[i
][i
];
1670 sprintf(fname
, "%s_%s.pdb", fn
, gmx_step_str(step
, buf
));
1671 out
= gmx_fio_fopen(fname
, "w");
1672 gmx_write_pdb_box(out
, dd
->bScrewPBC
? epbcSCREW
: epbcXYZ
, box
);
1674 for (i
= 0; i
< dd
->nnodes
; i
++)
1676 vol
= dd
->nnodes
/(box
[XX
][XX
]*box
[YY
][YY
]*box
[ZZ
][ZZ
]);
1677 for (d
= 0; d
< DIM
; d
++)
1679 vol
*= grid_r
[i
*2+1][d
] - grid_r
[i
*2][d
];
1681 for (z
= 0; z
< 2; z
++)
1683 for (y
= 0; y
< 2; y
++)
1685 for (x
= 0; x
< 2; x
++)
1687 cx
[XX
] = grid_r
[i
*2+x
][XX
];
1688 cx
[YY
] = grid_r
[i
*2+y
][YY
];
1689 cx
[ZZ
] = grid_r
[i
*2+z
][ZZ
];
1691 gmx_fprintf_pdb_atomline(out
, epdbATOM
, a
++, "CA", ' ', "GLY", ' ', i
+1, ' ',
1692 10*r
[XX
], 10*r
[YY
], 10*r
[ZZ
], 1.0, vol
, "");
1696 for (d
= 0; d
< DIM
; d
++)
1698 for (x
= 0; x
< 4; x
++)
1702 case 0: y
= 1 + i
*8 + 2*x
; break;
1703 case 1: y
= 1 + i
*8 + 2*x
- (x
% 2); break;
1704 case 2: y
= 1 + i
*8 + x
; break;
1706 fprintf(out
, "%6s%5d%5d\n", "CONECT", y
, y
+(1<<d
));
1710 gmx_fio_fclose(out
);
1715 void write_dd_pdb(const char *fn
, gmx_int64_t step
, const char *title
,
1716 const gmx_mtop_t
*mtop
, t_commrec
*cr
,
1717 int natoms
, rvec x
[], matrix box
)
1719 char fname
[STRLEN
], buf
[22];
1721 int i
, ii
, resnr
, c
;
1722 const char *atomname
, *resname
;
1729 natoms
= dd
->comm
->nat
[ddnatVSITE
];
1732 sprintf(fname
, "%s_%s_n%d.pdb", fn
, gmx_step_str(step
, buf
), cr
->sim_nodeid
);
1734 out
= gmx_fio_fopen(fname
, "w");
1736 fprintf(out
, "TITLE %s\n", title
);
1737 gmx_write_pdb_box(out
, dd
->bScrewPBC
? epbcSCREW
: epbcXYZ
, box
);
1739 for (i
= 0; i
< natoms
; i
++)
1741 ii
= dd
->gatindex
[i
];
1742 mtopGetAtomAndResidueName(mtop
, ii
, &molb
, &atomname
, &resnr
, &resname
, nullptr);
1743 if (i
< dd
->comm
->nat
[ddnatZONE
])
1746 while (i
>= dd
->cgindex
[dd
->comm
->zones
.cg_range
[c
+1]])
1752 else if (i
< dd
->comm
->nat
[ddnatVSITE
])
1754 b
= dd
->comm
->zones
.n
;
1758 b
= dd
->comm
->zones
.n
+ 1;
1760 gmx_fprintf_pdb_atomline(out
, epdbATOM
, ii
+1, atomname
, ' ', resname
, ' ', resnr
, ' ',
1761 10*x
[i
][XX
], 10*x
[i
][YY
], 10*x
[i
][ZZ
], 1.0, b
, "");
1763 fprintf(out
, "TER\n");
1765 gmx_fio_fclose(out
);
1768 real
dd_cutoff_multibody(const gmx_domdec_t
*dd
)
1770 gmx_domdec_comm_t
*comm
;
1777 if (comm
->bInterCGBondeds
)
1779 if (comm
->cutoff_mbody
> 0)
1781 r
= comm
->cutoff_mbody
;
1785 /* cutoff_mbody=0 means we do not have DLB */
1786 r
= comm
->cellsize_min
[dd
->dim
[0]];
1787 for (di
= 1; di
< dd
->ndim
; di
++)
1789 r
= std::min(r
, comm
->cellsize_min
[dd
->dim
[di
]]);
1791 if (comm
->bBondComm
)
1793 r
= std::max(r
, comm
->cutoff_mbody
);
1797 r
= std::min(r
, comm
->cutoff
);
1805 real
dd_cutoff_twobody(const gmx_domdec_t
*dd
)
1809 r_mb
= dd_cutoff_multibody(dd
);
1811 return std::max(dd
->comm
->cutoff
, r_mb
);
1815 static void dd_cart_coord2pmecoord(const gmx_domdec_t
*dd
, const ivec coord
,
1820 nc
= dd
->nc
[dd
->comm
->cartpmedim
];
1821 ntot
= dd
->comm
->ntot
[dd
->comm
->cartpmedim
];
1822 copy_ivec(coord
, coord_pme
);
1823 coord_pme
[dd
->comm
->cartpmedim
] =
1824 nc
+ (coord
[dd
->comm
->cartpmedim
]*(ntot
- nc
) + (ntot
- nc
)/2)/nc
;
1827 static int ddindex2pmeindex(const gmx_domdec_t
*dd
, int ddindex
)
1832 npme
= dd
->comm
->npmenodes
;
1834 /* Here we assign a PME node to communicate with this DD node
1835 * by assuming that the major index of both is x.
1836 * We add cr->npmenodes/2 to obtain an even distribution.
1838 return (ddindex
*npme
+ npme
/2)/npp
;
1841 static int *dd_interleaved_pme_ranks(const gmx_domdec_t
*dd
)
1846 snew(pme_rank
, dd
->comm
->npmenodes
);
1848 for (i
= 0; i
< dd
->nnodes
; i
++)
1850 p0
= ddindex2pmeindex(dd
, i
);
1851 p1
= ddindex2pmeindex(dd
, i
+1);
1852 if (i
+1 == dd
->nnodes
|| p1
> p0
)
1856 fprintf(debug
, "pme_rank[%d] = %d\n", n
, i
+1+n
);
1858 pme_rank
[n
] = i
+ 1 + n
;
1866 static int gmx_ddcoord2pmeindex(t_commrec
*cr
, int x
, int y
, int z
)
1874 if (dd->comm->bCartesian) {
1875 gmx_ddindex2xyz(dd->nc,ddindex,coords);
1876 dd_coords2pmecoords(dd,coords,coords_pme);
1877 copy_ivec(dd->ntot,nc);
1878 nc[dd->cartpmedim] -= dd->nc[dd->cartpmedim];
1879 coords_pme[dd->cartpmedim] -= dd->nc[dd->cartpmedim];
1881 slab = (coords_pme[XX]*nc[YY] + coords_pme[YY])*nc[ZZ] + coords_pme[ZZ];
1883 slab = (ddindex*cr->npmenodes + cr->npmenodes/2)/dd->nnodes;
1889 slab
= ddindex2pmeindex(dd
, dd_index(dd
->nc
, coords
));
1894 static int ddcoord2simnodeid(t_commrec
*cr
, int x
, int y
, int z
)
1896 gmx_domdec_comm_t
*comm
;
1898 int ddindex
, nodeid
= -1;
1900 comm
= cr
->dd
->comm
;
1905 if (comm
->bCartesianPP_PME
)
1908 MPI_Cart_rank(cr
->mpi_comm_mysim
, coords
, &nodeid
);
1913 ddindex
= dd_index(cr
->dd
->nc
, coords
);
1914 if (comm
->bCartesianPP
)
1916 nodeid
= comm
->ddindex2simnodeid
[ddindex
];
1922 nodeid
= ddindex
+ gmx_ddcoord2pmeindex(cr
, x
, y
, z
);
1934 static int dd_simnode2pmenode(const gmx_domdec_t
*dd
,
1935 const t_commrec gmx_unused
*cr
,
1940 const gmx_domdec_comm_t
*comm
= dd
->comm
;
1942 /* This assumes a uniform x domain decomposition grid cell size */
1943 if (comm
->bCartesianPP_PME
)
1946 ivec coord
, coord_pme
;
1947 MPI_Cart_coords(cr
->mpi_comm_mysim
, sim_nodeid
, DIM
, coord
);
1948 if (coord
[comm
->cartpmedim
] < dd
->nc
[comm
->cartpmedim
])
1950 /* This is a PP node */
1951 dd_cart_coord2pmecoord(dd
, coord
, coord_pme
);
1952 MPI_Cart_rank(cr
->mpi_comm_mysim
, coord_pme
, &pmenode
);
1956 else if (comm
->bCartesianPP
)
1958 if (sim_nodeid
< dd
->nnodes
)
1960 pmenode
= dd
->nnodes
+ ddindex2pmeindex(dd
, sim_nodeid
);
1965 /* This assumes DD cells with identical x coordinates
1966 * are numbered sequentially.
1968 if (dd
->comm
->pmenodes
== nullptr)
1970 if (sim_nodeid
< dd
->nnodes
)
1972 /* The DD index equals the nodeid */
1973 pmenode
= dd
->nnodes
+ ddindex2pmeindex(dd
, sim_nodeid
);
1979 while (sim_nodeid
> dd
->comm
->pmenodes
[i
])
1983 if (sim_nodeid
< dd
->comm
->pmenodes
[i
])
1985 pmenode
= dd
->comm
->pmenodes
[i
];
1993 void get_pme_nnodes(const gmx_domdec_t
*dd
,
1994 int *npmenodes_x
, int *npmenodes_y
)
1998 *npmenodes_x
= dd
->comm
->npmenodes_x
;
1999 *npmenodes_y
= dd
->comm
->npmenodes_y
;
2008 void get_pme_ddnodes(t_commrec
*cr
, int pmenodeid
,
2009 int *nmy_ddnodes
, int **my_ddnodes
, int *node_peer
)
2013 ivec coord
, coord_pme
;
2017 snew(*my_ddnodes
, (dd
->nnodes
+cr
->npmenodes
-1)/cr
->npmenodes
);
2020 for (x
= 0; x
< dd
->nc
[XX
]; x
++)
2022 for (y
= 0; y
< dd
->nc
[YY
]; y
++)
2024 for (z
= 0; z
< dd
->nc
[ZZ
]; z
++)
2026 if (dd
->comm
->bCartesianPP_PME
)
2031 dd_cart_coord2pmecoord(dd
, coord
, coord_pme
);
2032 if (dd
->ci
[XX
] == coord_pme
[XX
] &&
2033 dd
->ci
[YY
] == coord_pme
[YY
] &&
2034 dd
->ci
[ZZ
] == coord_pme
[ZZ
])
2036 (*my_ddnodes
)[(*nmy_ddnodes
)++] = ddcoord2simnodeid(cr
, x
, y
, z
);
2041 /* The slab corresponds to the nodeid in the PME group */
2042 if (gmx_ddcoord2pmeindex(cr
, x
, y
, z
) == pmenodeid
)
2044 (*my_ddnodes
)[(*nmy_ddnodes
)++] = ddcoord2simnodeid(cr
, x
, y
, z
);
2051 /* The last PP-only node is the peer node */
2052 *node_peer
= (*my_ddnodes
)[*nmy_ddnodes
-1];
2056 fprintf(debug
, "Receive coordinates from PP ranks:");
2057 for (x
= 0; x
< *nmy_ddnodes
; x
++)
2059 fprintf(debug
, " %d", (*my_ddnodes
)[x
]);
2061 fprintf(debug
, "\n");
2065 static gmx_bool
receive_vir_ener(const gmx_domdec_t
*dd
, const t_commrec
*cr
)
2067 gmx_bool bReceive
= TRUE
;
2069 if (cr
->npmenodes
< dd
->nnodes
)
2071 gmx_domdec_comm_t
*comm
= dd
->comm
;
2072 if (comm
->bCartesianPP_PME
)
2075 int pmenode
= dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
);
2077 MPI_Cart_coords(cr
->mpi_comm_mysim
, cr
->sim_nodeid
, DIM
, coords
);
2078 coords
[comm
->cartpmedim
]++;
2079 if (coords
[comm
->cartpmedim
] < dd
->nc
[comm
->cartpmedim
])
2082 MPI_Cart_rank(cr
->mpi_comm_mysim
, coords
, &rank
);
2083 if (dd_simnode2pmenode(dd
, cr
, rank
) == pmenode
)
2085 /* This is not the last PP node for pmenode */
2090 GMX_RELEASE_ASSERT(false, "Without MPI we should not have Cartesian PP-PME with #PMEnodes < #DDnodes");
2095 int pmenode
= dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
);
2096 if (cr
->sim_nodeid
+1 < cr
->nnodes
&&
2097 dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
+1) == pmenode
)
2099 /* This is not the last PP node for pmenode */
2108 static void set_zones_ncg_home(gmx_domdec_t
*dd
)
2110 gmx_domdec_zones_t
*zones
;
2113 zones
= &dd
->comm
->zones
;
2115 zones
->cg_range
[0] = 0;
2116 for (i
= 1; i
< zones
->n
+1; i
++)
2118 zones
->cg_range
[i
] = dd
->ncg_home
;
2120 /* zone_ncg1[0] should always be equal to ncg_home */
2121 dd
->comm
->zone_ncg1
[0] = dd
->ncg_home
;
2124 static void rebuild_cgindex(gmx_domdec_t
*dd
,
2125 const int *gcgs_index
, const t_state
*state
)
2127 int * gmx_restrict dd_cg_gl
= dd
->index_gl
;
2128 int * gmx_restrict cgindex
= dd
->cgindex
;
2131 /* Copy back the global charge group indices from state
2132 * and rebuild the local charge group to atom index.
2135 for (unsigned int i
= 0; i
< state
->cg_gl
.size(); i
++)
2138 int cg_gl
= state
->cg_gl
[i
];
2139 dd_cg_gl
[i
] = cg_gl
;
2140 nat
+= gcgs_index
[cg_gl
+1] - gcgs_index
[cg_gl
];
2142 cgindex
[state
->cg_gl
.size()] = nat
;
2144 dd
->ncg_home
= state
->cg_gl
.size();
2147 set_zones_ncg_home(dd
);
2150 static int ddcginfo(const cginfo_mb_t
*cginfo_mb
, int cg
)
2152 while (cg
>= cginfo_mb
->cg_end
)
2157 return cginfo_mb
->cginfo
[(cg
- cginfo_mb
->cg_start
) % cginfo_mb
->cg_mod
];
2160 static void dd_set_cginfo(int *index_gl
, int cg0
, int cg1
,
2161 t_forcerec
*fr
, char *bLocalCG
)
2163 cginfo_mb_t
*cginfo_mb
;
2169 cginfo_mb
= fr
->cginfo_mb
;
2170 cginfo
= fr
->cginfo
;
2172 for (cg
= cg0
; cg
< cg1
; cg
++)
2174 cginfo
[cg
] = ddcginfo(cginfo_mb
, index_gl
[cg
]);
2178 if (bLocalCG
!= nullptr)
2180 for (cg
= cg0
; cg
< cg1
; cg
++)
2182 bLocalCG
[index_gl
[cg
]] = TRUE
;
2187 static void make_dd_indices(gmx_domdec_t
*dd
,
2188 const int *gcgs_index
, int cg_start
)
2190 int nzone
, zone
, zone1
, cg0
, cg1
, cg1_p1
, cg
, cg_gl
, a
, a_gl
;
2191 int *zone2cg
, *zone_ncg1
, *index_gl
, *gatindex
;
2194 if (dd
->nat_tot
> dd
->gatindex_nalloc
)
2196 dd
->gatindex_nalloc
= over_alloc_dd(dd
->nat_tot
);
2197 srenew(dd
->gatindex
, dd
->gatindex_nalloc
);
2200 nzone
= dd
->comm
->zones
.n
;
2201 zone2cg
= dd
->comm
->zones
.cg_range
;
2202 zone_ncg1
= dd
->comm
->zone_ncg1
;
2203 index_gl
= dd
->index_gl
;
2204 gatindex
= dd
->gatindex
;
2205 bCGs
= dd
->comm
->bCGs
;
2207 if (zone2cg
[1] != dd
->ncg_home
)
2209 gmx_incons("dd->ncg_zone is not up to date");
2212 /* Make the local to global and global to local atom index */
2213 a
= dd
->cgindex
[cg_start
];
2214 for (zone
= 0; zone
< nzone
; zone
++)
2222 cg0
= zone2cg
[zone
];
2224 cg1
= zone2cg
[zone
+1];
2225 cg1_p1
= cg0
+ zone_ncg1
[zone
];
2227 for (cg
= cg0
; cg
< cg1
; cg
++)
2232 /* Signal that this cg is from more than one pulse away */
2235 cg_gl
= index_gl
[cg
];
2238 for (a_gl
= gcgs_index
[cg_gl
]; a_gl
< gcgs_index
[cg_gl
+1]; a_gl
++)
2241 ga2la_set(dd
->ga2la
, a_gl
, a
, zone1
);
2247 gatindex
[a
] = cg_gl
;
2248 ga2la_set(dd
->ga2la
, cg_gl
, a
, zone1
);
2255 static int check_bLocalCG(gmx_domdec_t
*dd
, int ncg_sys
, const char *bLocalCG
,
2261 if (bLocalCG
== nullptr)
2265 for (i
= 0; i
< dd
->ncg_tot
; i
++)
2267 if (!bLocalCG
[dd
->index_gl
[i
]])
2270 "DD rank %d, %s: cg %d, global cg %d is not marked in bLocalCG (ncg_home %d)\n", dd
->rank
, where
, i
+1, dd
->index_gl
[i
]+1, dd
->ncg_home
);
2275 for (i
= 0; i
< ncg_sys
; i
++)
2282 if (ngl
!= dd
->ncg_tot
)
2284 fprintf(stderr
, "DD rank %d, %s: In bLocalCG %d cgs are marked as local, whereas there are %d\n", dd
->rank
, where
, ngl
, dd
->ncg_tot
);
2291 static void check_index_consistency(gmx_domdec_t
*dd
,
2292 int natoms_sys
, int ncg_sys
,
2295 int nerr
, ngl
, i
, a
, cell
;
2300 if (dd
->comm
->DD_debug
> 1)
2302 snew(have
, natoms_sys
);
2303 for (a
= 0; a
< dd
->nat_tot
; a
++)
2305 if (have
[dd
->gatindex
[a
]] > 0)
2307 fprintf(stderr
, "DD rank %d: global atom %d occurs twice: index %d and %d\n", dd
->rank
, dd
->gatindex
[a
]+1, have
[dd
->gatindex
[a
]], a
+1);
2311 have
[dd
->gatindex
[a
]] = a
+ 1;
2317 snew(have
, dd
->nat_tot
);
2320 for (i
= 0; i
< natoms_sys
; i
++)
2322 if (ga2la_get(dd
->ga2la
, i
, &a
, &cell
))
2324 if (a
>= dd
->nat_tot
)
2326 fprintf(stderr
, "DD rank %d: global atom %d marked as local atom %d, which is larger than nat_tot (%d)\n", dd
->rank
, i
+1, a
+1, dd
->nat_tot
);
2332 if (dd
->gatindex
[a
] != i
)
2334 fprintf(stderr
, "DD rank %d: global atom %d marked as local atom %d, which has global atom index %d\n", dd
->rank
, i
+1, a
+1, dd
->gatindex
[a
]+1);
2341 if (ngl
!= dd
->nat_tot
)
2344 "DD rank %d, %s: %d global atom indices, %d local atoms\n",
2345 dd
->rank
, where
, ngl
, dd
->nat_tot
);
2347 for (a
= 0; a
< dd
->nat_tot
; a
++)
2352 "DD rank %d, %s: local atom %d, global %d has no global index\n",
2353 dd
->rank
, where
, a
+1, dd
->gatindex
[a
]+1);
2358 nerr
+= check_bLocalCG(dd
, ncg_sys
, dd
->comm
->bLocalCG
, where
);
2362 gmx_fatal(FARGS
, "DD rank %d, %s: %d atom/cg index inconsistencies",
2363 dd
->rank
, where
, nerr
);
2367 static void clear_dd_indices(gmx_domdec_t
*dd
, int cg_start
, int a_start
)
2374 /* Clear the whole list without searching */
2375 ga2la_clear(dd
->ga2la
);
2379 for (i
= a_start
; i
< dd
->nat_tot
; i
++)
2381 ga2la_del(dd
->ga2la
, dd
->gatindex
[i
]);
2385 bLocalCG
= dd
->comm
->bLocalCG
;
2388 for (i
= cg_start
; i
< dd
->ncg_tot
; i
++)
2390 bLocalCG
[dd
->index_gl
[i
]] = FALSE
;
2394 dd_clear_local_vsite_indices(dd
);
2396 if (dd
->constraints
)
2398 dd_clear_local_constraint_indices(dd
);
2402 /* This function should be used for moving the domain boudaries during DLB,
2403 * for obtaining the minimum cell size. It checks the initially set limit
2404 * comm->cellsize_min, for bonded and initial non-bonded cut-offs,
2405 * and, possibly, a longer cut-off limit set for PME load balancing.
2407 static real
cellsize_min_dlb(gmx_domdec_comm_t
*comm
, int dim_ind
, int dim
)
2411 cellsize_min
= comm
->cellsize_min
[dim
];
2413 if (!comm
->bVacDLBNoLimit
)
2415 /* The cut-off might have changed, e.g. by PME load balacning,
2416 * from the value used to set comm->cellsize_min, so check it.
2418 cellsize_min
= std::max(cellsize_min
, comm
->cutoff
/comm
->cd
[dim_ind
].np_dlb
);
2420 if (comm
->bPMELoadBalDLBLimits
)
2422 /* Check for the cut-off limit set by the PME load balancing */
2423 cellsize_min
= std::max(cellsize_min
, comm
->PMELoadBal_max_cutoff
/comm
->cd
[dim_ind
].np_dlb
);
2427 return cellsize_min
;
2430 static real
grid_jump_limit(gmx_domdec_comm_t
*comm
, real cutoff
,
2433 real grid_jump_limit
;
2435 /* The distance between the boundaries of cells at distance
2436 * x+-1,y+-1 or y+-1,z+-1 is limited by the cut-off restrictions
2437 * and by the fact that cells should not be shifted by more than
2438 * half their size, such that cg's only shift by one cell
2439 * at redecomposition.
2441 grid_jump_limit
= comm
->cellsize_limit
;
2442 if (!comm
->bVacDLBNoLimit
)
2444 if (comm
->bPMELoadBalDLBLimits
)
2446 cutoff
= std::max(cutoff
, comm
->PMELoadBal_max_cutoff
);
2448 grid_jump_limit
= std::max(grid_jump_limit
,
2449 cutoff
/comm
->cd
[dim_ind
].np
);
2452 return grid_jump_limit
;
2455 static gmx_bool
check_grid_jump(gmx_int64_t step
,
2461 gmx_domdec_comm_t
*comm
;
2470 for (d
= 1; d
< dd
->ndim
; d
++)
2473 limit
= grid_jump_limit(comm
, cutoff
, d
);
2474 bfac
= ddbox
->box_size
[dim
];
2475 if (ddbox
->tric_dir
[dim
])
2477 bfac
*= ddbox
->skew_fac
[dim
];
2479 if ((comm
->cell_f1
[d
] - comm
->cell_f_max0
[d
])*bfac
< limit
||
2480 (comm
->cell_f0
[d
] - comm
->cell_f_min1
[d
])*bfac
> -limit
)
2488 /* This error should never be triggered under normal
2489 * circumstances, but you never know ...
2491 gmx_fatal(FARGS
, "step %s: The domain decomposition grid has shifted too much in the %c-direction around cell %d %d %d. This should not have happened. Running with fewer ranks might avoid this issue.",
2492 gmx_step_str(step
, buf
),
2493 dim2char(dim
), dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
2501 static int dd_load_count(gmx_domdec_comm_t
*comm
)
2503 return (comm
->eFlop
? comm
->flop_n
: comm
->cycl_n
[ddCyclF
]);
2506 static float dd_force_load(gmx_domdec_comm_t
*comm
)
2513 if (comm
->eFlop
> 1)
2515 load
*= 1.0 + (comm
->eFlop
- 1)*(0.1*rand()/RAND_MAX
- 0.05);
2520 load
= comm
->cycl
[ddCyclF
];
2521 if (comm
->cycl_n
[ddCyclF
] > 1)
2523 /* Subtract the maximum of the last n cycle counts
2524 * to get rid of possible high counts due to other sources,
2525 * for instance system activity, that would otherwise
2526 * affect the dynamic load balancing.
2528 load
-= comm
->cycl_max
[ddCyclF
];
2532 if (comm
->cycl_n
[ddCyclWaitGPU
] && comm
->nrank_gpu_shared
> 1)
2534 float gpu_wait
, gpu_wait_sum
;
2536 gpu_wait
= comm
->cycl
[ddCyclWaitGPU
];
2537 if (comm
->cycl_n
[ddCyclF
] > 1)
2539 /* We should remove the WaitGPU time of the same MD step
2540 * as the one with the maximum F time, since the F time
2541 * and the wait time are not independent.
2542 * Furthermore, the step for the max F time should be chosen
2543 * the same on all ranks that share the same GPU.
2544 * But to keep the code simple, we remove the average instead.
2545 * The main reason for artificially long times at some steps
2546 * is spurious CPU activity or MPI time, so we don't expect
2547 * that changes in the GPU wait time matter a lot here.
2549 gpu_wait
*= (comm
->cycl_n
[ddCyclF
] - 1)/(float)comm
->cycl_n
[ddCyclF
];
2551 /* Sum the wait times over the ranks that share the same GPU */
2552 MPI_Allreduce(&gpu_wait
, &gpu_wait_sum
, 1, MPI_FLOAT
, MPI_SUM
,
2553 comm
->mpi_comm_gpu_shared
);
2554 /* Replace the wait time by the average over the ranks */
2555 load
+= -gpu_wait
+ gpu_wait_sum
/comm
->nrank_gpu_shared
;
2563 static void set_slb_pme_dim_f(gmx_domdec_t
*dd
, int dim
, real
**dim_f
)
2565 gmx_domdec_comm_t
*comm
;
2570 snew(*dim_f
, dd
->nc
[dim
]+1);
2572 for (i
= 1; i
< dd
->nc
[dim
]; i
++)
2574 if (comm
->slb_frac
[dim
])
2576 (*dim_f
)[i
] = (*dim_f
)[i
-1] + comm
->slb_frac
[dim
][i
-1];
2580 (*dim_f
)[i
] = (real
)i
/(real
)dd
->nc
[dim
];
2583 (*dim_f
)[dd
->nc
[dim
]] = 1;
2586 static void init_ddpme(gmx_domdec_t
*dd
, gmx_ddpme_t
*ddpme
, int dimind
)
2588 int pmeindex
, slab
, nso
, i
;
2591 if (dimind
== 0 && dd
->dim
[0] == YY
&& dd
->comm
->npmenodes_x
== 1)
2597 ddpme
->dim
= dimind
;
2599 ddpme
->dim_match
= (ddpme
->dim
== dd
->dim
[dimind
]);
2601 ddpme
->nslab
= (ddpme
->dim
== 0 ?
2602 dd
->comm
->npmenodes_x
:
2603 dd
->comm
->npmenodes_y
);
2605 if (ddpme
->nslab
<= 1)
2610 nso
= dd
->comm
->npmenodes
/ddpme
->nslab
;
2611 /* Determine for each PME slab the PP location range for dimension dim */
2612 snew(ddpme
->pp_min
, ddpme
->nslab
);
2613 snew(ddpme
->pp_max
, ddpme
->nslab
);
2614 for (slab
= 0; slab
< ddpme
->nslab
; slab
++)
2616 ddpme
->pp_min
[slab
] = dd
->nc
[dd
->dim
[dimind
]] - 1;
2617 ddpme
->pp_max
[slab
] = 0;
2619 for (i
= 0; i
< dd
->nnodes
; i
++)
2621 ddindex2xyz(dd
->nc
, i
, xyz
);
2622 /* For y only use our y/z slab.
2623 * This assumes that the PME x grid size matches the DD grid size.
2625 if (dimind
== 0 || xyz
[XX
] == dd
->ci
[XX
])
2627 pmeindex
= ddindex2pmeindex(dd
, i
);
2630 slab
= pmeindex
/nso
;
2634 slab
= pmeindex
% ddpme
->nslab
;
2636 ddpme
->pp_min
[slab
] = std::min(ddpme
->pp_min
[slab
], xyz
[dimind
]);
2637 ddpme
->pp_max
[slab
] = std::max(ddpme
->pp_max
[slab
], xyz
[dimind
]);
2641 set_slb_pme_dim_f(dd
, ddpme
->dim
, &ddpme
->slb_dim_f
);
2644 int dd_pme_maxshift_x(const gmx_domdec_t
*dd
)
2646 if (dd
->comm
->ddpme
[0].dim
== XX
)
2648 return dd
->comm
->ddpme
[0].maxshift
;
2656 int dd_pme_maxshift_y(const gmx_domdec_t
*dd
)
2658 if (dd
->comm
->ddpme
[0].dim
== YY
)
2660 return dd
->comm
->ddpme
[0].maxshift
;
2662 else if (dd
->comm
->npmedecompdim
>= 2 && dd
->comm
->ddpme
[1].dim
== YY
)
2664 return dd
->comm
->ddpme
[1].maxshift
;
2672 static void set_pme_maxshift(gmx_domdec_t
*dd
, gmx_ddpme_t
*ddpme
,
2673 gmx_bool bUniform
, const gmx_ddbox_t
*ddbox
,
2676 gmx_domdec_comm_t
*comm
;
2679 real range
, pme_boundary
;
2683 nc
= dd
->nc
[ddpme
->dim
];
2686 if (!ddpme
->dim_match
)
2688 /* PP decomposition is not along dim: the worst situation */
2691 else if (ns
<= 3 || (bUniform
&& ns
== nc
))
2693 /* The optimal situation */
2698 /* We need to check for all pme nodes which nodes they
2699 * could possibly need to communicate with.
2701 xmin
= ddpme
->pp_min
;
2702 xmax
= ddpme
->pp_max
;
2703 /* Allow for atoms to be maximally 2/3 times the cut-off
2704 * out of their DD cell. This is a reasonable balance between
2705 * between performance and support for most charge-group/cut-off
2708 range
= 2.0/3.0*comm
->cutoff
/ddbox
->box_size
[ddpme
->dim
];
2709 /* Avoid extra communication when we are exactly at a boundary */
2713 for (s
= 0; s
< ns
; s
++)
2715 /* PME slab s spreads atoms between box frac. s/ns and (s+1)/ns */
2716 pme_boundary
= (real
)s
/ns
;
2719 cell_f
[xmax
[s
-(sh
+1) ]+1] + range
> pme_boundary
) ||
2721 cell_f
[xmax
[s
-(sh
+1)+ns
]+1] - 1 + range
> pme_boundary
)))
2725 pme_boundary
= (real
)(s
+1)/ns
;
2728 cell_f
[xmin
[s
+(sh
+1) ] ] - range
< pme_boundary
) ||
2730 cell_f
[xmin
[s
+(sh
+1)-ns
] ] + 1 - range
< pme_boundary
)))
2737 ddpme
->maxshift
= sh
;
2741 fprintf(debug
, "PME slab communication range for dim %d is %d\n",
2742 ddpme
->dim
, ddpme
->maxshift
);
2746 static void check_box_size(gmx_domdec_t
*dd
, gmx_ddbox_t
*ddbox
)
2750 for (d
= 0; d
< dd
->ndim
; d
++)
2753 if (dim
< ddbox
->nboundeddim
&&
2754 ddbox
->box_size
[dim
]*ddbox
->skew_fac
[dim
] <
2755 dd
->nc
[dim
]*dd
->comm
->cellsize_limit
*DD_CELL_MARGIN
)
2757 gmx_fatal(FARGS
, "The %c-size of the box (%f) times the triclinic skew factor (%f) is smaller than the number of DD cells (%d) times the smallest allowed cell size (%f)\n",
2758 dim2char(dim
), ddbox
->box_size
[dim
], ddbox
->skew_fac
[dim
],
2759 dd
->nc
[dim
], dd
->comm
->cellsize_limit
);
2765 setcellsizeslbLOCAL
, setcellsizeslbMASTER
, setcellsizeslbPULSE_ONLY
2768 /* Set the domain boundaries. Use for static (or no) load balancing,
2769 * and also for the starting state for dynamic load balancing.
2770 * setmode determine if and where the boundaries are stored, use enum above.
2771 * Returns the number communication pulses in npulse.
2773 static void set_dd_cell_sizes_slb(gmx_domdec_t
*dd
, const gmx_ddbox_t
*ddbox
,
2774 int setmode
, ivec npulse
)
2776 gmx_domdec_comm_t
*comm
;
2779 real
*cell_x
, cell_dx
, cellsize
;
2783 for (d
= 0; d
< DIM
; d
++)
2785 cellsize_min
[d
] = ddbox
->box_size
[d
]*ddbox
->skew_fac
[d
];
2787 if (dd
->nc
[d
] == 1 || comm
->slb_frac
[d
] == nullptr)
2790 cell_dx
= ddbox
->box_size
[d
]/dd
->nc
[d
];
2793 case setcellsizeslbMASTER
:
2794 for (j
= 0; j
< dd
->nc
[d
]+1; j
++)
2796 dd
->ma
->cell_x
[d
][j
] = ddbox
->box0
[d
] + j
*cell_dx
;
2799 case setcellsizeslbLOCAL
:
2800 comm
->cell_x0
[d
] = ddbox
->box0
[d
] + (dd
->ci
[d
] )*cell_dx
;
2801 comm
->cell_x1
[d
] = ddbox
->box0
[d
] + (dd
->ci
[d
]+1)*cell_dx
;
2806 cellsize
= cell_dx
*ddbox
->skew_fac
[d
];
2807 while (cellsize
*npulse
[d
] < comm
->cutoff
)
2811 cellsize_min
[d
] = cellsize
;
2815 /* Statically load balanced grid */
2816 /* Also when we are not doing a master distribution we determine
2817 * all cell borders in a loop to obtain identical values
2818 * to the master distribution case and to determine npulse.
2820 if (setmode
== setcellsizeslbMASTER
)
2822 cell_x
= dd
->ma
->cell_x
[d
];
2826 snew(cell_x
, dd
->nc
[d
]+1);
2828 cell_x
[0] = ddbox
->box0
[d
];
2829 for (j
= 0; j
< dd
->nc
[d
]; j
++)
2831 cell_dx
= ddbox
->box_size
[d
]*comm
->slb_frac
[d
][j
];
2832 cell_x
[j
+1] = cell_x
[j
] + cell_dx
;
2833 cellsize
= cell_dx
*ddbox
->skew_fac
[d
];
2834 while (cellsize
*npulse
[d
] < comm
->cutoff
&&
2835 npulse
[d
] < dd
->nc
[d
]-1)
2839 cellsize_min
[d
] = std::min(cellsize_min
[d
], cellsize
);
2841 if (setmode
== setcellsizeslbLOCAL
)
2843 comm
->cell_x0
[d
] = cell_x
[dd
->ci
[d
]];
2844 comm
->cell_x1
[d
] = cell_x
[dd
->ci
[d
]+1];
2846 if (setmode
!= setcellsizeslbMASTER
)
2851 /* The following limitation is to avoid that a cell would receive
2852 * some of its own home charge groups back over the periodic boundary.
2853 * Double charge groups cause trouble with the global indices.
2855 if (d
< ddbox
->npbcdim
&&
2856 dd
->nc
[d
] > 1 && npulse
[d
] >= dd
->nc
[d
])
2858 char error_string
[STRLEN
];
2860 sprintf(error_string
,
2861 "The box size in direction %c (%f) times the triclinic skew factor (%f) is too small for a cut-off of %f with %d domain decomposition cells, use 1 or more than %d %s or increase the box size in this direction",
2862 dim2char(d
), ddbox
->box_size
[d
], ddbox
->skew_fac
[d
],
2864 dd
->nc
[d
], dd
->nc
[d
],
2865 dd
->nnodes
> dd
->nc
[d
] ? "cells" : "ranks");
2867 if (setmode
== setcellsizeslbLOCAL
)
2869 gmx_fatal_collective(FARGS
, dd
->mpi_comm_all
, DDMASTER(dd
),
2874 gmx_fatal(FARGS
, error_string
);
2881 copy_rvec(cellsize_min
, comm
->cellsize_min
);
2884 for (d
= 0; d
< comm
->npmedecompdim
; d
++)
2886 set_pme_maxshift(dd
, &comm
->ddpme
[d
],
2887 comm
->slb_frac
[dd
->dim
[d
]] == nullptr, ddbox
,
2888 comm
->ddpme
[d
].slb_dim_f
);
2893 static void dd_cell_sizes_dlb_root_enforce_limits(gmx_domdec_t
*dd
,
2894 int d
, int dim
, domdec_root_t
*root
,
2895 const gmx_ddbox_t
*ddbox
,
2896 gmx_bool bUniform
, gmx_int64_t step
, real cellsize_limit_f
, int range
[])
2898 gmx_domdec_comm_t
*comm
;
2899 int ncd
, i
, j
, nmin
, nmin_old
;
2900 gmx_bool bLimLo
, bLimHi
;
2902 real fac
, halfway
, cellsize_limit_f_i
, region_size
;
2903 gmx_bool bPBC
, bLastHi
= FALSE
;
2904 int nrange
[] = {range
[0], range
[1]};
2906 region_size
= root
->cell_f
[range
[1]]-root
->cell_f
[range
[0]];
2912 bPBC
= (dim
< ddbox
->npbcdim
);
2914 cell_size
= root
->buf_ncd
;
2918 fprintf(debug
, "enforce_limits: %d %d\n", range
[0], range
[1]);
2921 /* First we need to check if the scaling does not make cells
2922 * smaller than the smallest allowed size.
2923 * We need to do this iteratively, since if a cell is too small,
2924 * it needs to be enlarged, which makes all the other cells smaller,
2925 * which could in turn make another cell smaller than allowed.
2927 for (i
= range
[0]; i
< range
[1]; i
++)
2929 root
->bCellMin
[i
] = FALSE
;
2935 /* We need the total for normalization */
2937 for (i
= range
[0]; i
< range
[1]; i
++)
2939 if (root
->bCellMin
[i
] == FALSE
)
2941 fac
+= cell_size
[i
];
2944 fac
= ( region_size
- nmin
*cellsize_limit_f
)/fac
; /* substracting cells already set to cellsize_limit_f */
2945 /* Determine the cell boundaries */
2946 for (i
= range
[0]; i
< range
[1]; i
++)
2948 if (root
->bCellMin
[i
] == FALSE
)
2950 cell_size
[i
] *= fac
;
2951 if (!bPBC
&& (i
== 0 || i
== dd
->nc
[dim
] -1))
2953 cellsize_limit_f_i
= 0;
2957 cellsize_limit_f_i
= cellsize_limit_f
;
2959 if (cell_size
[i
] < cellsize_limit_f_i
)
2961 root
->bCellMin
[i
] = TRUE
;
2962 cell_size
[i
] = cellsize_limit_f_i
;
2966 root
->cell_f
[i
+1] = root
->cell_f
[i
] + cell_size
[i
];
2969 while (nmin
> nmin_old
);
2972 cell_size
[i
] = root
->cell_f
[i
+1] - root
->cell_f
[i
];
2973 /* For this check we should not use DD_CELL_MARGIN,
2974 * but a slightly smaller factor,
2975 * since rounding could get use below the limit.
2977 if (bPBC
&& cell_size
[i
] < cellsize_limit_f
*DD_CELL_MARGIN2
/DD_CELL_MARGIN
)
2980 gmx_fatal(FARGS
, "step %s: the dynamic load balancing could not balance dimension %c: box size %f, triclinic skew factor %f, #cells %d, minimum cell size %f\n",
2981 gmx_step_str(step
, buf
),
2982 dim2char(dim
), ddbox
->box_size
[dim
], ddbox
->skew_fac
[dim
],
2983 ncd
, comm
->cellsize_min
[dim
]);
2986 root
->bLimited
= (nmin
> 0) || (range
[0] > 0) || (range
[1] < ncd
);
2990 /* Check if the boundary did not displace more than halfway
2991 * each of the cells it bounds, as this could cause problems,
2992 * especially when the differences between cell sizes are large.
2993 * If changes are applied, they will not make cells smaller
2994 * than the cut-off, as we check all the boundaries which
2995 * might be affected by a change and if the old state was ok,
2996 * the cells will at most be shrunk back to their old size.
2998 for (i
= range
[0]+1; i
< range
[1]; i
++)
3000 halfway
= 0.5*(root
->old_cell_f
[i
] + root
->old_cell_f
[i
-1]);
3001 if (root
->cell_f
[i
] < halfway
)
3003 root
->cell_f
[i
] = halfway
;
3004 /* Check if the change also causes shifts of the next boundaries */
3005 for (j
= i
+1; j
< range
[1]; j
++)
3007 if (root
->cell_f
[j
] < root
->cell_f
[j
-1] + cellsize_limit_f
)
3009 root
->cell_f
[j
] = root
->cell_f
[j
-1] + cellsize_limit_f
;
3013 halfway
= 0.5*(root
->old_cell_f
[i
] + root
->old_cell_f
[i
+1]);
3014 if (root
->cell_f
[i
] > halfway
)
3016 root
->cell_f
[i
] = halfway
;
3017 /* Check if the change also causes shifts of the next boundaries */
3018 for (j
= i
-1; j
>= range
[0]+1; j
--)
3020 if (root
->cell_f
[j
] > root
->cell_f
[j
+1] - cellsize_limit_f
)
3022 root
->cell_f
[j
] = root
->cell_f
[j
+1] - cellsize_limit_f
;
3029 /* nrange is defined as [lower, upper) range for new call to enforce_limits */
3030 /* find highest violation of LimLo (a) and the following violation of LimHi (thus the lowest following) (b)
3031 * then call enforce_limits for (oldb,a), (a,b). In the next step: (b,nexta). oldb and nexta can be the boundaries.
3032 * for a and b nrange is used */
3035 /* Take care of the staggering of the cell boundaries */
3038 for (i
= range
[0]; i
< range
[1]; i
++)
3040 root
->cell_f_max0
[i
] = root
->cell_f
[i
];
3041 root
->cell_f_min1
[i
] = root
->cell_f
[i
+1];
3046 for (i
= range
[0]+1; i
< range
[1]; i
++)
3048 bLimLo
= (root
->cell_f
[i
] < root
->bound_min
[i
]);
3049 bLimHi
= (root
->cell_f
[i
] > root
->bound_max
[i
]);
3050 if (bLimLo
&& bLimHi
)
3052 /* Both limits violated, try the best we can */
3053 /* For this case we split the original range (range) in two parts and care about the other limitiations in the next iteration. */
3054 root
->cell_f
[i
] = 0.5*(root
->bound_min
[i
] + root
->bound_max
[i
]);
3055 nrange
[0] = range
[0];
3057 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3060 nrange
[1] = range
[1];
3061 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3067 /* root->cell_f[i] = root->bound_min[i]; */
3068 nrange
[1] = i
; /* only store violation location. There could be a LimLo violation following with an higher index */
3071 else if (bLimHi
&& !bLastHi
)
3074 if (nrange
[1] < range
[1]) /* found a LimLo before */
3076 root
->cell_f
[nrange
[1]] = root
->bound_min
[nrange
[1]];
3077 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3078 nrange
[0] = nrange
[1];
3080 root
->cell_f
[i
] = root
->bound_max
[i
];
3082 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3084 nrange
[1] = range
[1];
3087 if (nrange
[1] < range
[1]) /* found last a LimLo */
3089 root
->cell_f
[nrange
[1]] = root
->bound_min
[nrange
[1]];
3090 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3091 nrange
[0] = nrange
[1];
3092 nrange
[1] = range
[1];
3093 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3095 else if (nrange
[0] > range
[0]) /* found at least one LimHi */
3097 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, nrange
);
3104 static void set_dd_cell_sizes_dlb_root(gmx_domdec_t
*dd
,
3105 int d
, int dim
, domdec_root_t
*root
,
3106 const gmx_ddbox_t
*ddbox
,
3107 gmx_bool bDynamicBox
,
3108 gmx_bool bUniform
, gmx_int64_t step
)
3110 gmx_domdec_comm_t
*comm
;
3111 int ncd
, d1
, i
, pos
;
3113 real load_aver
, load_i
, imbalance
, change
, change_max
, sc
;
3114 real cellsize_limit_f
, dist_min_f
, dist_min_f_hard
, space
;
3118 int range
[] = { 0, 0 };
3122 /* Convert the maximum change from the input percentage to a fraction */
3123 change_limit
= comm
->dlb_scale_lim
*0.01;
3127 bPBC
= (dim
< ddbox
->npbcdim
);
3129 cell_size
= root
->buf_ncd
;
3131 /* Store the original boundaries */
3132 for (i
= 0; i
< ncd
+1; i
++)
3134 root
->old_cell_f
[i
] = root
->cell_f
[i
];
3138 for (i
= 0; i
< ncd
; i
++)
3140 cell_size
[i
] = 1.0/ncd
;
3143 else if (dd_load_count(comm
) > 0)
3145 load_aver
= comm
->load
[d
].sum_m
/ncd
;
3147 for (i
= 0; i
< ncd
; i
++)
3149 /* Determine the relative imbalance of cell i */
3150 load_i
= comm
->load
[d
].load
[i
*comm
->load
[d
].nload
+2];
3151 imbalance
= (load_i
- load_aver
)/(load_aver
> 0 ? load_aver
: 1);
3152 /* Determine the change of the cell size using underrelaxation */
3153 change
= -relax
*imbalance
;
3154 change_max
= std::max(change_max
, std::max(change
, -change
));
3156 /* Limit the amount of scaling.
3157 * We need to use the same rescaling for all cells in one row,
3158 * otherwise the load balancing might not converge.
3161 if (change_max
> change_limit
)
3163 sc
*= change_limit
/change_max
;
3165 for (i
= 0; i
< ncd
; i
++)
3167 /* Determine the relative imbalance of cell i */
3168 load_i
= comm
->load
[d
].load
[i
*comm
->load
[d
].nload
+2];
3169 imbalance
= (load_i
- load_aver
)/(load_aver
> 0 ? load_aver
: 1);
3170 /* Determine the change of the cell size using underrelaxation */
3171 change
= -sc
*imbalance
;
3172 cell_size
[i
] = (root
->cell_f
[i
+1]-root
->cell_f
[i
])*(1 + change
);
3176 cellsize_limit_f
= cellsize_min_dlb(comm
, d
, dim
)/ddbox
->box_size
[dim
];
3177 cellsize_limit_f
*= DD_CELL_MARGIN
;
3178 dist_min_f_hard
= grid_jump_limit(comm
, comm
->cutoff
, d
)/ddbox
->box_size
[dim
];
3179 dist_min_f
= dist_min_f_hard
* DD_CELL_MARGIN
;
3180 if (ddbox
->tric_dir
[dim
])
3182 cellsize_limit_f
/= ddbox
->skew_fac
[dim
];
3183 dist_min_f
/= ddbox
->skew_fac
[dim
];
3185 if (bDynamicBox
&& d
> 0)
3187 dist_min_f
*= DD_PRES_SCALE_MARGIN
;
3189 if (d
> 0 && !bUniform
)
3191 /* Make sure that the grid is not shifted too much */
3192 for (i
= 1; i
< ncd
; i
++)
3194 if (root
->cell_f_min1
[i
] - root
->cell_f_max0
[i
-1] < 2 * dist_min_f_hard
)
3196 gmx_incons("Inconsistent DD boundary staggering limits!");
3198 root
->bound_min
[i
] = root
->cell_f_max0
[i
-1] + dist_min_f
;
3199 space
= root
->cell_f
[i
] - (root
->cell_f_max0
[i
-1] + dist_min_f
);
3202 root
->bound_min
[i
] += 0.5*space
;
3204 root
->bound_max
[i
] = root
->cell_f_min1
[i
] - dist_min_f
;
3205 space
= root
->cell_f
[i
] - (root
->cell_f_min1
[i
] - dist_min_f
);
3208 root
->bound_max
[i
] += 0.5*space
;
3213 "dim %d boundary %d %.3f < %.3f < %.3f < %.3f < %.3f\n",
3215 root
->cell_f_max0
[i
-1] + dist_min_f
,
3216 root
->bound_min
[i
], root
->cell_f
[i
], root
->bound_max
[i
],
3217 root
->cell_f_min1
[i
] - dist_min_f
);
3222 root
->cell_f
[0] = 0;
3223 root
->cell_f
[ncd
] = 1;
3224 dd_cell_sizes_dlb_root_enforce_limits(dd
, d
, dim
, root
, ddbox
, bUniform
, step
, cellsize_limit_f
, range
);
3227 /* After the checks above, the cells should obey the cut-off
3228 * restrictions, but it does not hurt to check.
3230 for (i
= 0; i
< ncd
; i
++)
3234 fprintf(debug
, "Relative bounds dim %d cell %d: %f %f\n",
3235 dim
, i
, root
->cell_f
[i
], root
->cell_f
[i
+1]);
3238 if ((bPBC
|| (i
!= 0 && i
!= dd
->nc
[dim
]-1)) &&
3239 root
->cell_f
[i
+1] - root
->cell_f
[i
] <
3240 cellsize_limit_f
/DD_CELL_MARGIN
)
3244 "\nWARNING step %s: direction %c, cell %d too small: %f\n",
3245 gmx_step_str(step
, buf
), dim2char(dim
), i
,
3246 (root
->cell_f
[i
+1] - root
->cell_f
[i
])
3247 *ddbox
->box_size
[dim
]*ddbox
->skew_fac
[dim
]);
3252 /* Store the cell boundaries of the lower dimensions at the end */
3253 for (d1
= 0; d1
< d
; d1
++)
3255 root
->cell_f
[pos
++] = comm
->cell_f0
[d1
];
3256 root
->cell_f
[pos
++] = comm
->cell_f1
[d1
];
3259 if (d
< comm
->npmedecompdim
)
3261 /* The master determines the maximum shift for
3262 * the coordinate communication between separate PME nodes.
3264 set_pme_maxshift(dd
, &comm
->ddpme
[d
], bUniform
, ddbox
, root
->cell_f
);
3266 root
->cell_f
[pos
++] = comm
->ddpme
[0].maxshift
;
3269 root
->cell_f
[pos
++] = comm
->ddpme
[1].maxshift
;
3273 static void relative_to_absolute_cell_bounds(gmx_domdec_t
*dd
,
3274 const gmx_ddbox_t
*ddbox
,
3277 gmx_domdec_comm_t
*comm
;
3282 /* Set the cell dimensions */
3283 dim
= dd
->dim
[dimind
];
3284 comm
->cell_x0
[dim
] = comm
->cell_f0
[dimind
]*ddbox
->box_size
[dim
];
3285 comm
->cell_x1
[dim
] = comm
->cell_f1
[dimind
]*ddbox
->box_size
[dim
];
3286 if (dim
>= ddbox
->nboundeddim
)
3288 comm
->cell_x0
[dim
] += ddbox
->box0
[dim
];
3289 comm
->cell_x1
[dim
] += ddbox
->box0
[dim
];
3293 static void distribute_dd_cell_sizes_dlb(gmx_domdec_t
*dd
,
3294 int d
, int dim
, real
*cell_f_row
,
3295 const gmx_ddbox_t
*ddbox
)
3297 gmx_domdec_comm_t
*comm
;
3303 /* Each node would only need to know two fractions,
3304 * but it is probably cheaper to broadcast the whole array.
3306 MPI_Bcast(cell_f_row
, DD_CELL_F_SIZE(dd
, d
)*sizeof(real
), MPI_BYTE
,
3307 0, comm
->mpi_comm_load
[d
]);
3309 /* Copy the fractions for this dimension from the buffer */
3310 comm
->cell_f0
[d
] = cell_f_row
[dd
->ci
[dim
] ];
3311 comm
->cell_f1
[d
] = cell_f_row
[dd
->ci
[dim
]+1];
3312 /* The whole array was communicated, so set the buffer position */
3313 pos
= dd
->nc
[dim
] + 1;
3314 for (d1
= 0; d1
<= d
; d1
++)
3318 /* Copy the cell fractions of the lower dimensions */
3319 comm
->cell_f0
[d1
] = cell_f_row
[pos
++];
3320 comm
->cell_f1
[d1
] = cell_f_row
[pos
++];
3322 relative_to_absolute_cell_bounds(dd
, ddbox
, d1
);
3324 /* Convert the communicated shift from float to int */
3325 comm
->ddpme
[0].maxshift
= (int)(cell_f_row
[pos
++] + 0.5);
3328 comm
->ddpme
[1].maxshift
= (int)(cell_f_row
[pos
++] + 0.5);
3332 static void set_dd_cell_sizes_dlb_change(gmx_domdec_t
*dd
,
3333 const gmx_ddbox_t
*ddbox
,
3334 gmx_bool bDynamicBox
,
3335 gmx_bool bUniform
, gmx_int64_t step
)
3337 gmx_domdec_comm_t
*comm
;
3339 gmx_bool bRowMember
, bRowRoot
;
3344 for (d
= 0; d
< dd
->ndim
; d
++)
3349 for (d1
= d
; d1
< dd
->ndim
; d1
++)
3351 if (dd
->ci
[dd
->dim
[d1
]] > 0)
3364 set_dd_cell_sizes_dlb_root(dd
, d
, dim
, comm
->root
[d
],
3365 ddbox
, bDynamicBox
, bUniform
, step
);
3366 cell_f_row
= comm
->root
[d
]->cell_f
;
3370 cell_f_row
= comm
->cell_f_row
;
3372 distribute_dd_cell_sizes_dlb(dd
, d
, dim
, cell_f_row
, ddbox
);
3377 static void set_dd_cell_sizes_dlb_nochange(gmx_domdec_t
*dd
,
3378 const gmx_ddbox_t
*ddbox
)
3382 /* This function assumes the box is static and should therefore
3383 * not be called when the box has changed since the last
3384 * call to dd_partition_system.
3386 for (d
= 0; d
< dd
->ndim
; d
++)
3388 relative_to_absolute_cell_bounds(dd
, ddbox
, d
);
3394 static void set_dd_cell_sizes_dlb(gmx_domdec_t
*dd
,
3395 const gmx_ddbox_t
*ddbox
, gmx_bool bDynamicBox
,
3396 gmx_bool bUniform
, gmx_bool bDoDLB
, gmx_int64_t step
,
3397 gmx_wallcycle_t wcycle
)
3399 gmx_domdec_comm_t
*comm
;
3406 wallcycle_start(wcycle
, ewcDDCOMMBOUND
);
3407 set_dd_cell_sizes_dlb_change(dd
, ddbox
, bDynamicBox
, bUniform
, step
);
3408 wallcycle_stop(wcycle
, ewcDDCOMMBOUND
);
3410 else if (bDynamicBox
)
3412 set_dd_cell_sizes_dlb_nochange(dd
, ddbox
);
3415 /* Set the dimensions for which no DD is used */
3416 for (dim
= 0; dim
< DIM
; dim
++)
3418 if (dd
->nc
[dim
] == 1)
3420 comm
->cell_x0
[dim
] = 0;
3421 comm
->cell_x1
[dim
] = ddbox
->box_size
[dim
];
3422 if (dim
>= ddbox
->nboundeddim
)
3424 comm
->cell_x0
[dim
] += ddbox
->box0
[dim
];
3425 comm
->cell_x1
[dim
] += ddbox
->box0
[dim
];
3431 static void realloc_comm_ind(gmx_domdec_t
*dd
, ivec npulse
)
3434 gmx_domdec_comm_dim_t
*cd
;
3436 for (d
= 0; d
< dd
->ndim
; d
++)
3438 cd
= &dd
->comm
->cd
[d
];
3439 np
= npulse
[dd
->dim
[d
]];
3440 if (np
> cd
->np_nalloc
)
3444 fprintf(debug
, "(Re)allocing cd for %c to %d pulses\n",
3445 dim2char(dd
->dim
[d
]), np
);
3447 if (DDMASTER(dd
) && cd
->np_nalloc
> 0)
3449 fprintf(stderr
, "\nIncreasing the number of cell to communicate in dimension %c to %d for the first time\n", dim2char(dd
->dim
[d
]), np
);
3451 srenew(cd
->ind
, np
);
3452 for (i
= cd
->np_nalloc
; i
< np
; i
++)
3454 cd
->ind
[i
].index
= nullptr;
3455 cd
->ind
[i
].nalloc
= 0;
3464 static void set_dd_cell_sizes(gmx_domdec_t
*dd
,
3465 gmx_ddbox_t
*ddbox
, gmx_bool bDynamicBox
,
3466 gmx_bool bUniform
, gmx_bool bDoDLB
, gmx_int64_t step
,
3467 gmx_wallcycle_t wcycle
)
3469 gmx_domdec_comm_t
*comm
;
3475 /* Copy the old cell boundaries for the cg displacement check */
3476 copy_rvec(comm
->cell_x0
, comm
->old_cell_x0
);
3477 copy_rvec(comm
->cell_x1
, comm
->old_cell_x1
);
3483 check_box_size(dd
, ddbox
);
3485 set_dd_cell_sizes_dlb(dd
, ddbox
, bDynamicBox
, bUniform
, bDoDLB
, step
, wcycle
);
3489 set_dd_cell_sizes_slb(dd
, ddbox
, setcellsizeslbLOCAL
, npulse
);
3490 realloc_comm_ind(dd
, npulse
);
3495 for (d
= 0; d
< DIM
; d
++)
3497 fprintf(debug
, "cell_x[%d] %f - %f skew_fac %f\n",
3498 d
, comm
->cell_x0
[d
], comm
->cell_x1
[d
], ddbox
->skew_fac
[d
]);
3503 static void comm_dd_ns_cell_sizes(gmx_domdec_t
*dd
,
3505 rvec cell_ns_x0
, rvec cell_ns_x1
,
3508 gmx_domdec_comm_t
*comm
;
3513 for (dim_ind
= 0; dim_ind
< dd
->ndim
; dim_ind
++)
3515 dim
= dd
->dim
[dim_ind
];
3517 /* Without PBC we don't have restrictions on the outer cells */
3518 if (!(dim
>= ddbox
->npbcdim
&&
3519 (dd
->ci
[dim
] == 0 || dd
->ci
[dim
] == dd
->nc
[dim
] - 1)) &&
3521 (comm
->cell_x1
[dim
] - comm
->cell_x0
[dim
])*ddbox
->skew_fac
[dim
] <
3522 comm
->cellsize_min
[dim
])
3525 gmx_fatal(FARGS
, "step %s: The %c-size (%f) times the triclinic skew factor (%f) is smaller than the smallest allowed cell size (%f) for domain decomposition grid cell %d %d %d",
3526 gmx_step_str(step
, buf
), dim2char(dim
),
3527 comm
->cell_x1
[dim
] - comm
->cell_x0
[dim
],
3528 ddbox
->skew_fac
[dim
],
3529 dd
->comm
->cellsize_min
[dim
],
3530 dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
3534 if ((isDlbOn(dd
->comm
) && dd
->ndim
> 1) || ddbox
->nboundeddim
< DIM
)
3536 /* Communicate the boundaries and update cell_ns_x0/1 */
3537 dd_move_cellx(dd
, ddbox
, cell_ns_x0
, cell_ns_x1
);
3538 if (isDlbOn(dd
->comm
) && dd
->ndim
> 1)
3540 check_grid_jump(step
, dd
, dd
->comm
->cutoff
, ddbox
, TRUE
);
3545 static void make_tric_corr_matrix(int npbcdim
, matrix box
, matrix tcm
)
3549 tcm
[YY
][XX
] = -box
[YY
][XX
]/box
[YY
][YY
];
3557 tcm
[ZZ
][XX
] = -(box
[ZZ
][YY
]*tcm
[YY
][XX
] + box
[ZZ
][XX
])/box
[ZZ
][ZZ
];
3558 tcm
[ZZ
][YY
] = -box
[ZZ
][YY
]/box
[ZZ
][ZZ
];
3567 static void check_screw_box(matrix box
)
3569 /* Mathematical limitation */
3570 if (box
[YY
][XX
] != 0 || box
[ZZ
][XX
] != 0)
3572 gmx_fatal(FARGS
, "With screw pbc the unit cell can not have non-zero off-diagonal x-components");
3575 /* Limitation due to the asymmetry of the eighth shell method */
3576 if (box
[ZZ
][YY
] != 0)
3578 gmx_fatal(FARGS
, "pbc=screw with non-zero box_zy is not supported");
3582 static void distribute_cg(FILE *fplog
,
3583 matrix box
, ivec tric_dir
, t_block
*cgs
, rvec pos
[],
3586 gmx_domdec_master_t
*ma
;
3587 int **tmp_ind
= nullptr, *tmp_nalloc
= nullptr;
3588 int i
, icg
, j
, k
, k0
, k1
, d
;
3592 real nrcg
, inv_ncg
, pos_d
;
3598 snew(tmp_nalloc
, dd
->nnodes
);
3599 snew(tmp_ind
, dd
->nnodes
);
3600 for (i
= 0; i
< dd
->nnodes
; i
++)
3602 tmp_nalloc
[i
] = over_alloc_large(cgs
->nr
/dd
->nnodes
+1);
3603 snew(tmp_ind
[i
], tmp_nalloc
[i
]);
3606 /* Clear the count */
3607 for (i
= 0; i
< dd
->nnodes
; i
++)
3613 make_tric_corr_matrix(dd
->npbcdim
, box
, tcm
);
3615 cgindex
= cgs
->index
;
3617 /* Compute the center of geometry for all charge groups */
3618 for (icg
= 0; icg
< cgs
->nr
; icg
++)
3621 k1
= cgindex
[icg
+1];
3625 copy_rvec(pos
[k0
], cg_cm
);
3632 for (k
= k0
; (k
< k1
); k
++)
3634 rvec_inc(cg_cm
, pos
[k
]);
3636 for (d
= 0; (d
< DIM
); d
++)
3638 cg_cm
[d
] *= inv_ncg
;
3641 /* Put the charge group in the box and determine the cell index */
3642 for (d
= DIM
-1; d
>= 0; d
--)
3645 if (d
< dd
->npbcdim
)
3647 bScrew
= (dd
->bScrewPBC
&& d
== XX
);
3648 if (tric_dir
[d
] && dd
->nc
[d
] > 1)
3650 /* Use triclinic coordintates for this dimension */
3651 for (j
= d
+1; j
< DIM
; j
++)
3653 pos_d
+= cg_cm
[j
]*tcm
[j
][d
];
3656 while (pos_d
>= box
[d
][d
])
3659 rvec_dec(cg_cm
, box
[d
]);
3662 cg_cm
[YY
] = box
[YY
][YY
] - cg_cm
[YY
];
3663 cg_cm
[ZZ
] = box
[ZZ
][ZZ
] - cg_cm
[ZZ
];
3665 for (k
= k0
; (k
< k1
); k
++)
3667 rvec_dec(pos
[k
], box
[d
]);
3670 pos
[k
][YY
] = box
[YY
][YY
] - pos
[k
][YY
];
3671 pos
[k
][ZZ
] = box
[ZZ
][ZZ
] - pos
[k
][ZZ
];
3678 rvec_inc(cg_cm
, box
[d
]);
3681 cg_cm
[YY
] = box
[YY
][YY
] - cg_cm
[YY
];
3682 cg_cm
[ZZ
] = box
[ZZ
][ZZ
] - cg_cm
[ZZ
];
3684 for (k
= k0
; (k
< k1
); k
++)
3686 rvec_inc(pos
[k
], box
[d
]);
3689 pos
[k
][YY
] = box
[YY
][YY
] - pos
[k
][YY
];
3690 pos
[k
][ZZ
] = box
[ZZ
][ZZ
] - pos
[k
][ZZ
];
3695 /* This could be done more efficiently */
3697 while (ind
[d
]+1 < dd
->nc
[d
] && pos_d
>= ma
->cell_x
[d
][ind
[d
]+1])
3702 i
= dd_index(dd
->nc
, ind
);
3703 if (ma
->ncg
[i
] == tmp_nalloc
[i
])
3705 tmp_nalloc
[i
] = over_alloc_large(ma
->ncg
[i
]+1);
3706 srenew(tmp_ind
[i
], tmp_nalloc
[i
]);
3708 tmp_ind
[i
][ma
->ncg
[i
]] = icg
;
3710 ma
->nat
[i
] += cgindex
[icg
+1] - cgindex
[icg
];
3714 for (i
= 0; i
< dd
->nnodes
; i
++)
3717 for (k
= 0; k
< ma
->ncg
[i
]; k
++)
3719 ma
->cg
[k1
++] = tmp_ind
[i
][k
];
3722 ma
->index
[dd
->nnodes
] = k1
;
3724 for (i
= 0; i
< dd
->nnodes
; i
++)
3733 // Use double for the sums to avoid natoms^2 overflowing
3735 int nat_sum
, nat_min
, nat_max
;
3740 nat_min
= ma
->nat
[0];
3741 nat_max
= ma
->nat
[0];
3742 for (i
= 0; i
< dd
->nnodes
; i
++)
3744 nat_sum
+= ma
->nat
[i
];
3745 // cast to double to avoid integer overflows when squaring
3746 nat2_sum
+= gmx::square(static_cast<double>(ma
->nat
[i
]));
3747 nat_min
= std::min(nat_min
, ma
->nat
[i
]);
3748 nat_max
= std::max(nat_max
, ma
->nat
[i
]);
3750 nat_sum
/= dd
->nnodes
;
3751 nat2_sum
/= dd
->nnodes
;
3753 fprintf(fplog
, "Atom distribution over %d domains: av %d stddev %d min %d max %d\n",
3756 static_cast<int>(std::sqrt(nat2_sum
- gmx::square(static_cast<double>(nat_sum
)) + 0.5)),
3761 static void get_cg_distribution(FILE *fplog
, gmx_domdec_t
*dd
,
3762 t_block
*cgs
, matrix box
, gmx_ddbox_t
*ddbox
,
3765 gmx_domdec_master_t
*ma
= nullptr;
3768 int *ibuf
, buf2
[2] = { 0, 0 };
3769 gmx_bool bMaster
= DDMASTER(dd
);
3777 check_screw_box(box
);
3780 set_dd_cell_sizes_slb(dd
, ddbox
, setcellsizeslbMASTER
, npulse
);
3782 distribute_cg(fplog
, box
, ddbox
->tric_dir
, cgs
, pos
, dd
);
3783 for (i
= 0; i
< dd
->nnodes
; i
++)
3785 ma
->ibuf
[2*i
] = ma
->ncg
[i
];
3786 ma
->ibuf
[2*i
+1] = ma
->nat
[i
];
3794 dd_scatter(dd
, 2*sizeof(int), ibuf
, buf2
);
3796 dd
->ncg_home
= buf2
[0];
3797 dd
->nat_home
= buf2
[1];
3798 dd
->ncg_tot
= dd
->ncg_home
;
3799 dd
->nat_tot
= dd
->nat_home
;
3800 if (dd
->ncg_home
> dd
->cg_nalloc
|| dd
->cg_nalloc
== 0)
3802 dd
->cg_nalloc
= over_alloc_dd(dd
->ncg_home
);
3803 srenew(dd
->index_gl
, dd
->cg_nalloc
);
3804 srenew(dd
->cgindex
, dd
->cg_nalloc
+1);
3808 for (i
= 0; i
< dd
->nnodes
; i
++)
3810 ma
->ibuf
[i
] = ma
->ncg
[i
]*sizeof(int);
3811 ma
->ibuf
[dd
->nnodes
+i
] = ma
->index
[i
]*sizeof(int);
3816 bMaster
? ma
->ibuf
: nullptr,
3817 bMaster
? ma
->ibuf
+dd
->nnodes
: nullptr,
3818 bMaster
? ma
->cg
: nullptr,
3819 dd
->ncg_home
*sizeof(int), dd
->index_gl
);
3821 /* Determine the home charge group sizes */
3823 for (i
= 0; i
< dd
->ncg_home
; i
++)
3825 cg_gl
= dd
->index_gl
[i
];
3827 dd
->cgindex
[i
] + cgs
->index
[cg_gl
+1] - cgs
->index
[cg_gl
];
3832 fprintf(debug
, "Home charge groups:\n");
3833 for (i
= 0; i
< dd
->ncg_home
; i
++)
3835 fprintf(debug
, " %d", dd
->index_gl
[i
]);
3838 fprintf(debug
, "\n");
3841 fprintf(debug
, "\n");
3845 static int compact_and_copy_vec_at(int ncg
, int *move
,
3848 rvec
*src
, gmx_domdec_comm_t
*comm
,
3851 int m
, icg
, i
, i0
, i1
, nrcg
;
3857 for (m
= 0; m
< DIM
*2; m
++)
3863 for (icg
= 0; icg
< ncg
; icg
++)
3865 i1
= cgindex
[icg
+1];
3871 /* Compact the home array in place */
3872 for (i
= i0
; i
< i1
; i
++)
3874 copy_rvec(src
[i
], src
[home_pos
++]);
3880 /* Copy to the communication buffer */
3882 pos_vec
[m
] += 1 + vec
*nrcg
;
3883 for (i
= i0
; i
< i1
; i
++)
3885 copy_rvec(src
[i
], comm
->cgcm_state
[m
][pos_vec
[m
]++]);
3887 pos_vec
[m
] += (nvec
- vec
- 1)*nrcg
;
3891 home_pos
+= i1
- i0
;
3899 static int compact_and_copy_vec_cg(int ncg
, int *move
,
3901 int nvec
, rvec
*src
, gmx_domdec_comm_t
*comm
,
3904 int m
, icg
, i0
, i1
, nrcg
;
3910 for (m
= 0; m
< DIM
*2; m
++)
3916 for (icg
= 0; icg
< ncg
; icg
++)
3918 i1
= cgindex
[icg
+1];
3924 /* Compact the home array in place */
3925 copy_rvec(src
[icg
], src
[home_pos
++]);
3931 /* Copy to the communication buffer */
3932 copy_rvec(src
[icg
], comm
->cgcm_state
[m
][pos_vec
[m
]]);
3933 pos_vec
[m
] += 1 + nrcg
*nvec
;
3945 static int compact_ind(int ncg
, int *move
,
3946 int *index_gl
, int *cgindex
,
3948 gmx_ga2la_t
*ga2la
, char *bLocalCG
,
3951 int cg
, nat
, a0
, a1
, a
, a_gl
;
3956 for (cg
= 0; cg
< ncg
; cg
++)
3962 /* Compact the home arrays in place.
3963 * Anything that can be done here avoids access to global arrays.
3965 cgindex
[home_pos
] = nat
;
3966 for (a
= a0
; a
< a1
; a
++)
3969 gatindex
[nat
] = a_gl
;
3970 /* The cell number stays 0, so we don't need to set it */
3971 ga2la_change_la(ga2la
, a_gl
, nat
);
3974 index_gl
[home_pos
] = index_gl
[cg
];
3975 cginfo
[home_pos
] = cginfo
[cg
];
3976 /* The charge group remains local, so bLocalCG does not change */
3981 /* Clear the global indices */
3982 for (a
= a0
; a
< a1
; a
++)
3984 ga2la_del(ga2la
, gatindex
[a
]);
3988 bLocalCG
[index_gl
[cg
]] = FALSE
;
3992 cgindex
[home_pos
] = nat
;
3997 static void clear_and_mark_ind(int ncg
, int *move
,
3998 int *index_gl
, int *cgindex
, int *gatindex
,
3999 gmx_ga2la_t
*ga2la
, char *bLocalCG
,
4004 for (cg
= 0; cg
< ncg
; cg
++)
4010 /* Clear the global indices */
4011 for (a
= a0
; a
< a1
; a
++)
4013 ga2la_del(ga2la
, gatindex
[a
]);
4017 bLocalCG
[index_gl
[cg
]] = FALSE
;
4019 /* Signal that this cg has moved using the ns cell index.
4020 * Here we set it to -1. fill_grid will change it
4021 * from -1 to NSGRID_SIGNAL_MOVED_FAC*grid->ncells.
4023 cell_index
[cg
] = -1;
4028 static void print_cg_move(FILE *fplog
,
4030 gmx_int64_t step
, int cg
, int dim
, int dir
,
4031 gmx_bool bHaveCgcmOld
, real limitd
,
4032 rvec cm_old
, rvec cm_new
, real pos_d
)
4034 gmx_domdec_comm_t
*comm
;
4039 fprintf(fplog
, "\nStep %s:\n", gmx_step_str(step
, buf
));
4042 fprintf(fplog
, "%s %d moved more than the distance allowed by the domain decomposition (%f) in direction %c\n",
4043 dd
->comm
->bCGs
? "The charge group starting at atom" : "Atom",
4044 ddglatnr(dd
, dd
->cgindex
[cg
]), limitd
, dim2char(dim
));
4048 /* We don't have a limiting distance available: don't print it */
4049 fprintf(fplog
, "%s %d moved more than the distance allowed by the domain decomposition in direction %c\n",
4050 dd
->comm
->bCGs
? "The charge group starting at atom" : "Atom",
4051 ddglatnr(dd
, dd
->cgindex
[cg
]), dim2char(dim
));
4053 fprintf(fplog
, "distance out of cell %f\n",
4054 dir
== 1 ? pos_d
- comm
->cell_x1
[dim
] : pos_d
- comm
->cell_x0
[dim
]);
4057 fprintf(fplog
, "Old coordinates: %8.3f %8.3f %8.3f\n",
4058 cm_old
[XX
], cm_old
[YY
], cm_old
[ZZ
]);
4060 fprintf(fplog
, "New coordinates: %8.3f %8.3f %8.3f\n",
4061 cm_new
[XX
], cm_new
[YY
], cm_new
[ZZ
]);
4062 fprintf(fplog
, "Old cell boundaries in direction %c: %8.3f %8.3f\n",
4064 comm
->old_cell_x0
[dim
], comm
->old_cell_x1
[dim
]);
4065 fprintf(fplog
, "New cell boundaries in direction %c: %8.3f %8.3f\n",
4067 comm
->cell_x0
[dim
], comm
->cell_x1
[dim
]);
4070 static void cg_move_error(FILE *fplog
,
4072 gmx_int64_t step
, int cg
, int dim
, int dir
,
4073 gmx_bool bHaveCgcmOld
, real limitd
,
4074 rvec cm_old
, rvec cm_new
, real pos_d
)
4078 print_cg_move(fplog
, dd
, step
, cg
, dim
, dir
,
4079 bHaveCgcmOld
, limitd
, cm_old
, cm_new
, pos_d
);
4081 print_cg_move(stderr
, dd
, step
, cg
, dim
, dir
,
4082 bHaveCgcmOld
, limitd
, cm_old
, cm_new
, pos_d
);
4084 "%s moved too far between two domain decomposition steps\n"
4085 "This usually means that your system is not well equilibrated",
4086 dd
->comm
->bCGs
? "A charge group" : "An atom");
4089 static void rotate_state_atom(t_state
*state
, int a
)
4091 if (state
->flags
& (1 << estX
))
4093 /* Rotate the complete state; for a rectangular box only */
4094 state
->x
[a
][YY
] = state
->box
[YY
][YY
] - state
->x
[a
][YY
];
4095 state
->x
[a
][ZZ
] = state
->box
[ZZ
][ZZ
] - state
->x
[a
][ZZ
];
4097 if (state
->flags
& (1 << estV
))
4099 state
->v
[a
][YY
] = -state
->v
[a
][YY
];
4100 state
->v
[a
][ZZ
] = -state
->v
[a
][ZZ
];
4102 if (state
->flags
& (1 << estCGP
))
4104 state
->cg_p
[a
][YY
] = -state
->cg_p
[a
][YY
];
4105 state
->cg_p
[a
][ZZ
] = -state
->cg_p
[a
][ZZ
];
4109 static int *get_moved(gmx_domdec_comm_t
*comm
, int natoms
)
4111 if (natoms
> comm
->moved_nalloc
)
4113 /* Contents should be preserved here */
4114 comm
->moved_nalloc
= over_alloc_dd(natoms
);
4115 srenew(comm
->moved
, comm
->moved_nalloc
);
4121 static void calc_cg_move(FILE *fplog
, gmx_int64_t step
,
4124 ivec tric_dir
, matrix tcm
,
4125 rvec cell_x0
, rvec cell_x1
,
4126 rvec limitd
, rvec limit0
, rvec limit1
,
4128 int cg_start
, int cg_end
,
4133 int cg
, k
, k0
, k1
, d
, dim
, d2
;
4138 real inv_ncg
, pos_d
;
4141 npbcdim
= dd
->npbcdim
;
4143 for (cg
= cg_start
; cg
< cg_end
; cg
++)
4150 copy_rvec(state
->x
[k0
], cm_new
);
4157 for (k
= k0
; (k
< k1
); k
++)
4159 rvec_inc(cm_new
, state
->x
[k
]);
4161 for (d
= 0; (d
< DIM
); d
++)
4163 cm_new
[d
] = inv_ncg
*cm_new
[d
];
4168 /* Do pbc and check DD cell boundary crossings */
4169 for (d
= DIM
-1; d
>= 0; d
--)
4173 bScrew
= (dd
->bScrewPBC
&& d
== XX
);
4174 /* Determine the location of this cg in lattice coordinates */
4178 for (d2
= d
+1; d2
< DIM
; d2
++)
4180 pos_d
+= cm_new
[d2
]*tcm
[d2
][d
];
4183 /* Put the charge group in the triclinic unit-cell */
4184 if (pos_d
>= cell_x1
[d
])
4186 if (pos_d
>= limit1
[d
])
4188 cg_move_error(fplog
, dd
, step
, cg
, d
, 1,
4189 cg_cm
!= as_rvec_array(state
->x
.data()), limitd
[d
],
4190 cg_cm
[cg
], cm_new
, pos_d
);
4193 if (dd
->ci
[d
] == dd
->nc
[d
] - 1)
4195 rvec_dec(cm_new
, state
->box
[d
]);
4198 cm_new
[YY
] = state
->box
[YY
][YY
] - cm_new
[YY
];
4199 cm_new
[ZZ
] = state
->box
[ZZ
][ZZ
] - cm_new
[ZZ
];
4201 for (k
= k0
; (k
< k1
); k
++)
4203 rvec_dec(state
->x
[k
], state
->box
[d
]);
4206 rotate_state_atom(state
, k
);
4211 else if (pos_d
< cell_x0
[d
])
4213 if (pos_d
< limit0
[d
])
4215 cg_move_error(fplog
, dd
, step
, cg
, d
, -1,
4216 cg_cm
!= as_rvec_array(state
->x
.data()), limitd
[d
],
4217 cg_cm
[cg
], cm_new
, pos_d
);
4222 rvec_inc(cm_new
, state
->box
[d
]);
4225 cm_new
[YY
] = state
->box
[YY
][YY
] - cm_new
[YY
];
4226 cm_new
[ZZ
] = state
->box
[ZZ
][ZZ
] - cm_new
[ZZ
];
4228 for (k
= k0
; (k
< k1
); k
++)
4230 rvec_inc(state
->x
[k
], state
->box
[d
]);
4233 rotate_state_atom(state
, k
);
4239 else if (d
< npbcdim
)
4241 /* Put the charge group in the rectangular unit-cell */
4242 while (cm_new
[d
] >= state
->box
[d
][d
])
4244 rvec_dec(cm_new
, state
->box
[d
]);
4245 for (k
= k0
; (k
< k1
); k
++)
4247 rvec_dec(state
->x
[k
], state
->box
[d
]);
4250 while (cm_new
[d
] < 0)
4252 rvec_inc(cm_new
, state
->box
[d
]);
4253 for (k
= k0
; (k
< k1
); k
++)
4255 rvec_inc(state
->x
[k
], state
->box
[d
]);
4261 copy_rvec(cm_new
, cg_cm
[cg
]);
4263 /* Determine where this cg should go */
4266 for (d
= 0; d
< dd
->ndim
; d
++)
4271 flag
|= DD_FLAG_FW(d
);
4277 else if (dev
[dim
] == -1)
4279 flag
|= DD_FLAG_BW(d
);
4282 if (dd
->nc
[dim
] > 2)
4293 /* Temporarily store the flag in move */
4294 move
[cg
] = mc
+ flag
;
4298 static void dd_redistribute_cg(FILE *fplog
, gmx_int64_t step
,
4299 gmx_domdec_t
*dd
, ivec tric_dir
,
4300 t_state
*state
, PaddedRVecVector
*f
,
4309 int ncg
[DIM
*2] = { 0 }, nat
[DIM
*2] = { 0 };
4310 int i
, cg
, k
, d
, dim
, dim2
, dir
, d2
, d3
;
4311 int mc
, cdd
, nrcg
, ncg_recv
, nvs
, nvr
, nvec
, vec
;
4312 int sbuf
[2], rbuf
[2];
4313 int home_pos_cg
, home_pos_at
, buf_pos
;
4317 rvec
*cg_cm
= nullptr, cell_x0
, cell_x1
, limitd
, limit0
, limit1
;
4319 cginfo_mb_t
*cginfo_mb
;
4320 gmx_domdec_comm_t
*comm
;
4322 int nthread
, thread
;
4326 check_screw_box(state
->box
);
4330 if (fr
->cutoff_scheme
== ecutsGROUP
)
4335 // Positions are always present, so there's nothing to flag
4336 bool bV
= state
->flags
& (1<<estV
);
4337 bool bCGP
= state
->flags
& (1<<estCGP
);
4339 if (dd
->ncg_tot
> comm
->nalloc_int
)
4341 comm
->nalloc_int
= over_alloc_dd(dd
->ncg_tot
);
4342 srenew(comm
->buf_int
, comm
->nalloc_int
);
4344 move
= comm
->buf_int
;
4346 npbcdim
= dd
->npbcdim
;
4348 for (d
= 0; (d
< DIM
); d
++)
4350 limitd
[d
] = dd
->comm
->cellsize_min
[d
];
4351 if (d
>= npbcdim
&& dd
->ci
[d
] == 0)
4353 cell_x0
[d
] = -GMX_FLOAT_MAX
;
4357 cell_x0
[d
] = comm
->cell_x0
[d
];
4359 if (d
>= npbcdim
&& dd
->ci
[d
] == dd
->nc
[d
] - 1)
4361 cell_x1
[d
] = GMX_FLOAT_MAX
;
4365 cell_x1
[d
] = comm
->cell_x1
[d
];
4369 limit0
[d
] = comm
->old_cell_x0
[d
] - limitd
[d
];
4370 limit1
[d
] = comm
->old_cell_x1
[d
] + limitd
[d
];
4374 /* We check after communication if a charge group moved
4375 * more than one cell. Set the pre-comm check limit to float_max.
4377 limit0
[d
] = -GMX_FLOAT_MAX
;
4378 limit1
[d
] = GMX_FLOAT_MAX
;
4382 make_tric_corr_matrix(npbcdim
, state
->box
, tcm
);
4384 cgindex
= dd
->cgindex
;
4386 nthread
= gmx_omp_nthreads_get(emntDomdec
);
4388 /* Compute the center of geometry for all home charge groups
4389 * and put them in the box and determine where they should go.
4391 #pragma omp parallel for num_threads(nthread) schedule(static)
4392 for (thread
= 0; thread
< nthread
; thread
++)
4396 calc_cg_move(fplog
, step
, dd
, state
, tric_dir
, tcm
,
4397 cell_x0
, cell_x1
, limitd
, limit0
, limit1
,
4399 ( thread
*dd
->ncg_home
)/nthread
,
4400 ((thread
+1)*dd
->ncg_home
)/nthread
,
4401 fr
->cutoff_scheme
== ecutsGROUP
? cg_cm
: as_rvec_array(state
->x
.data()),
4404 GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR
;
4407 for (cg
= 0; cg
< dd
->ncg_home
; cg
++)
4412 flag
= mc
& ~DD_FLAG_NRCG
;
4413 mc
= mc
& DD_FLAG_NRCG
;
4416 if (ncg
[mc
]+1 > comm
->cggl_flag_nalloc
[mc
])
4418 comm
->cggl_flag_nalloc
[mc
] = over_alloc_dd(ncg
[mc
]+1);
4419 srenew(comm
->cggl_flag
[mc
], comm
->cggl_flag_nalloc
[mc
]*DD_CGIBS
);
4421 comm
->cggl_flag
[mc
][ncg
[mc
]*DD_CGIBS
] = dd
->index_gl
[cg
];
4422 /* We store the cg size in the lower 16 bits
4423 * and the place where the charge group should go
4424 * in the next 6 bits. This saves some communication volume.
4426 nrcg
= cgindex
[cg
+1] - cgindex
[cg
];
4427 comm
->cggl_flag
[mc
][ncg
[mc
]*DD_CGIBS
+1] = nrcg
| flag
;
4433 inc_nrnb(nrnb
, eNR_CGCM
, dd
->nat_home
);
4434 inc_nrnb(nrnb
, eNR_RESETX
, dd
->ncg_home
);
4437 for (i
= 0; i
< dd
->ndim
*2; i
++)
4439 *ncg_moved
+= ncg
[i
];
4452 /* Make sure the communication buffers are large enough */
4453 for (mc
= 0; mc
< dd
->ndim
*2; mc
++)
4455 nvr
= ncg
[mc
] + nat
[mc
]*nvec
;
4456 if (nvr
> comm
->cgcm_state_nalloc
[mc
])
4458 comm
->cgcm_state_nalloc
[mc
] = over_alloc_dd(nvr
);
4459 srenew(comm
->cgcm_state
[mc
], comm
->cgcm_state_nalloc
[mc
]);
4463 switch (fr
->cutoff_scheme
)
4466 /* Recalculating cg_cm might be cheaper than communicating,
4467 * but that could give rise to rounding issues.
4470 compact_and_copy_vec_cg(dd
->ncg_home
, move
, cgindex
,
4471 nvec
, cg_cm
, comm
, bCompact
);
4474 /* Without charge groups we send the moved atom coordinates
4475 * over twice. This is so the code below can be used without
4476 * many conditionals for both for with and without charge groups.
4479 compact_and_copy_vec_cg(dd
->ncg_home
, move
, cgindex
,
4480 nvec
, as_rvec_array(state
->x
.data()), comm
, FALSE
);
4483 home_pos_cg
-= *ncg_moved
;
4487 gmx_incons("unimplemented");
4493 compact_and_copy_vec_at(dd
->ncg_home
, move
, cgindex
,
4494 nvec
, vec
++, as_rvec_array(state
->x
.data()),
4498 compact_and_copy_vec_at(dd
->ncg_home
, move
, cgindex
,
4499 nvec
, vec
++, as_rvec_array(state
->v
.data()),
4504 compact_and_copy_vec_at(dd
->ncg_home
, move
, cgindex
,
4505 nvec
, vec
++, as_rvec_array(state
->cg_p
.data()),
4511 compact_ind(dd
->ncg_home
, move
,
4512 dd
->index_gl
, dd
->cgindex
, dd
->gatindex
,
4513 dd
->ga2la
, comm
->bLocalCG
,
4518 if (fr
->cutoff_scheme
== ecutsVERLET
)
4520 moved
= get_moved(comm
, dd
->ncg_home
);
4522 for (k
= 0; k
< dd
->ncg_home
; k
++)
4529 moved
= fr
->ns
->grid
->cell_index
;
4532 clear_and_mark_ind(dd
->ncg_home
, move
,
4533 dd
->index_gl
, dd
->cgindex
, dd
->gatindex
,
4534 dd
->ga2la
, comm
->bLocalCG
,
4538 cginfo_mb
= fr
->cginfo_mb
;
4540 *ncg_stay_home
= home_pos_cg
;
4541 for (d
= 0; d
< dd
->ndim
; d
++)
4546 for (dir
= 0; dir
< (dd
->nc
[dim
] == 2 ? 1 : 2); dir
++)
4549 /* Communicate the cg and atom counts */
4554 fprintf(debug
, "Sending ddim %d dir %d: ncg %d nat %d\n",
4555 d
, dir
, sbuf
[0], sbuf
[1]);
4557 dd_sendrecv_int(dd
, d
, dir
, sbuf
, 2, rbuf
, 2);
4559 if ((ncg_recv
+rbuf
[0])*DD_CGIBS
> comm
->nalloc_int
)
4561 comm
->nalloc_int
= over_alloc_dd((ncg_recv
+rbuf
[0])*DD_CGIBS
);
4562 srenew(comm
->buf_int
, comm
->nalloc_int
);
4565 /* Communicate the charge group indices, sizes and flags */
4566 dd_sendrecv_int(dd
, d
, dir
,
4567 comm
->cggl_flag
[cdd
], sbuf
[0]*DD_CGIBS
,
4568 comm
->buf_int
+ncg_recv
*DD_CGIBS
, rbuf
[0]*DD_CGIBS
);
4570 nvs
= ncg
[cdd
] + nat
[cdd
]*nvec
;
4571 i
= rbuf
[0] + rbuf
[1] *nvec
;
4572 vec_rvec_check_alloc(&comm
->vbuf
, nvr
+i
);
4574 /* Communicate cgcm and state */
4575 dd_sendrecv_rvec(dd
, d
, dir
,
4576 comm
->cgcm_state
[cdd
], nvs
,
4577 comm
->vbuf
.v
+nvr
, i
);
4578 ncg_recv
+= rbuf
[0];
4582 dd_check_alloc_ncg(fr
, state
, f
, home_pos_cg
+ ncg_recv
);
4583 if (fr
->cutoff_scheme
== ecutsGROUP
)
4585 /* Here we resize to more than necessary and shrink later */
4586 dd_resize_state(state
, f
, home_pos_at
+ ncg_recv
*MAX_CGCGSIZE
);
4589 /* Process the received charge groups */
4591 for (cg
= 0; cg
< ncg_recv
; cg
++)
4593 flag
= comm
->buf_int
[cg
*DD_CGIBS
+1];
4595 if (dim
>= npbcdim
&& dd
->nc
[dim
] > 2)
4597 /* No pbc in this dim and more than one domain boundary.
4598 * We do a separate check if a charge group didn't move too far.
4600 if (((flag
& DD_FLAG_FW(d
)) &&
4601 comm
->vbuf
.v
[buf_pos
][dim
] > cell_x1
[dim
]) ||
4602 ((flag
& DD_FLAG_BW(d
)) &&
4603 comm
->vbuf
.v
[buf_pos
][dim
] < cell_x0
[dim
]))
4605 cg_move_error(fplog
, dd
, step
, cg
, dim
,
4606 (flag
& DD_FLAG_FW(d
)) ? 1 : 0,
4607 fr
->cutoff_scheme
== ecutsGROUP
, 0,
4608 comm
->vbuf
.v
[buf_pos
],
4609 comm
->vbuf
.v
[buf_pos
],
4610 comm
->vbuf
.v
[buf_pos
][dim
]);
4617 /* Check which direction this cg should go */
4618 for (d2
= d
+1; (d2
< dd
->ndim
&& mc
== -1); d2
++)
4620 if (isDlbOn(dd
->comm
))
4622 /* The cell boundaries for dimension d2 are not equal
4623 * for each cell row of the lower dimension(s),
4624 * therefore we might need to redetermine where
4625 * this cg should go.
4628 /* If this cg crosses the box boundary in dimension d2
4629 * we can use the communicated flag, so we do not
4630 * have to worry about pbc.
4632 if (!((dd
->ci
[dim2
] == dd
->nc
[dim2
]-1 &&
4633 (flag
& DD_FLAG_FW(d2
))) ||
4634 (dd
->ci
[dim2
] == 0 &&
4635 (flag
& DD_FLAG_BW(d2
)))))
4637 /* Clear the two flags for this dimension */
4638 flag
&= ~(DD_FLAG_FW(d2
) | DD_FLAG_BW(d2
));
4639 /* Determine the location of this cg
4640 * in lattice coordinates
4642 pos_d
= comm
->vbuf
.v
[buf_pos
][dim2
];
4645 for (d3
= dim2
+1; d3
< DIM
; d3
++)
4648 comm
->vbuf
.v
[buf_pos
][d3
]*tcm
[d3
][dim2
];
4651 /* Check of we are not at the box edge.
4652 * pbc is only handled in the first step above,
4653 * but this check could move over pbc while
4654 * the first step did not due to different rounding.
4656 if (pos_d
>= cell_x1
[dim2
] &&
4657 dd
->ci
[dim2
] != dd
->nc
[dim2
]-1)
4659 flag
|= DD_FLAG_FW(d2
);
4661 else if (pos_d
< cell_x0
[dim2
] &&
4664 flag
|= DD_FLAG_BW(d2
);
4666 comm
->buf_int
[cg
*DD_CGIBS
+1] = flag
;
4669 /* Set to which neighboring cell this cg should go */
4670 if (flag
& DD_FLAG_FW(d2
))
4674 else if (flag
& DD_FLAG_BW(d2
))
4676 if (dd
->nc
[dd
->dim
[d2
]] > 2)
4688 nrcg
= flag
& DD_FLAG_NRCG
;
4691 if (home_pos_cg
+1 > dd
->cg_nalloc
)
4693 dd
->cg_nalloc
= over_alloc_dd(home_pos_cg
+1);
4694 srenew(dd
->index_gl
, dd
->cg_nalloc
);
4695 srenew(dd
->cgindex
, dd
->cg_nalloc
+1);
4697 /* Set the global charge group index and size */
4698 dd
->index_gl
[home_pos_cg
] = comm
->buf_int
[cg
*DD_CGIBS
];
4699 dd
->cgindex
[home_pos_cg
+1] = dd
->cgindex
[home_pos_cg
] + nrcg
;
4700 /* Copy the state from the buffer */
4701 if (fr
->cutoff_scheme
== ecutsGROUP
)
4704 copy_rvec(comm
->vbuf
.v
[buf_pos
], cg_cm
[home_pos_cg
]);
4708 /* Set the cginfo */
4709 fr
->cginfo
[home_pos_cg
] = ddcginfo(cginfo_mb
,
4710 dd
->index_gl
[home_pos_cg
]);
4713 comm
->bLocalCG
[dd
->index_gl
[home_pos_cg
]] = TRUE
;
4716 for (i
= 0; i
< nrcg
; i
++)
4718 copy_rvec(comm
->vbuf
.v
[buf_pos
++],
4719 state
->x
[home_pos_at
+i
]);
4723 for (i
= 0; i
< nrcg
; i
++)
4725 copy_rvec(comm
->vbuf
.v
[buf_pos
++],
4726 state
->v
[home_pos_at
+i
]);
4731 for (i
= 0; i
< nrcg
; i
++)
4733 copy_rvec(comm
->vbuf
.v
[buf_pos
++],
4734 state
->cg_p
[home_pos_at
+i
]);
4738 home_pos_at
+= nrcg
;
4742 /* Reallocate the buffers if necessary */
4743 if (ncg
[mc
]+1 > comm
->cggl_flag_nalloc
[mc
])
4745 comm
->cggl_flag_nalloc
[mc
] = over_alloc_dd(ncg
[mc
]+1);
4746 srenew(comm
->cggl_flag
[mc
], comm
->cggl_flag_nalloc
[mc
]*DD_CGIBS
);
4748 nvr
= ncg
[mc
] + nat
[mc
]*nvec
;
4749 if (nvr
+ 1 + nrcg
*nvec
> comm
->cgcm_state_nalloc
[mc
])
4751 comm
->cgcm_state_nalloc
[mc
] = over_alloc_dd(nvr
+ 1 + nrcg
*nvec
);
4752 srenew(comm
->cgcm_state
[mc
], comm
->cgcm_state_nalloc
[mc
]);
4754 /* Copy from the receive to the send buffers */
4755 memcpy(comm
->cggl_flag
[mc
] + ncg
[mc
]*DD_CGIBS
,
4756 comm
->buf_int
+ cg
*DD_CGIBS
,
4757 DD_CGIBS
*sizeof(int));
4758 memcpy(comm
->cgcm_state
[mc
][nvr
],
4759 comm
->vbuf
.v
[buf_pos
],
4760 (1+nrcg
*nvec
)*sizeof(rvec
));
4761 buf_pos
+= 1 + nrcg
*nvec
;
4768 /* With sorting (!bCompact) the indices are now only partially up to date
4769 * and ncg_home and nat_home are not the real count, since there are
4770 * "holes" in the arrays for the charge groups that moved to neighbors.
4772 if (fr
->cutoff_scheme
== ecutsVERLET
)
4774 moved
= get_moved(comm
, home_pos_cg
);
4776 for (i
= dd
->ncg_home
; i
< home_pos_cg
; i
++)
4781 dd
->ncg_home
= home_pos_cg
;
4782 dd
->nat_home
= home_pos_at
;
4784 if (fr
->cutoff_scheme
== ecutsGROUP
&& !bCompact
)
4786 /* We overallocated before, we need to set the right size here */
4787 dd_resize_state(state
, f
, dd
->nat_home
);
4793 "Finished repartitioning: cgs moved out %d, new home %d\n",
4794 *ncg_moved
, dd
->ncg_home
-*ncg_moved
);
4799 void dd_cycles_add(const gmx_domdec_t
*dd
, float cycles
, int ddCycl
)
4801 /* Note that the cycles value can be incorrect, either 0 or some
4802 * extremely large value, when our thread migrated to another core
4803 * with an unsynchronized cycle counter. If this happens less often
4804 * that once per nstlist steps, this will not cause issues, since
4805 * we later subtract the maximum value from the sum over nstlist steps.
4806 * A zero count will slightly lower the total, but that's a small effect.
4807 * Note that the main purpose of the subtraction of the maximum value
4808 * is to avoid throwing off the load balancing when stalls occur due
4809 * e.g. system activity or network congestion.
4811 dd
->comm
->cycl
[ddCycl
] += cycles
;
4812 dd
->comm
->cycl_n
[ddCycl
]++;
4813 if (cycles
> dd
->comm
->cycl_max
[ddCycl
])
4815 dd
->comm
->cycl_max
[ddCycl
] = cycles
;
4819 static double force_flop_count(t_nrnb
*nrnb
)
4826 for (i
= 0; i
< eNR_NBKERNEL_FREE_ENERGY
; i
++)
4828 /* To get closer to the real timings, we half the count
4829 * for the normal loops and again half it for water loops.
4832 if (strstr(name
, "W3") != nullptr || strstr(name
, "W4") != nullptr)
4834 sum
+= nrnb
->n
[i
]*0.25*cost_nrnb(i
);
4838 sum
+= nrnb
->n
[i
]*0.50*cost_nrnb(i
);
4841 for (i
= eNR_NBKERNEL_FREE_ENERGY
; i
<= eNR_NB14
; i
++)
4844 if (strstr(name
, "W3") != nullptr || strstr(name
, "W4") != nullptr)
4846 sum
+= nrnb
->n
[i
]*cost_nrnb(i
);
4849 for (i
= eNR_BONDS
; i
<= eNR_WALLS
; i
++)
4851 sum
+= nrnb
->n
[i
]*cost_nrnb(i
);
4857 void dd_force_flop_start(gmx_domdec_t
*dd
, t_nrnb
*nrnb
)
4859 if (dd
->comm
->eFlop
)
4861 dd
->comm
->flop
-= force_flop_count(nrnb
);
4864 void dd_force_flop_stop(gmx_domdec_t
*dd
, t_nrnb
*nrnb
)
4866 if (dd
->comm
->eFlop
)
4868 dd
->comm
->flop
+= force_flop_count(nrnb
);
4873 static void clear_dd_cycle_counts(gmx_domdec_t
*dd
)
4877 for (i
= 0; i
< ddCyclNr
; i
++)
4879 dd
->comm
->cycl
[i
] = 0;
4880 dd
->comm
->cycl_n
[i
] = 0;
4881 dd
->comm
->cycl_max
[i
] = 0;
4884 dd
->comm
->flop_n
= 0;
4887 static void get_load_distribution(gmx_domdec_t
*dd
, gmx_wallcycle_t wcycle
)
4889 gmx_domdec_comm_t
*comm
;
4890 domdec_load_t
*load
;
4891 domdec_root_t
*root
= nullptr;
4893 float cell_frac
= 0, sbuf
[DD_NLOAD_MAX
];
4898 fprintf(debug
, "get_load_distribution start\n");
4901 wallcycle_start(wcycle
, ewcDDCOMMLOAD
);
4905 bSepPME
= (dd
->pme_nodeid
>= 0);
4907 if (dd
->ndim
== 0 && bSepPME
)
4909 /* Without decomposition, but with PME nodes, we need the load */
4910 comm
->load
[0].mdf
= comm
->cycl
[ddCyclPPduringPME
];
4911 comm
->load
[0].pme
= comm
->cycl
[ddCyclPME
];
4914 for (d
= dd
->ndim
-1; d
>= 0; d
--)
4917 /* Check if we participate in the communication in this dimension */
4918 if (d
== dd
->ndim
-1 ||
4919 (dd
->ci
[dd
->dim
[d
+1]] == 0 && dd
->ci
[dd
->dim
[dd
->ndim
-1]] == 0))
4921 load
= &comm
->load
[d
];
4922 if (isDlbOn(dd
->comm
))
4924 cell_frac
= comm
->cell_f1
[d
] - comm
->cell_f0
[d
];
4927 if (d
== dd
->ndim
-1)
4929 sbuf
[pos
++] = dd_force_load(comm
);
4930 sbuf
[pos
++] = sbuf
[0];
4931 if (isDlbOn(dd
->comm
))
4933 sbuf
[pos
++] = sbuf
[0];
4934 sbuf
[pos
++] = cell_frac
;
4937 sbuf
[pos
++] = comm
->cell_f_max0
[d
];
4938 sbuf
[pos
++] = comm
->cell_f_min1
[d
];
4943 sbuf
[pos
++] = comm
->cycl
[ddCyclPPduringPME
];
4944 sbuf
[pos
++] = comm
->cycl
[ddCyclPME
];
4949 sbuf
[pos
++] = comm
->load
[d
+1].sum
;
4950 sbuf
[pos
++] = comm
->load
[d
+1].max
;
4951 if (isDlbOn(dd
->comm
))
4953 sbuf
[pos
++] = comm
->load
[d
+1].sum_m
;
4954 sbuf
[pos
++] = comm
->load
[d
+1].cvol_min
*cell_frac
;
4955 sbuf
[pos
++] = comm
->load
[d
+1].flags
;
4958 sbuf
[pos
++] = comm
->cell_f_max0
[d
];
4959 sbuf
[pos
++] = comm
->cell_f_min1
[d
];
4964 sbuf
[pos
++] = comm
->load
[d
+1].mdf
;
4965 sbuf
[pos
++] = comm
->load
[d
+1].pme
;
4969 /* Communicate a row in DD direction d.
4970 * The communicators are setup such that the root always has rank 0.
4973 MPI_Gather(sbuf
, load
->nload
*sizeof(float), MPI_BYTE
,
4974 load
->load
, load
->nload
*sizeof(float), MPI_BYTE
,
4975 0, comm
->mpi_comm_load
[d
]);
4977 if (dd
->ci
[dim
] == dd
->master_ci
[dim
])
4979 /* We are the root, process this row */
4982 root
= comm
->root
[d
];
4992 for (i
= 0; i
< dd
->nc
[dim
]; i
++)
4994 load
->sum
+= load
->load
[pos
++];
4995 load
->max
= std::max(load
->max
, load
->load
[pos
]);
4997 if (isDlbOn(dd
->comm
))
5001 /* This direction could not be load balanced properly,
5002 * therefore we need to use the maximum iso the average load.
5004 load
->sum_m
= std::max(load
->sum_m
, load
->load
[pos
]);
5008 load
->sum_m
+= load
->load
[pos
];
5011 load
->cvol_min
= std::min(load
->cvol_min
, load
->load
[pos
]);
5015 load
->flags
= (int)(load
->load
[pos
++] + 0.5);
5019 root
->cell_f_max0
[i
] = load
->load
[pos
++];
5020 root
->cell_f_min1
[i
] = load
->load
[pos
++];
5025 load
->mdf
= std::max(load
->mdf
, load
->load
[pos
]);
5027 load
->pme
= std::max(load
->pme
, load
->load
[pos
]);
5031 if (isDlbOn(comm
) && root
->bLimited
)
5033 load
->sum_m
*= dd
->nc
[dim
];
5034 load
->flags
|= (1<<d
);
5042 comm
->nload
+= dd_load_count(comm
);
5043 comm
->load_step
+= comm
->cycl
[ddCyclStep
];
5044 comm
->load_sum
+= comm
->load
[0].sum
;
5045 comm
->load_max
+= comm
->load
[0].max
;
5048 for (d
= 0; d
< dd
->ndim
; d
++)
5050 if (comm
->load
[0].flags
& (1<<d
))
5052 comm
->load_lim
[d
]++;
5058 comm
->load_mdf
+= comm
->load
[0].mdf
;
5059 comm
->load_pme
+= comm
->load
[0].pme
;
5063 wallcycle_stop(wcycle
, ewcDDCOMMLOAD
);
5067 fprintf(debug
, "get_load_distribution finished\n");
5071 static float dd_force_load_fraction(gmx_domdec_t
*dd
)
5073 /* Return the relative performance loss on the total run time
5074 * due to the force calculation load imbalance.
5076 if (dd
->comm
->nload
> 0 && dd
->comm
->load_step
> 0)
5078 return dd
->comm
->load_sum
/(dd
->comm
->load_step
*dd
->nnodes
);
5086 static float dd_force_imb_perf_loss(gmx_domdec_t
*dd
)
5088 /* Return the relative performance loss on the total run time
5089 * due to the force calculation load imbalance.
5091 if (dd
->comm
->nload
> 0 && dd
->comm
->load_step
> 0)
5094 (dd
->comm
->load_max
*dd
->nnodes
- dd
->comm
->load_sum
)/
5095 (dd
->comm
->load_step
*dd
->nnodes
);
5103 static void print_dd_load_av(FILE *fplog
, gmx_domdec_t
*dd
)
5105 gmx_domdec_comm_t
*comm
= dd
->comm
;
5107 /* Only the master rank prints loads and only if we measured loads */
5108 if (!DDMASTER(dd
) || comm
->nload
== 0)
5114 int numPpRanks
= dd
->nnodes
;
5115 int numPmeRanks
= (dd
->pme_nodeid
>= 0) ? comm
->npmenodes
: 0;
5116 int numRanks
= numPpRanks
+ numPmeRanks
;
5117 float lossFraction
= 0;
5119 /* Print the average load imbalance and performance loss */
5120 if (dd
->nnodes
> 1 && comm
->load_sum
> 0)
5122 float imbalance
= comm
->load_max
*numPpRanks
/comm
->load_sum
- 1;
5123 lossFraction
= dd_force_imb_perf_loss(dd
);
5125 std::string msg
= "\n Dynamic load balancing report:\n";
5126 std::string dlbStateStr
= "";
5128 switch (dd
->comm
->dlbState
)
5131 dlbStateStr
= "DLB was off during the run per user request.";
5133 case edlbsOffForever
:
5134 /* Currectly this can happen due to performance loss observed, cell size
5135 * limitations or incompatibility with other settings observed during
5136 * determineInitialDlbState(). */
5137 dlbStateStr
= "DLB got disabled because it was unsuitable to use.";
5139 case edlbsOffCanTurnOn
:
5140 dlbStateStr
= "DLB was off during the run due to low measured imbalance.";
5142 case edlbsOffTemporarilyLocked
:
5143 dlbStateStr
= "DLB was locked at the end of the run due to unfinished PP-PME balancing.";
5145 case edlbsOnCanTurnOff
:
5146 dlbStateStr
= "DLB was turned on during the run due to measured imbalance.";
5149 dlbStateStr
= "DLB was permanently on during the run per user request.";
5152 GMX_ASSERT(false, "Undocumented DLB state");
5155 msg
+= " " + dlbStateStr
+ "\n";
5156 msg
+= gmx::formatString(" Average load imbalance: %.1f%%.\n", imbalance
*100);
5157 msg
+= gmx::formatString(" The balanceable part of the MD step is %d%%, load imbalance is computed from this.\n",
5158 static_cast<int>(dd_force_load_fraction(dd
)*100 + 0.5));
5159 msg
+= gmx::formatString(" Part of the total run time spent waiting due to load imbalance: %.1f%%.\n",
5161 fprintf(fplog
, "%s", msg
.c_str());
5162 fprintf(stderr
, "%s", msg
.c_str());
5165 /* Print during what percentage of steps the load balancing was limited */
5166 bool dlbWasLimited
= false;
5169 sprintf(buf
, " Steps where the load balancing was limited by -rdd, -rcon and/or -dds:");
5170 for (int d
= 0; d
< dd
->ndim
; d
++)
5172 int limitPercentage
= (200*comm
->load_lim
[d
] + 1)/(2*comm
->nload
);
5173 sprintf(buf
+strlen(buf
), " %c %d %%",
5174 dim2char(dd
->dim
[d
]), limitPercentage
);
5175 if (limitPercentage
>= 50)
5177 dlbWasLimited
= true;
5180 sprintf(buf
+ strlen(buf
), "\n");
5181 fprintf(fplog
, "%s", buf
);
5182 fprintf(stderr
, "%s", buf
);
5185 /* Print the performance loss due to separate PME - PP rank imbalance */
5186 float lossFractionPme
= 0;
5187 if (numPmeRanks
> 0 && comm
->load_mdf
> 0 && comm
->load_step
> 0)
5189 float pmeForceRatio
= comm
->load_pme
/comm
->load_mdf
;
5190 lossFractionPme
= (comm
->load_pme
- comm
->load_mdf
)/comm
->load_step
;
5191 if (lossFractionPme
<= 0)
5193 lossFractionPme
*= numPmeRanks
/static_cast<float>(numRanks
);
5197 lossFractionPme
*= numPpRanks
/static_cast<float>(numRanks
);
5199 sprintf(buf
, " Average PME mesh/force load: %5.3f\n", pmeForceRatio
);
5200 fprintf(fplog
, "%s", buf
);
5201 fprintf(stderr
, "%s", buf
);
5202 sprintf(buf
, " Part of the total run time spent waiting due to PP/PME imbalance: %.1f %%\n", fabs(lossFractionPme
)*100);
5203 fprintf(fplog
, "%s", buf
);
5204 fprintf(stderr
, "%s", buf
);
5206 fprintf(fplog
, "\n");
5207 fprintf(stderr
, "\n");
5209 if (lossFraction
>= DD_PERF_LOSS_WARN
)
5212 "NOTE: %.1f %% of the available CPU time was lost due to load imbalance\n"
5213 " in the domain decomposition.\n", lossFraction
*100);
5216 sprintf(buf
+strlen(buf
), " You might want to use dynamic load balancing (option -dlb.)\n");
5218 else if (dlbWasLimited
)
5220 sprintf(buf
+strlen(buf
), " You might want to decrease the cell size limit (options -rdd, -rcon and/or -dds).\n");
5222 fprintf(fplog
, "%s\n", buf
);
5223 fprintf(stderr
, "%s\n", buf
);
5225 if (numPmeRanks
> 0 && fabs(lossFractionPme
) >= DD_PERF_LOSS_WARN
)
5228 "NOTE: %.1f %% performance was lost because the PME ranks\n"
5229 " had %s work to do than the PP ranks.\n"
5230 " You might want to %s the number of PME ranks\n"
5231 " or %s the cut-off and the grid spacing.\n",
5232 fabs(lossFractionPme
*100),
5233 (lossFractionPme
< 0) ? "less" : "more",
5234 (lossFractionPme
< 0) ? "decrease" : "increase",
5235 (lossFractionPme
< 0) ? "decrease" : "increase");
5236 fprintf(fplog
, "%s\n", buf
);
5237 fprintf(stderr
, "%s\n", buf
);
5241 static float dd_vol_min(gmx_domdec_t
*dd
)
5243 return dd
->comm
->load
[0].cvol_min
*dd
->nnodes
;
5246 static gmx_bool
dd_load_flags(gmx_domdec_t
*dd
)
5248 return dd
->comm
->load
[0].flags
;
5251 static float dd_f_imbal(gmx_domdec_t
*dd
)
5253 if (dd
->comm
->load
[0].sum
> 0)
5255 return dd
->comm
->load
[0].max
*dd
->nnodes
/dd
->comm
->load
[0].sum
- 1.0f
;
5259 /* Something is wrong in the cycle counting, report no load imbalance */
5264 float dd_pme_f_ratio(gmx_domdec_t
*dd
)
5266 /* Should only be called on the DD master rank */
5267 assert(DDMASTER(dd
));
5269 if (dd
->comm
->load
[0].mdf
> 0 && dd
->comm
->cycl_n
[ddCyclPME
] > 0)
5271 return dd
->comm
->load
[0].pme
/dd
->comm
->load
[0].mdf
;
5279 static void dd_print_load(FILE *fplog
, gmx_domdec_t
*dd
, gmx_int64_t step
)
5284 flags
= dd_load_flags(dd
);
5288 "DD load balancing is limited by minimum cell size in dimension");
5289 for (d
= 0; d
< dd
->ndim
; d
++)
5293 fprintf(fplog
, " %c", dim2char(dd
->dim
[d
]));
5296 fprintf(fplog
, "\n");
5298 fprintf(fplog
, "DD step %s", gmx_step_str(step
, buf
));
5299 if (isDlbOn(dd
->comm
))
5301 fprintf(fplog
, " vol min/aver %5.3f%c",
5302 dd_vol_min(dd
), flags
? '!' : ' ');
5306 fprintf(fplog
, " load imb.: force %4.1f%%", dd_f_imbal(dd
)*100);
5308 if (dd
->comm
->cycl_n
[ddCyclPME
])
5310 fprintf(fplog
, " pme mesh/force %5.3f", dd_pme_f_ratio(dd
));
5312 fprintf(fplog
, "\n\n");
5315 static void dd_print_load_verbose(gmx_domdec_t
*dd
)
5317 if (isDlbOn(dd
->comm
))
5319 fprintf(stderr
, "vol %4.2f%c ",
5320 dd_vol_min(dd
), dd_load_flags(dd
) ? '!' : ' ');
5324 fprintf(stderr
, "imb F %2d%% ", (int)(dd_f_imbal(dd
)*100+0.5));
5326 if (dd
->comm
->cycl_n
[ddCyclPME
])
5328 fprintf(stderr
, "pme/F %4.2f ", dd_pme_f_ratio(dd
));
5333 static void make_load_communicator(gmx_domdec_t
*dd
, int dim_ind
, ivec loc
)
5338 domdec_root_t
*root
;
5339 gmx_bool bPartOfGroup
= FALSE
;
5341 dim
= dd
->dim
[dim_ind
];
5342 copy_ivec(loc
, loc_c
);
5343 for (i
= 0; i
< dd
->nc
[dim
]; i
++)
5346 rank
= dd_index(dd
->nc
, loc_c
);
5347 if (rank
== dd
->rank
)
5349 /* This process is part of the group */
5350 bPartOfGroup
= TRUE
;
5353 MPI_Comm_split(dd
->mpi_comm_all
, bPartOfGroup
? 0 : MPI_UNDEFINED
, dd
->rank
,
5357 dd
->comm
->mpi_comm_load
[dim_ind
] = c_row
;
5358 if (!isDlbDisabled(dd
->comm
))
5360 if (dd
->ci
[dim
] == dd
->master_ci
[dim
])
5362 /* This is the root process of this row */
5363 snew(dd
->comm
->root
[dim_ind
], 1);
5364 root
= dd
->comm
->root
[dim_ind
];
5365 snew(root
->cell_f
, DD_CELL_F_SIZE(dd
, dim_ind
));
5366 snew(root
->old_cell_f
, dd
->nc
[dim
]+1);
5367 snew(root
->bCellMin
, dd
->nc
[dim
]);
5370 snew(root
->cell_f_max0
, dd
->nc
[dim
]);
5371 snew(root
->cell_f_min1
, dd
->nc
[dim
]);
5372 snew(root
->bound_min
, dd
->nc
[dim
]);
5373 snew(root
->bound_max
, dd
->nc
[dim
]);
5375 snew(root
->buf_ncd
, dd
->nc
[dim
]);
5379 /* This is not a root process, we only need to receive cell_f */
5380 snew(dd
->comm
->cell_f_row
, DD_CELL_F_SIZE(dd
, dim_ind
));
5383 if (dd
->ci
[dim
] == dd
->master_ci
[dim
])
5385 snew(dd
->comm
->load
[dim_ind
].load
, dd
->nc
[dim
]*DD_NLOAD_MAX
);
5391 void dd_setup_dlb_resource_sharing(t_commrec
*cr
,
5395 int physicalnode_id_hash
;
5397 MPI_Comm mpi_comm_pp_physicalnode
;
5399 if (!(cr
->duty
& DUTY_PP
) || gpu_id
< 0)
5401 /* Only ranks with short-ranged tasks (currently) use GPUs.
5402 * If we don't have GPUs assigned, there are no resources to share.
5407 physicalnode_id_hash
= gmx_physicalnode_id_hash();
5413 fprintf(debug
, "dd_setup_dd_dlb_gpu_sharing:\n");
5414 fprintf(debug
, "DD PP rank %d physical node hash %d gpu_id %d\n",
5415 dd
->rank
, physicalnode_id_hash
, gpu_id
);
5417 /* Split the PP communicator over the physical nodes */
5418 /* TODO: See if we should store this (before), as it's also used for
5419 * for the nodecomm summution.
5421 MPI_Comm_split(dd
->mpi_comm_all
, physicalnode_id_hash
, dd
->rank
,
5422 &mpi_comm_pp_physicalnode
);
5423 MPI_Comm_split(mpi_comm_pp_physicalnode
, gpu_id
, dd
->rank
,
5424 &dd
->comm
->mpi_comm_gpu_shared
);
5425 MPI_Comm_free(&mpi_comm_pp_physicalnode
);
5426 MPI_Comm_size(dd
->comm
->mpi_comm_gpu_shared
, &dd
->comm
->nrank_gpu_shared
);
5430 fprintf(debug
, "nrank_gpu_shared %d\n", dd
->comm
->nrank_gpu_shared
);
5433 /* Note that some ranks could share a GPU, while others don't */
5435 if (dd
->comm
->nrank_gpu_shared
== 1)
5437 MPI_Comm_free(&dd
->comm
->mpi_comm_gpu_shared
);
5440 GMX_UNUSED_VALUE(cr
);
5441 GMX_UNUSED_VALUE(gpu_id
);
5445 static void make_load_communicators(gmx_domdec_t gmx_unused
*dd
)
5448 int dim0
, dim1
, i
, j
;
5453 fprintf(debug
, "Making load communicators\n");
5456 snew(dd
->comm
->load
, std::max(dd
->ndim
, 1));
5457 snew(dd
->comm
->mpi_comm_load
, std::max(dd
->ndim
, 1));
5465 make_load_communicator(dd
, 0, loc
);
5469 for (i
= 0; i
< dd
->nc
[dim0
]; i
++)
5472 make_load_communicator(dd
, 1, loc
);
5478 for (i
= 0; i
< dd
->nc
[dim0
]; i
++)
5482 for (j
= 0; j
< dd
->nc
[dim1
]; j
++)
5485 make_load_communicator(dd
, 2, loc
);
5492 fprintf(debug
, "Finished making load communicators\n");
5497 /*! \brief Sets up the relation between neighboring domains and zones */
5498 static void setup_neighbor_relations(gmx_domdec_t
*dd
)
5500 int d
, dim
, i
, j
, m
;
5502 gmx_domdec_zones_t
*zones
;
5503 gmx_domdec_ns_ranges_t
*izone
;
5505 for (d
= 0; d
< dd
->ndim
; d
++)
5508 copy_ivec(dd
->ci
, tmp
);
5509 tmp
[dim
] = (tmp
[dim
] + 1) % dd
->nc
[dim
];
5510 dd
->neighbor
[d
][0] = ddcoord2ddnodeid(dd
, tmp
);
5511 copy_ivec(dd
->ci
, tmp
);
5512 tmp
[dim
] = (tmp
[dim
] - 1 + dd
->nc
[dim
]) % dd
->nc
[dim
];
5513 dd
->neighbor
[d
][1] = ddcoord2ddnodeid(dd
, tmp
);
5516 fprintf(debug
, "DD rank %d neighbor ranks in dir %d are + %d - %d\n",
5519 dd
->neighbor
[d
][1]);
5523 int nzone
= (1 << dd
->ndim
);
5524 int nizone
= (1 << std::max(dd
->ndim
- 1, 0));
5525 assert(nizone
>= 1 && nizone
<= DD_MAXIZONE
);
5527 zones
= &dd
->comm
->zones
;
5529 for (i
= 0; i
< nzone
; i
++)
5532 clear_ivec(zones
->shift
[i
]);
5533 for (d
= 0; d
< dd
->ndim
; d
++)
5535 zones
->shift
[i
][dd
->dim
[d
]] = dd_zo
[i
][m
++];
5540 for (i
= 0; i
< nzone
; i
++)
5542 for (d
= 0; d
< DIM
; d
++)
5544 s
[d
] = dd
->ci
[d
] - zones
->shift
[i
][d
];
5549 else if (s
[d
] >= dd
->nc
[d
])
5555 zones
->nizone
= nizone
;
5556 for (i
= 0; i
< zones
->nizone
; i
++)
5558 assert(ddNonbondedZonePairRanges
[i
][0] == i
);
5560 izone
= &zones
->izone
[i
];
5561 /* dd_zp3 is for 3D decomposition, for fewer dimensions use only
5562 * j-zones up to nzone.
5564 izone
->j0
= std::min(ddNonbondedZonePairRanges
[i
][1], nzone
);
5565 izone
->j1
= std::min(ddNonbondedZonePairRanges
[i
][2], nzone
);
5566 for (dim
= 0; dim
< DIM
; dim
++)
5568 if (dd
->nc
[dim
] == 1)
5570 /* All shifts should be allowed */
5571 izone
->shift0
[dim
] = -1;
5572 izone
->shift1
[dim
] = 1;
5576 /* Determine the min/max j-zone shift wrt the i-zone */
5577 izone
->shift0
[dim
] = 1;
5578 izone
->shift1
[dim
] = -1;
5579 for (j
= izone
->j0
; j
< izone
->j1
; j
++)
5581 int shift_diff
= zones
->shift
[j
][dim
] - zones
->shift
[i
][dim
];
5582 if (shift_diff
< izone
->shift0
[dim
])
5584 izone
->shift0
[dim
] = shift_diff
;
5586 if (shift_diff
> izone
->shift1
[dim
])
5588 izone
->shift1
[dim
] = shift_diff
;
5595 if (!isDlbDisabled(dd
->comm
))
5597 snew(dd
->comm
->root
, dd
->ndim
);
5600 if (dd
->comm
->bRecordLoad
)
5602 make_load_communicators(dd
);
5606 static void make_pp_communicator(FILE *fplog
,
5608 t_commrec gmx_unused
*cr
,
5609 int gmx_unused reorder
)
5612 gmx_domdec_comm_t
*comm
;
5619 if (comm
->bCartesianPP
)
5621 /* Set up cartesian communication for the particle-particle part */
5624 fprintf(fplog
, "Will use a Cartesian communicator: %d x %d x %d\n",
5625 dd
->nc
[XX
], dd
->nc
[YY
], dd
->nc
[ZZ
]);
5628 for (int i
= 0; i
< DIM
; i
++)
5632 MPI_Cart_create(cr
->mpi_comm_mygroup
, DIM
, dd
->nc
, periods
, reorder
,
5634 /* We overwrite the old communicator with the new cartesian one */
5635 cr
->mpi_comm_mygroup
= comm_cart
;
5638 dd
->mpi_comm_all
= cr
->mpi_comm_mygroup
;
5639 MPI_Comm_rank(dd
->mpi_comm_all
, &dd
->rank
);
5641 if (comm
->bCartesianPP_PME
)
5643 /* Since we want to use the original cartesian setup for sim,
5644 * and not the one after split, we need to make an index.
5646 snew(comm
->ddindex2ddnodeid
, dd
->nnodes
);
5647 comm
->ddindex2ddnodeid
[dd_index(dd
->nc
, dd
->ci
)] = dd
->rank
;
5648 gmx_sumi(dd
->nnodes
, comm
->ddindex2ddnodeid
, cr
);
5649 /* Get the rank of the DD master,
5650 * above we made sure that the master node is a PP node.
5660 MPI_Allreduce(&rank
, &dd
->masterrank
, 1, MPI_INT
, MPI_SUM
, dd
->mpi_comm_all
);
5662 else if (comm
->bCartesianPP
)
5664 if (cr
->npmenodes
== 0)
5666 /* The PP communicator is also
5667 * the communicator for this simulation
5669 cr
->mpi_comm_mysim
= cr
->mpi_comm_mygroup
;
5671 cr
->nodeid
= dd
->rank
;
5673 MPI_Cart_coords(dd
->mpi_comm_all
, dd
->rank
, DIM
, dd
->ci
);
5675 /* We need to make an index to go from the coordinates
5676 * to the nodeid of this simulation.
5678 snew(comm
->ddindex2simnodeid
, dd
->nnodes
);
5679 snew(buf
, dd
->nnodes
);
5680 if (cr
->duty
& DUTY_PP
)
5682 buf
[dd_index(dd
->nc
, dd
->ci
)] = cr
->sim_nodeid
;
5684 /* Communicate the ddindex to simulation nodeid index */
5685 MPI_Allreduce(buf
, comm
->ddindex2simnodeid
, dd
->nnodes
, MPI_INT
, MPI_SUM
,
5686 cr
->mpi_comm_mysim
);
5689 /* Determine the master coordinates and rank.
5690 * The DD master should be the same node as the master of this sim.
5692 for (int i
= 0; i
< dd
->nnodes
; i
++)
5694 if (comm
->ddindex2simnodeid
[i
] == 0)
5696 ddindex2xyz(dd
->nc
, i
, dd
->master_ci
);
5697 MPI_Cart_rank(dd
->mpi_comm_all
, dd
->master_ci
, &dd
->masterrank
);
5702 fprintf(debug
, "The master rank is %d\n", dd
->masterrank
);
5707 /* No Cartesian communicators */
5708 /* We use the rank in dd->comm->all as DD index */
5709 ddindex2xyz(dd
->nc
, dd
->rank
, dd
->ci
);
5710 /* The simulation master nodeid is 0, so the DD master rank is also 0 */
5712 clear_ivec(dd
->master_ci
);
5719 "Domain decomposition rank %d, coordinates %d %d %d\n\n",
5720 dd
->rank
, dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
5725 "Domain decomposition rank %d, coordinates %d %d %d\n\n",
5726 dd
->rank
, dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
5730 static void receive_ddindex2simnodeid(gmx_domdec_t
*dd
,
5734 gmx_domdec_comm_t
*comm
= dd
->comm
;
5736 if (!comm
->bCartesianPP_PME
&& comm
->bCartesianPP
)
5739 snew(comm
->ddindex2simnodeid
, dd
->nnodes
);
5740 snew(buf
, dd
->nnodes
);
5741 if (cr
->duty
& DUTY_PP
)
5743 buf
[dd_index(dd
->nc
, dd
->ci
)] = cr
->sim_nodeid
;
5745 /* Communicate the ddindex to simulation nodeid index */
5746 MPI_Allreduce(buf
, comm
->ddindex2simnodeid
, dd
->nnodes
, MPI_INT
, MPI_SUM
,
5747 cr
->mpi_comm_mysim
);
5751 GMX_UNUSED_VALUE(dd
);
5752 GMX_UNUSED_VALUE(cr
);
5756 static gmx_domdec_master_t
*init_gmx_domdec_master_t(gmx_domdec_t
*dd
,
5757 int ncg
, int natoms
)
5759 gmx_domdec_master_t
*ma
;
5764 snew(ma
->ncg
, dd
->nnodes
);
5765 snew(ma
->index
, dd
->nnodes
+1);
5767 snew(ma
->nat
, dd
->nnodes
);
5768 snew(ma
->ibuf
, dd
->nnodes
*2);
5769 snew(ma
->cell_x
, DIM
);
5770 for (i
= 0; i
< DIM
; i
++)
5772 snew(ma
->cell_x
[i
], dd
->nc
[i
]+1);
5775 if (dd
->nnodes
<= GMX_DD_NNODES_SENDRECV
)
5781 snew(ma
->vbuf
, natoms
);
5787 static void split_communicator(FILE *fplog
, t_commrec
*cr
, gmx_domdec_t
*dd
,
5788 int gmx_unused dd_rank_order
,
5789 int gmx_unused reorder
)
5791 gmx_domdec_comm_t
*comm
;
5800 if (comm
->bCartesianPP
)
5802 for (i
= 1; i
< DIM
; i
++)
5804 bDiv
[i
] = ((cr
->npmenodes
*dd
->nc
[i
]) % (dd
->nnodes
) == 0);
5806 if (bDiv
[YY
] || bDiv
[ZZ
])
5808 comm
->bCartesianPP_PME
= TRUE
;
5809 /* If we have 2D PME decomposition, which is always in x+y,
5810 * we stack the PME only nodes in z.
5811 * Otherwise we choose the direction that provides the thinnest slab
5812 * of PME only nodes as this will have the least effect
5813 * on the PP communication.
5814 * But for the PME communication the opposite might be better.
5816 if (bDiv
[ZZ
] && (comm
->npmenodes_y
> 1 ||
5818 dd
->nc
[YY
] > dd
->nc
[ZZ
]))
5820 comm
->cartpmedim
= ZZ
;
5824 comm
->cartpmedim
= YY
;
5826 comm
->ntot
[comm
->cartpmedim
]
5827 += (cr
->npmenodes
*dd
->nc
[comm
->cartpmedim
])/dd
->nnodes
;
5831 fprintf(fplog
, "Number of PME-only ranks (%d) is not a multiple of nx*ny (%d*%d) or nx*nz (%d*%d)\n", cr
->npmenodes
, dd
->nc
[XX
], dd
->nc
[YY
], dd
->nc
[XX
], dd
->nc
[ZZ
]);
5833 "Will not use a Cartesian communicator for PP <-> PME\n\n");
5838 if (comm
->bCartesianPP_PME
)
5845 fprintf(fplog
, "Will use a Cartesian communicator for PP <-> PME: %d x %d x %d\n", comm
->ntot
[XX
], comm
->ntot
[YY
], comm
->ntot
[ZZ
]);
5848 for (i
= 0; i
< DIM
; i
++)
5852 MPI_Cart_create(cr
->mpi_comm_mysim
, DIM
, comm
->ntot
, periods
, reorder
,
5854 MPI_Comm_rank(comm_cart
, &rank
);
5855 if (MASTER(cr
) && rank
!= 0)
5857 gmx_fatal(FARGS
, "MPI rank 0 was renumbered by MPI_Cart_create, we do not allow this");
5860 /* With this assigment we loose the link to the original communicator
5861 * which will usually be MPI_COMM_WORLD, unless have multisim.
5863 cr
->mpi_comm_mysim
= comm_cart
;
5864 cr
->sim_nodeid
= rank
;
5866 MPI_Cart_coords(cr
->mpi_comm_mysim
, cr
->sim_nodeid
, DIM
, dd
->ci
);
5870 fprintf(fplog
, "Cartesian rank %d, coordinates %d %d %d\n\n",
5871 cr
->sim_nodeid
, dd
->ci
[XX
], dd
->ci
[YY
], dd
->ci
[ZZ
]);
5874 if (dd
->ci
[comm
->cartpmedim
] < dd
->nc
[comm
->cartpmedim
])
5878 if (cr
->npmenodes
== 0 ||
5879 dd
->ci
[comm
->cartpmedim
] >= dd
->nc
[comm
->cartpmedim
])
5881 cr
->duty
= DUTY_PME
;
5884 /* Split the sim communicator into PP and PME only nodes */
5885 MPI_Comm_split(cr
->mpi_comm_mysim
,
5887 dd_index(comm
->ntot
, dd
->ci
),
5888 &cr
->mpi_comm_mygroup
);
5892 switch (dd_rank_order
)
5894 case ddrankorderPP_PME
:
5897 fprintf(fplog
, "Order of the ranks: PP first, PME last\n");
5900 case ddrankorderINTERLEAVE
:
5901 /* Interleave the PP-only and PME-only ranks */
5904 fprintf(fplog
, "Interleaving PP and PME ranks\n");
5906 comm
->pmenodes
= dd_interleaved_pme_ranks(dd
);
5908 case ddrankorderCARTESIAN
:
5911 gmx_fatal(FARGS
, "Unknown dd_rank_order=%d", dd_rank_order
);
5914 if (dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
) == -1)
5916 cr
->duty
= DUTY_PME
;
5923 /* Split the sim communicator into PP and PME only nodes */
5924 MPI_Comm_split(cr
->mpi_comm_mysim
,
5927 &cr
->mpi_comm_mygroup
);
5928 MPI_Comm_rank(cr
->mpi_comm_mygroup
, &cr
->nodeid
);
5934 fprintf(fplog
, "This rank does only %s work.\n\n",
5935 (cr
->duty
& DUTY_PP
) ? "particle-particle" : "PME-mesh");
5939 /*! \brief Generates the MPI communicators for domain decomposition */
5940 static void make_dd_communicators(FILE *fplog
, t_commrec
*cr
,
5941 gmx_domdec_t
*dd
, int dd_rank_order
)
5943 gmx_domdec_comm_t
*comm
;
5948 copy_ivec(dd
->nc
, comm
->ntot
);
5950 comm
->bCartesianPP
= (dd_rank_order
== ddrankorderCARTESIAN
);
5951 comm
->bCartesianPP_PME
= FALSE
;
5953 /* Reorder the nodes by default. This might change the MPI ranks.
5954 * Real reordering is only supported on very few architectures,
5955 * Blue Gene is one of them.
5957 CartReorder
= (getenv("GMX_NO_CART_REORDER") == nullptr);
5959 if (cr
->npmenodes
> 0)
5961 /* Split the communicator into a PP and PME part */
5962 split_communicator(fplog
, cr
, dd
, dd_rank_order
, CartReorder
);
5963 if (comm
->bCartesianPP_PME
)
5965 /* We (possibly) reordered the nodes in split_communicator,
5966 * so it is no longer required in make_pp_communicator.
5968 CartReorder
= FALSE
;
5973 /* All nodes do PP and PME */
5975 /* We do not require separate communicators */
5976 cr
->mpi_comm_mygroup
= cr
->mpi_comm_mysim
;
5980 if (cr
->duty
& DUTY_PP
)
5982 /* Copy or make a new PP communicator */
5983 make_pp_communicator(fplog
, dd
, cr
, CartReorder
);
5987 receive_ddindex2simnodeid(dd
, cr
);
5990 if (!(cr
->duty
& DUTY_PME
))
5992 /* Set up the commnuication to our PME node */
5993 dd
->pme_nodeid
= dd_simnode2pmenode(dd
, cr
, cr
->sim_nodeid
);
5994 dd
->pme_receive_vir_ener
= receive_vir_ener(dd
, cr
);
5997 fprintf(debug
, "My pme_nodeid %d receive ener %d\n",
5998 dd
->pme_nodeid
, dd
->pme_receive_vir_ener
);
6003 dd
->pme_nodeid
= -1;
6008 dd
->ma
= init_gmx_domdec_master_t(dd
,
6010 comm
->cgs_gl
.index
[comm
->cgs_gl
.nr
]);
6014 static real
*get_slb_frac(FILE *fplog
, const char *dir
, int nc
, const char *size_string
)
6016 real
*slb_frac
, tot
;
6021 if (nc
> 1 && size_string
!= nullptr)
6025 fprintf(fplog
, "Using static load balancing for the %s direction\n",
6030 for (i
= 0; i
< nc
; i
++)
6033 sscanf(size_string
, "%20lf%n", &dbl
, &n
);
6036 gmx_fatal(FARGS
, "Incorrect or not enough DD cell size entries for direction %s: '%s'", dir
, size_string
);
6045 fprintf(fplog
, "Relative cell sizes:");
6047 for (i
= 0; i
< nc
; i
++)
6052 fprintf(fplog
, " %5.3f", slb_frac
[i
]);
6057 fprintf(fplog
, "\n");
6064 static int multi_body_bondeds_count(const gmx_mtop_t
*mtop
)
6067 gmx_mtop_ilistloop_t iloop
;
6071 iloop
= gmx_mtop_ilistloop_init(mtop
);
6072 while (gmx_mtop_ilistloop_next(iloop
, &il
, &nmol
))
6074 for (ftype
= 0; ftype
< F_NRE
; ftype
++)
6076 if ((interaction_function
[ftype
].flags
& IF_BOND
) &&
6079 n
+= nmol
*il
[ftype
].nr
/(1 + NRAL(ftype
));
6087 static int dd_getenv(FILE *fplog
, const char *env_var
, int def
)
6093 val
= getenv(env_var
);
6096 if (sscanf(val
, "%20d", &nst
) <= 0)
6102 fprintf(fplog
, "Found env.var. %s = %s, using value %d\n",
6110 static void dd_warning(t_commrec
*cr
, FILE *fplog
, const char *warn_string
)
6114 fprintf(stderr
, "\n%s\n", warn_string
);
6118 fprintf(fplog
, "\n%s\n", warn_string
);
6122 static void check_dd_restrictions(t_commrec
*cr
, const gmx_domdec_t
*dd
,
6123 const t_inputrec
*ir
, FILE *fplog
)
6125 if (ir
->ePBC
== epbcSCREW
&&
6126 (dd
->nc
[XX
] == 1 || dd
->nc
[YY
] > 1 || dd
->nc
[ZZ
] > 1))
6128 gmx_fatal(FARGS
, "With pbc=%s can only do domain decomposition in the x-direction", epbc_names
[ir
->ePBC
]);
6131 if (ir
->ns_type
== ensSIMPLE
)
6133 gmx_fatal(FARGS
, "Domain decomposition does not support simple neighbor searching, use grid searching or run with one MPI rank");
6136 if (ir
->nstlist
== 0)
6138 gmx_fatal(FARGS
, "Domain decomposition does not work with nstlist=0");
6141 if (ir
->comm_mode
== ecmANGULAR
&& ir
->ePBC
!= epbcNONE
)
6143 dd_warning(cr
, fplog
, "comm-mode angular will give incorrect results when the comm group partially crosses a periodic boundary");
6147 static real
average_cellsize_min(gmx_domdec_t
*dd
, gmx_ddbox_t
*ddbox
)
6152 r
= ddbox
->box_size
[XX
];
6153 for (di
= 0; di
< dd
->ndim
; di
++)
6156 /* Check using the initial average cell size */
6157 r
= std::min(r
, ddbox
->box_size
[d
]*ddbox
->skew_fac
[d
]/dd
->nc
[d
]);
6163 /*! \brief Depending on the DLB initial value return the DLB switched off state or issue an error.
6165 static int forceDlbOffOrBail(int cmdlineDlbState
,
6166 const std::string
&reasonStr
,
6170 std::string dlbNotSupportedErr
= "Dynamic load balancing requested, but ";
6171 std::string dlbDisableNote
= "NOTE: disabling dynamic load balancing as ";
6173 if (cmdlineDlbState
== edlbsOnUser
)
6175 gmx_fatal(FARGS
, (dlbNotSupportedErr
+ reasonStr
).c_str());
6177 else if (cmdlineDlbState
== edlbsOffCanTurnOn
)
6179 dd_warning(cr
, fplog
, (dlbDisableNote
+ reasonStr
+ "\n").c_str());
6181 return edlbsOffForever
;
6184 /*! \brief Return the dynamic load balancer's initial state based on initial conditions and user inputs.
6186 * This function parses the parameters of "-dlb" command line option setting
6187 * corresponding state values. Then it checks the consistency of the determined
6188 * state with other run parameters and settings. As a result, the initial state
6189 * may be altered or an error may be thrown if incompatibility of options is detected.
6191 * \param [in] fplog Pointer to mdrun log file.
6192 * \param [in] cr Pointer to MPI communication object.
6193 * \param [in] dlb_opt Pointer to the '-dlb' command line argument's option.
6194 * \param [in] bRecordLoad True if the load balancer is recording load information.
6195 * \param [in] Flags Simulation flags passed from main.
6196 * \param [in] ir Pointer mdrun to input parameters.
6197 * \returns DLB initial/startup state.
6199 static int determineInitialDlbState(FILE *fplog
, t_commrec
*cr
,
6200 const char *dlb_opt
, gmx_bool bRecordLoad
,
6201 unsigned long Flags
, const t_inputrec
*ir
)
6207 case 'a': dlbState
= edlbsOffCanTurnOn
; break;
6208 case 'n': dlbState
= edlbsOffUser
; break;
6209 case 'y': dlbState
= edlbsOnUser
; break;
6210 default: gmx_incons("Unknown dlb_opt");
6213 /* Reruns don't support DLB: bail or override auto mode */
6214 if (Flags
& MD_RERUN
)
6216 std::string reasonStr
= "it is not supported in reruns.";
6217 return forceDlbOffOrBail(dlbState
, reasonStr
, cr
, fplog
);
6220 /* Unsupported integrators */
6221 if (!EI_DYNAMICS(ir
->eI
))
6223 auto reasonStr
= gmx::formatString("it is only supported with dynamics, not with integrator '%s'.", EI(ir
->eI
));
6224 return forceDlbOffOrBail(dlbState
, reasonStr
, cr
, fplog
);
6227 /* Without cycle counters we can't time work to balance on */
6230 std::string reasonStr
= "cycle counters unsupported or not enabled in the operating system kernel.";
6231 return forceDlbOffOrBail(dlbState
, reasonStr
, cr
, fplog
);
6234 if (Flags
& MD_REPRODUCIBLE
)
6236 std::string reasonStr
= "you started a reproducible run.";
6241 case edlbsOffForever
:
6242 GMX_RELEASE_ASSERT(false, "edlbsOffForever is not a valid initial state");
6244 case edlbsOffCanTurnOn
:
6245 return forceDlbOffOrBail(dlbState
, reasonStr
, cr
, fplog
);
6247 case edlbsOnCanTurnOff
:
6248 GMX_RELEASE_ASSERT(false, "edlbsOffCanTurnOff is not a valid initial state");
6251 return forceDlbOffOrBail(dlbState
, reasonStr
+ " In load balanced runs binary reproducibility cannot be ensured.", cr
, fplog
);
6254 gmx_fatal(FARGS
, "Death horror: undefined case (%d) for load balancing choice", dlbState
);
6262 static void set_dd_dim(FILE *fplog
, gmx_domdec_t
*dd
)
6267 if (getenv("GMX_DD_ORDER_ZYX") != nullptr)
6269 /* Decomposition order z,y,x */
6272 fprintf(fplog
, "Using domain decomposition order z, y, x\n");
6274 for (dim
= DIM
-1; dim
>= 0; dim
--)
6276 if (dd
->nc
[dim
] > 1)
6278 dd
->dim
[dd
->ndim
++] = dim
;
6284 /* Decomposition order x,y,z */
6285 for (dim
= 0; dim
< DIM
; dim
++)
6287 if (dd
->nc
[dim
] > 1)
6289 dd
->dim
[dd
->ndim
++] = dim
;
6295 static gmx_domdec_comm_t
*init_dd_comm()
6297 gmx_domdec_comm_t
*comm
;
6301 snew(comm
->cggl_flag
, DIM
*2);
6302 snew(comm
->cgcm_state
, DIM
*2);
6303 for (i
= 0; i
< DIM
*2; i
++)
6305 comm
->cggl_flag_nalloc
[i
] = 0;
6306 comm
->cgcm_state_nalloc
[i
] = 0;
6309 comm
->nalloc_int
= 0;
6310 comm
->buf_int
= nullptr;
6312 vec_rvec_init(&comm
->vbuf
);
6314 comm
->n_load_have
= 0;
6315 comm
->n_load_collect
= 0;
6317 for (i
= 0; i
< ddnatNR
-ddnatZONE
; i
++)
6319 comm
->sum_nat
[i
] = 0;
6323 comm
->load_step
= 0;
6326 clear_ivec(comm
->load_lim
);
6330 /* This should be replaced by a unique pointer */
6331 comm
->balanceRegion
= ddBalanceRegionAllocate();
6336 /*! \brief Set the cell size and interaction limits, as well as the DD grid */
6337 static void set_dd_limits_and_grid(FILE *fplog
, t_commrec
*cr
, gmx_domdec_t
*dd
,
6338 unsigned long Flags
,
6339 ivec nc
, int nPmeRanks
,
6340 real comm_distance_min
, real rconstr
,
6341 const char *dlb_opt
, real dlb_scale
,
6342 const char *sizex
, const char *sizey
, const char *sizez
,
6343 const gmx_mtop_t
*mtop
,
6344 const t_inputrec
*ir
,
6345 matrix box
, const rvec
*x
,
6347 int *npme_x
, int *npme_y
)
6350 real r_bonded_limit
= -1;
6351 const real tenPercentMargin
= 1.1;
6352 gmx_domdec_comm_t
*comm
= dd
->comm
;
6354 snew(comm
->cggl_flag
, DIM
*2);
6355 snew(comm
->cgcm_state
, DIM
*2);
6357 dd
->npbcdim
= ePBC2npbcdim(ir
->ePBC
);
6358 dd
->bScrewPBC
= (ir
->ePBC
== epbcSCREW
);
6360 dd
->pme_recv_f_alloc
= 0;
6361 dd
->pme_recv_f_buf
= nullptr;
6363 /* Initialize to GPU share count to 0, might change later */
6364 comm
->nrank_gpu_shared
= 0;
6366 comm
->dlbState
= determineInitialDlbState(fplog
, cr
, dlb_opt
, comm
->bRecordLoad
, Flags
, ir
);
6367 dd_dlb_set_should_check_whether_to_turn_dlb_on(dd
, TRUE
);
6368 /* To consider turning DLB on after 2*nstlist steps we need to check
6369 * at partitioning count 3. Thus we need to increase the first count by 2.
6371 comm
->ddPartioningCountFirstDlbOff
+= 2;
6375 fprintf(fplog
, "Dynamic load balancing: %s\n",
6376 edlbs_names
[comm
->dlbState
]);
6378 comm
->bPMELoadBalDLBLimits
= FALSE
;
6380 /* Allocate the charge group/atom sorting struct */
6381 snew(comm
->sort
, 1);
6383 comm
->bCGs
= (ncg_mtop(mtop
) < mtop
->natoms
);
6385 comm
->bInterCGBondeds
= ((ncg_mtop(mtop
) > mtop
->mols
.nr
) ||
6386 mtop
->bIntermolecularInteractions
);
6387 if (comm
->bInterCGBondeds
)
6389 comm
->bInterCGMultiBody
= (multi_body_bondeds_count(mtop
) > 0);
6393 comm
->bInterCGMultiBody
= FALSE
;
6396 dd
->bInterCGcons
= inter_charge_group_constraints(mtop
);
6397 dd
->bInterCGsettles
= inter_charge_group_settles(mtop
);
6401 /* Set the cut-off to some very large value,
6402 * so we don't need if statements everywhere in the code.
6403 * We use sqrt, since the cut-off is squared in some places.
6405 comm
->cutoff
= GMX_CUTOFF_INF
;
6409 comm
->cutoff
= ir
->rlist
;
6411 comm
->cutoff_mbody
= 0;
6413 comm
->cellsize_limit
= 0;
6414 comm
->bBondComm
= FALSE
;
6416 /* Atoms should be able to move by up to half the list buffer size (if > 0)
6417 * within nstlist steps. Since boundaries are allowed to displace by half
6418 * a cell size, DD cells should be at least the size of the list buffer.
6420 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
,
6421 ir
->rlist
- std::max(ir
->rvdw
, ir
->rcoulomb
));
6423 if (comm
->bInterCGBondeds
)
6425 if (comm_distance_min
> 0)
6427 comm
->cutoff_mbody
= comm_distance_min
;
6428 if (Flags
& MD_DDBONDCOMM
)
6430 comm
->bBondComm
= (comm
->cutoff_mbody
> comm
->cutoff
);
6434 comm
->cutoff
= std::max(comm
->cutoff
, comm
->cutoff_mbody
);
6436 r_bonded_limit
= comm
->cutoff_mbody
;
6438 else if (ir
->bPeriodicMols
)
6440 /* Can not easily determine the required cut-off */
6441 dd_warning(cr
, fplog
, "NOTE: Periodic molecules are present in this system. Because of this, the domain decomposition algorithm cannot easily determine the minimum cell size that it requires for treating bonded interactions. Instead, domain decomposition will assume that half the non-bonded cut-off will be a suitable lower bound.\n");
6442 comm
->cutoff_mbody
= comm
->cutoff
/2;
6443 r_bonded_limit
= comm
->cutoff_mbody
;
6451 dd_bonded_cg_distance(fplog
, mtop
, ir
, x
, box
,
6452 Flags
& MD_DDBONDCHECK
, &r_2b
, &r_mb
);
6454 gmx_bcast(sizeof(r_2b
), &r_2b
, cr
);
6455 gmx_bcast(sizeof(r_mb
), &r_mb
, cr
);
6457 /* We use an initial margin of 10% for the minimum cell size,
6458 * except when we are just below the non-bonded cut-off.
6460 if (Flags
& MD_DDBONDCOMM
)
6462 if (std::max(r_2b
, r_mb
) > comm
->cutoff
)
6464 r_bonded
= std::max(r_2b
, r_mb
);
6465 r_bonded_limit
= tenPercentMargin
*r_bonded
;
6466 comm
->bBondComm
= TRUE
;
6471 r_bonded_limit
= std::min(tenPercentMargin
*r_bonded
, comm
->cutoff
);
6473 /* We determine cutoff_mbody later */
6477 /* No special bonded communication,
6478 * simply increase the DD cut-off.
6480 r_bonded_limit
= tenPercentMargin
*std::max(r_2b
, r_mb
);
6481 comm
->cutoff_mbody
= r_bonded_limit
;
6482 comm
->cutoff
= std::max(comm
->cutoff
, comm
->cutoff_mbody
);
6488 "Minimum cell size due to bonded interactions: %.3f nm\n",
6491 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
, r_bonded_limit
);
6494 if (dd
->bInterCGcons
&& rconstr
<= 0)
6496 /* There is a cell size limit due to the constraints (P-LINCS) */
6497 rconstr
= constr_r_max(fplog
, mtop
, ir
);
6501 "Estimated maximum distance required for P-LINCS: %.3f nm\n",
6503 if (rconstr
> comm
->cellsize_limit
)
6505 fprintf(fplog
, "This distance will limit the DD cell size, you can override this with -rcon\n");
6509 else if (rconstr
> 0 && fplog
)
6511 /* Here we do not check for dd->bInterCGcons,
6512 * because one can also set a cell size limit for virtual sites only
6513 * and at this point we don't know yet if there are intercg v-sites.
6516 "User supplied maximum distance required for P-LINCS: %.3f nm\n",
6519 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
, rconstr
);
6521 comm
->cgs_gl
= gmx_mtop_global_cgs(mtop
);
6525 copy_ivec(nc
, dd
->nc
);
6526 set_dd_dim(fplog
, dd
);
6527 set_ddbox_cr(cr
, &dd
->nc
, ir
, box
, &comm
->cgs_gl
, x
, ddbox
);
6531 cr
->npmenodes
= nPmeRanks
;
6535 /* When the DD grid is set explicitly and -npme is set to auto,
6536 * don't use PME ranks. We check later if the DD grid is
6537 * compatible with the total number of ranks.
6542 real acs
= average_cellsize_min(dd
, ddbox
);
6543 if (acs
< comm
->cellsize_limit
)
6547 fprintf(fplog
, "ERROR: The initial cell size (%f) is smaller than the cell size limit (%f)\n", acs
, comm
->cellsize_limit
);
6549 gmx_fatal_collective(FARGS
, cr
->mpi_comm_mysim
, MASTER(cr
),
6550 "The initial cell size (%f) is smaller than the cell size limit (%f), change options -dd, -rdd or -rcon, see the log file for details",
6551 acs
, comm
->cellsize_limit
);
6556 set_ddbox_cr(cr
, nullptr, ir
, box
, &comm
->cgs_gl
, x
, ddbox
);
6558 /* We need to choose the optimal DD grid and possibly PME nodes */
6560 dd_choose_grid(fplog
, cr
, dd
, ir
, mtop
, box
, ddbox
,
6562 !isDlbDisabled(comm
),
6564 comm
->cellsize_limit
, comm
->cutoff
,
6565 comm
->bInterCGBondeds
);
6567 if (dd
->nc
[XX
] == 0)
6570 gmx_bool bC
= (dd
->bInterCGcons
&& rconstr
> r_bonded_limit
);
6571 sprintf(buf
, "Change the number of ranks or mdrun option %s%s%s",
6572 !bC
? "-rdd" : "-rcon",
6573 comm
->dlbState
!= edlbsOffUser
? " or -dds" : "",
6574 bC
? " or your LINCS settings" : "");
6576 gmx_fatal_collective(FARGS
, cr
->mpi_comm_mysim
, MASTER(cr
),
6577 "There is no domain decomposition for %d ranks that is compatible with the given box and a minimum cell size of %g nm\n"
6579 "Look in the log file for details on the domain decomposition",
6580 cr
->nnodes
-cr
->npmenodes
, limit
, buf
);
6582 set_dd_dim(fplog
, dd
);
6588 "Domain decomposition grid %d x %d x %d, separate PME ranks %d\n",
6589 dd
->nc
[XX
], dd
->nc
[YY
], dd
->nc
[ZZ
], cr
->npmenodes
);
6592 dd
->nnodes
= dd
->nc
[XX
]*dd
->nc
[YY
]*dd
->nc
[ZZ
];
6593 if (cr
->nnodes
- dd
->nnodes
!= cr
->npmenodes
)
6595 gmx_fatal_collective(FARGS
, cr
->mpi_comm_mysim
, MASTER(cr
),
6596 "The size of the domain decomposition grid (%d) does not match the number of ranks (%d). The total number of ranks is %d",
6597 dd
->nnodes
, cr
->nnodes
- cr
->npmenodes
, cr
->nnodes
);
6599 if (cr
->npmenodes
> dd
->nnodes
)
6601 gmx_fatal_collective(FARGS
, cr
->mpi_comm_mysim
, MASTER(cr
),
6602 "The number of separate PME ranks (%d) is larger than the number of PP ranks (%d), this is not supported.", cr
->npmenodes
, dd
->nnodes
);
6604 if (cr
->npmenodes
> 0)
6606 comm
->npmenodes
= cr
->npmenodes
;
6610 comm
->npmenodes
= dd
->nnodes
;
6613 if (EEL_PME(ir
->coulombtype
) || EVDW_PME(ir
->vdwtype
))
6615 /* The following choices should match those
6616 * in comm_cost_est in domdec_setup.c.
6617 * Note that here the checks have to take into account
6618 * that the decomposition might occur in a different order than xyz
6619 * (for instance through the env.var. GMX_DD_ORDER_ZYX),
6620 * in which case they will not match those in comm_cost_est,
6621 * but since that is mainly for testing purposes that's fine.
6623 if (dd
->ndim
>= 2 && dd
->dim
[0] == XX
&& dd
->dim
[1] == YY
&&
6624 comm
->npmenodes
> dd
->nc
[XX
] && comm
->npmenodes
% dd
->nc
[XX
] == 0 &&
6625 getenv("GMX_PMEONEDD") == nullptr)
6627 comm
->npmedecompdim
= 2;
6628 comm
->npmenodes_x
= dd
->nc
[XX
];
6629 comm
->npmenodes_y
= comm
->npmenodes
/comm
->npmenodes_x
;
6633 /* In case nc is 1 in both x and y we could still choose to
6634 * decompose pme in y instead of x, but we use x for simplicity.
6636 comm
->npmedecompdim
= 1;
6637 if (dd
->dim
[0] == YY
)
6639 comm
->npmenodes_x
= 1;
6640 comm
->npmenodes_y
= comm
->npmenodes
;
6644 comm
->npmenodes_x
= comm
->npmenodes
;
6645 comm
->npmenodes_y
= 1;
6650 fprintf(fplog
, "PME domain decomposition: %d x %d x %d\n",
6651 comm
->npmenodes_x
, comm
->npmenodes_y
, 1);
6656 comm
->npmedecompdim
= 0;
6657 comm
->npmenodes_x
= 0;
6658 comm
->npmenodes_y
= 0;
6661 /* Technically we don't need both of these,
6662 * but it simplifies code not having to recalculate it.
6664 *npme_x
= comm
->npmenodes_x
;
6665 *npme_y
= comm
->npmenodes_y
;
6667 snew(comm
->slb_frac
, DIM
);
6668 if (isDlbDisabled(comm
))
6670 comm
->slb_frac
[XX
] = get_slb_frac(fplog
, "x", dd
->nc
[XX
], sizex
);
6671 comm
->slb_frac
[YY
] = get_slb_frac(fplog
, "y", dd
->nc
[YY
], sizey
);
6672 comm
->slb_frac
[ZZ
] = get_slb_frac(fplog
, "z", dd
->nc
[ZZ
], sizez
);
6675 if (comm
->bInterCGBondeds
&& comm
->cutoff_mbody
== 0)
6677 if (comm
->bBondComm
|| !isDlbDisabled(comm
))
6679 /* Set the bonded communication distance to halfway
6680 * the minimum and the maximum,
6681 * since the extra communication cost is nearly zero.
6683 real acs
= average_cellsize_min(dd
, ddbox
);
6684 comm
->cutoff_mbody
= 0.5*(r_bonded
+ acs
);
6685 if (!isDlbDisabled(comm
))
6687 /* Check if this does not limit the scaling */
6688 comm
->cutoff_mbody
= std::min(comm
->cutoff_mbody
, dlb_scale
*acs
);
6690 if (!comm
->bBondComm
)
6692 /* Without bBondComm do not go beyond the n.b. cut-off */
6693 comm
->cutoff_mbody
= std::min(comm
->cutoff_mbody
, comm
->cutoff
);
6694 if (comm
->cellsize_limit
>= comm
->cutoff
)
6696 /* We don't loose a lot of efficieny
6697 * when increasing it to the n.b. cut-off.
6698 * It can even be slightly faster, because we need
6699 * less checks for the communication setup.
6701 comm
->cutoff_mbody
= comm
->cutoff
;
6704 /* Check if we did not end up below our original limit */
6705 comm
->cutoff_mbody
= std::max(comm
->cutoff_mbody
, r_bonded_limit
);
6707 if (comm
->cutoff_mbody
> comm
->cellsize_limit
)
6709 comm
->cellsize_limit
= comm
->cutoff_mbody
;
6712 /* Without DLB and cutoff_mbody<cutoff, cutoff_mbody is dynamic */
6717 fprintf(debug
, "Bonded atom communication beyond the cut-off: %d\n"
6718 "cellsize limit %f\n",
6719 comm
->bBondComm
, comm
->cellsize_limit
);
6724 check_dd_restrictions(cr
, dd
, ir
, fplog
);
6728 static void set_dlb_limits(gmx_domdec_t
*dd
)
6733 for (d
= 0; d
< dd
->ndim
; d
++)
6735 dd
->comm
->cd
[d
].np
= dd
->comm
->cd
[d
].np_dlb
;
6736 dd
->comm
->cellsize_min
[dd
->dim
[d
]] =
6737 dd
->comm
->cellsize_min_dlb
[dd
->dim
[d
]];
6742 static void turn_on_dlb(FILE *fplog
, t_commrec
*cr
, gmx_int64_t step
)
6745 gmx_domdec_comm_t
*comm
;
6752 cellsize_min
= comm
->cellsize_min
[dd
->dim
[0]];
6753 for (d
= 1; d
< dd
->ndim
; d
++)
6755 cellsize_min
= std::min(cellsize_min
, comm
->cellsize_min
[dd
->dim
[d
]]);
6758 /* Turn off DLB if we're too close to the cell size limit. */
6759 if (cellsize_min
< comm
->cellsize_limit
*1.05)
6761 auto str
= gmx::formatString("step %" GMX_PRId64
" Measured %.1f %% performance loss due to load imbalance, "
6762 "but the minimum cell size is smaller than 1.05 times the cell size limit."
6763 "Will no longer try dynamic load balancing.\n", step
, dd_force_imb_perf_loss(dd
)*100);
6764 dd_warning(cr
, fplog
, str
.c_str());
6766 comm
->dlbState
= edlbsOffForever
;
6771 sprintf(buf
, "step %" GMX_PRId64
" Turning on dynamic load balancing, because the performance loss due to load imbalance is %.1f %%.\n", step
, dd_force_imb_perf_loss(dd
)*100);
6772 dd_warning(cr
, fplog
, buf
);
6773 comm
->dlbState
= edlbsOnCanTurnOff
;
6775 /* Store the non-DLB performance, so we can check if DLB actually
6776 * improves performance.
6778 GMX_RELEASE_ASSERT(comm
->cycl_n
[ddCyclStep
] > 0, "When we turned on DLB, we should have measured cycles");
6779 comm
->cyclesPerStepBeforeDLB
= comm
->cycl
[ddCyclStep
]/comm
->cycl_n
[ddCyclStep
];
6783 /* We can set the required cell size info here,
6784 * so we do not need to communicate this.
6785 * The grid is completely uniform.
6787 for (d
= 0; d
< dd
->ndim
; d
++)
6791 comm
->load
[d
].sum_m
= comm
->load
[d
].sum
;
6793 nc
= dd
->nc
[dd
->dim
[d
]];
6794 for (i
= 0; i
< nc
; i
++)
6796 comm
->root
[d
]->cell_f
[i
] = i
/(real
)nc
;
6799 comm
->root
[d
]->cell_f_max0
[i
] = i
/(real
)nc
;
6800 comm
->root
[d
]->cell_f_min1
[i
] = (i
+1)/(real
)nc
;
6803 comm
->root
[d
]->cell_f
[nc
] = 1.0;
6808 static void turn_off_dlb(FILE *fplog
, t_commrec
*cr
, gmx_int64_t step
)
6810 gmx_domdec_t
*dd
= cr
->dd
;
6813 sprintf(buf
, "step %" GMX_PRId64
" Turning off dynamic load balancing, because it is degrading performance.\n", step
);
6814 dd_warning(cr
, fplog
, buf
);
6815 dd
->comm
->dlbState
= edlbsOffCanTurnOn
;
6816 dd
->comm
->haveTurnedOffDlb
= true;
6817 dd
->comm
->ddPartioningCountFirstDlbOff
= dd
->ddp_count
;
6820 static void turn_off_dlb_forever(FILE *fplog
, t_commrec
*cr
, gmx_int64_t step
)
6822 GMX_RELEASE_ASSERT(cr
->dd
->comm
->dlbState
== edlbsOffCanTurnOn
, "Can only turn off DLB forever when it was in the can-turn-on state");
6824 sprintf(buf
, "step %" GMX_PRId64
" Will no longer try dynamic load balancing, as it degraded performance.\n", step
);
6825 dd_warning(cr
, fplog
, buf
);
6826 cr
->dd
->comm
->dlbState
= edlbsOffForever
;
6829 static char *init_bLocalCG(const gmx_mtop_t
*mtop
)
6834 ncg
= ncg_mtop(mtop
);
6835 snew(bLocalCG
, ncg
);
6836 for (cg
= 0; cg
< ncg
; cg
++)
6838 bLocalCG
[cg
] = FALSE
;
6844 void dd_init_bondeds(FILE *fplog
,
6846 const gmx_mtop_t
*mtop
,
6847 const gmx_vsite_t
*vsite
,
6848 const t_inputrec
*ir
,
6849 gmx_bool bBCheck
, cginfo_mb_t
*cginfo_mb
)
6851 gmx_domdec_comm_t
*comm
;
6853 dd_make_reverse_top(fplog
, dd
, mtop
, vsite
, ir
, bBCheck
);
6857 if (comm
->bBondComm
)
6859 /* Communicate atoms beyond the cut-off for bonded interactions */
6862 comm
->cglink
= make_charge_group_links(mtop
, dd
, cginfo_mb
);
6864 comm
->bLocalCG
= init_bLocalCG(mtop
);
6868 /* Only communicate atoms based on cut-off */
6869 comm
->cglink
= nullptr;
6870 comm
->bLocalCG
= nullptr;
6874 static void print_dd_settings(FILE *fplog
, gmx_domdec_t
*dd
,
6875 const gmx_mtop_t
*mtop
, const t_inputrec
*ir
,
6876 gmx_bool bDynLoadBal
, real dlb_scale
,
6877 const gmx_ddbox_t
*ddbox
)
6879 gmx_domdec_comm_t
*comm
;
6885 if (fplog
== nullptr)
6894 fprintf(fplog
, "The maximum number of communication pulses is:");
6895 for (d
= 0; d
< dd
->ndim
; d
++)
6897 fprintf(fplog
, " %c %d", dim2char(dd
->dim
[d
]), comm
->cd
[d
].np_dlb
);
6899 fprintf(fplog
, "\n");
6900 fprintf(fplog
, "The minimum size for domain decomposition cells is %.3f nm\n", comm
->cellsize_limit
);
6901 fprintf(fplog
, "The requested allowed shrink of DD cells (option -dds) is: %.2f\n", dlb_scale
);
6902 fprintf(fplog
, "The allowed shrink of domain decomposition cells is:");
6903 for (d
= 0; d
< DIM
; d
++)
6907 if (d
>= ddbox
->npbcdim
&& dd
->nc
[d
] == 2)
6914 comm
->cellsize_min_dlb
[d
]/
6915 (ddbox
->box_size
[d
]*ddbox
->skew_fac
[d
]/dd
->nc
[d
]);
6917 fprintf(fplog
, " %c %.2f", dim2char(d
), shrink
);
6920 fprintf(fplog
, "\n");
6924 set_dd_cell_sizes_slb(dd
, ddbox
, setcellsizeslbPULSE_ONLY
, np
);
6925 fprintf(fplog
, "The initial number of communication pulses is:");
6926 for (d
= 0; d
< dd
->ndim
; d
++)
6928 fprintf(fplog
, " %c %d", dim2char(dd
->dim
[d
]), np
[dd
->dim
[d
]]);
6930 fprintf(fplog
, "\n");
6931 fprintf(fplog
, "The initial domain decomposition cell size is:");
6932 for (d
= 0; d
< DIM
; d
++)
6936 fprintf(fplog
, " %c %.2f nm",
6937 dim2char(d
), dd
->comm
->cellsize_min
[d
]);
6940 fprintf(fplog
, "\n\n");
6943 gmx_bool bInterCGVsites
= count_intercg_vsites(mtop
);
6945 if (comm
->bInterCGBondeds
||
6947 dd
->bInterCGcons
|| dd
->bInterCGsettles
)
6949 fprintf(fplog
, "The maximum allowed distance for charge groups involved in interactions is:\n");
6950 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6951 "non-bonded interactions", "", comm
->cutoff
);
6955 limit
= dd
->comm
->cellsize_limit
;
6959 if (dynamic_dd_box(ddbox
, ir
))
6961 fprintf(fplog
, "(the following are initial values, they could change due to box deformation)\n");
6963 limit
= dd
->comm
->cellsize_min
[XX
];
6964 for (d
= 1; d
< DIM
; d
++)
6966 limit
= std::min(limit
, dd
->comm
->cellsize_min
[d
]);
6970 if (comm
->bInterCGBondeds
)
6972 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6973 "two-body bonded interactions", "(-rdd)",
6974 std::max(comm
->cutoff
, comm
->cutoff_mbody
));
6975 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6976 "multi-body bonded interactions", "(-rdd)",
6977 (comm
->bBondComm
|| isDlbOn(dd
->comm
)) ? comm
->cutoff_mbody
: std::min(comm
->cutoff
, limit
));
6981 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6982 "virtual site constructions", "(-rcon)", limit
);
6984 if (dd
->bInterCGcons
|| dd
->bInterCGsettles
)
6986 sprintf(buf
, "atoms separated by up to %d constraints",
6988 fprintf(fplog
, "%40s %-7s %6.3f nm\n",
6989 buf
, "(-rcon)", limit
);
6991 fprintf(fplog
, "\n");
6997 static void set_cell_limits_dlb(gmx_domdec_t
*dd
,
6999 const t_inputrec
*ir
,
7000 const gmx_ddbox_t
*ddbox
)
7002 gmx_domdec_comm_t
*comm
;
7003 int d
, dim
, npulse
, npulse_d_max
, npulse_d
;
7008 bNoCutOff
= (ir
->rvdw
== 0 || ir
->rcoulomb
== 0);
7010 /* Determine the maximum number of comm. pulses in one dimension */
7012 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
, comm
->cutoff_mbody
);
7014 /* Determine the maximum required number of grid pulses */
7015 if (comm
->cellsize_limit
>= comm
->cutoff
)
7017 /* Only a single pulse is required */
7020 else if (!bNoCutOff
&& comm
->cellsize_limit
> 0)
7022 /* We round down slightly here to avoid overhead due to the latency
7023 * of extra communication calls when the cut-off
7024 * would be only slightly longer than the cell size.
7025 * Later cellsize_limit is redetermined,
7026 * so we can not miss interactions due to this rounding.
7028 npulse
= (int)(0.96 + comm
->cutoff
/comm
->cellsize_limit
);
7032 /* There is no cell size limit */
7033 npulse
= std::max(dd
->nc
[XX
]-1, std::max(dd
->nc
[YY
]-1, dd
->nc
[ZZ
]-1));
7036 if (!bNoCutOff
&& npulse
> 1)
7038 /* See if we can do with less pulses, based on dlb_scale */
7040 for (d
= 0; d
< dd
->ndim
; d
++)
7043 npulse_d
= (int)(1 + dd
->nc
[dim
]*comm
->cutoff
7044 /(ddbox
->box_size
[dim
]*ddbox
->skew_fac
[dim
]*dlb_scale
));
7045 npulse_d_max
= std::max(npulse_d_max
, npulse_d
);
7047 npulse
= std::min(npulse
, npulse_d_max
);
7050 /* This env var can override npulse */
7051 d
= dd_getenv(debug
, "GMX_DD_NPULSE", 0);
7058 comm
->bVacDLBNoLimit
= (ir
->ePBC
== epbcNONE
);
7059 for (d
= 0; d
< dd
->ndim
; d
++)
7061 comm
->cd
[d
].np_dlb
= std::min(npulse
, dd
->nc
[dd
->dim
[d
]]-1);
7062 comm
->cd
[d
].np_nalloc
= comm
->cd
[d
].np_dlb
;
7063 snew(comm
->cd
[d
].ind
, comm
->cd
[d
].np_nalloc
);
7064 comm
->maxpulse
= std::max(comm
->maxpulse
, comm
->cd
[d
].np_dlb
);
7065 if (comm
->cd
[d
].np_dlb
< dd
->nc
[dd
->dim
[d
]]-1)
7067 comm
->bVacDLBNoLimit
= FALSE
;
7071 /* cellsize_limit is set for LINCS in init_domain_decomposition */
7072 if (!comm
->bVacDLBNoLimit
)
7074 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
,
7075 comm
->cutoff
/comm
->maxpulse
);
7077 comm
->cellsize_limit
= std::max(comm
->cellsize_limit
, comm
->cutoff_mbody
);
7078 /* Set the minimum cell size for each DD dimension */
7079 for (d
= 0; d
< dd
->ndim
; d
++)
7081 if (comm
->bVacDLBNoLimit
||
7082 comm
->cd
[d
].np_dlb
*comm
->cellsize_limit
>= comm
->cutoff
)
7084 comm
->cellsize_min_dlb
[dd
->dim
[d
]] = comm
->cellsize_limit
;
7088 comm
->cellsize_min_dlb
[dd
->dim
[d
]] =
7089 comm
->cutoff
/comm
->cd
[d
].np_dlb
;
7092 if (comm
->cutoff_mbody
<= 0)
7094 comm
->cutoff_mbody
= std::min(comm
->cutoff
, comm
->cellsize_limit
);
7102 gmx_bool
dd_bonded_molpbc(const gmx_domdec_t
*dd
, int ePBC
)
7104 /* If each molecule is a single charge group
7105 * or we use domain decomposition for each periodic dimension,
7106 * we do not need to take pbc into account for the bonded interactions.
7108 return (ePBC
!= epbcNONE
&& dd
->comm
->bInterCGBondeds
&&
7111 (dd
->nc
[ZZ
] > 1 || ePBC
== epbcXY
)));
7114 /*! \brief Sets grid size limits and PP-PME setup, prints settings to log */
7115 static void set_ddgrid_parameters(FILE *fplog
, gmx_domdec_t
*dd
, real dlb_scale
,
7116 const gmx_mtop_t
*mtop
, const t_inputrec
*ir
,
7117 const gmx_ddbox_t
*ddbox
)
7119 gmx_domdec_comm_t
*comm
;
7125 if (EEL_PME(ir
->coulombtype
) || EVDW_PME(ir
->vdwtype
))
7127 init_ddpme(dd
, &comm
->ddpme
[0], 0);
7128 if (comm
->npmedecompdim
>= 2)
7130 init_ddpme(dd
, &comm
->ddpme
[1], 1);
7135 comm
->npmenodes
= 0;
7136 if (dd
->pme_nodeid
>= 0)
7138 gmx_fatal_collective(FARGS
, dd
->mpi_comm_all
, DDMASTER(dd
),
7139 "Can not have separate PME ranks without PME electrostatics");
7145 fprintf(debug
, "The DD cut-off is %f\n", comm
->cutoff
);
7147 if (!isDlbDisabled(comm
))
7149 set_cell_limits_dlb(dd
, dlb_scale
, ir
, ddbox
);
7152 print_dd_settings(fplog
, dd
, mtop
, ir
, isDlbOn(comm
), dlb_scale
, ddbox
);
7153 if (comm
->dlbState
== edlbsOffCanTurnOn
)
7157 fprintf(fplog
, "When dynamic load balancing gets turned on, these settings will change to:\n");
7159 print_dd_settings(fplog
, dd
, mtop
, ir
, TRUE
, dlb_scale
, ddbox
);
7162 if (ir
->ePBC
== epbcNONE
)
7164 vol_frac
= 1 - 1/(double)dd
->nnodes
;
7169 (1 + comm_box_frac(dd
->nc
, comm
->cutoff
, ddbox
))/(double)dd
->nnodes
;
7173 fprintf(debug
, "Volume fraction for all DD zones: %f\n", vol_frac
);
7175 natoms_tot
= comm
->cgs_gl
.index
[comm
->cgs_gl
.nr
];
7177 dd
->ga2la
= ga2la_init(natoms_tot
, static_cast<int>(vol_frac
*natoms_tot
));
7180 /*! \brief Set some important DD parameters that can be modified by env.vars */
7181 static void set_dd_envvar_options(FILE *fplog
, gmx_domdec_t
*dd
, int rank_mysim
)
7183 gmx_domdec_comm_t
*comm
= dd
->comm
;
7185 dd
->bSendRecv2
= dd_getenv(fplog
, "GMX_DD_USE_SENDRECV2", 0);
7186 comm
->dlb_scale_lim
= dd_getenv(fplog
, "GMX_DLB_MAX_BOX_SCALING", 10);
7187 comm
->eFlop
= dd_getenv(fplog
, "GMX_DLB_BASED_ON_FLOPS", 0);
7188 int recload
= dd_getenv(fplog
, "GMX_DD_RECORD_LOAD", 1);
7189 comm
->nstDDDump
= dd_getenv(fplog
, "GMX_DD_NST_DUMP", 0);
7190 comm
->nstDDDumpGrid
= dd_getenv(fplog
, "GMX_DD_NST_DUMP_GRID", 0);
7191 comm
->DD_debug
= dd_getenv(fplog
, "GMX_DD_DEBUG", 0);
7193 if (dd
->bSendRecv2
&& fplog
)
7195 fprintf(fplog
, "Will use two sequential MPI_Sendrecv calls instead of two simultaneous non-blocking MPI_Irecv and MPI_Isend pairs for constraint and vsite communication\n");
7202 fprintf(fplog
, "Will load balance based on FLOP count\n");
7204 if (comm
->eFlop
> 1)
7206 srand(1 + rank_mysim
);
7208 comm
->bRecordLoad
= TRUE
;
7212 comm
->bRecordLoad
= (wallcycle_have_counter() && recload
> 0);
7216 gmx_domdec_t
*init_domain_decomposition(FILE *fplog
, t_commrec
*cr
,
7217 unsigned long Flags
,
7218 ivec nc
, int nPmeRanks
,
7220 real comm_distance_min
, real rconstr
,
7221 const char *dlb_opt
, real dlb_scale
,
7222 const char *sizex
, const char *sizey
, const char *sizez
,
7223 const gmx_mtop_t
*mtop
,
7224 const t_inputrec
*ir
,
7225 matrix box
, rvec
*x
,
7227 int *npme_x
, int *npme_y
)
7234 "\nInitializing Domain Decomposition on %d ranks\n", cr
->nnodes
);
7239 dd
->comm
= init_dd_comm();
7241 set_dd_envvar_options(fplog
, dd
, cr
->nodeid
);
7243 set_dd_limits_and_grid(fplog
, cr
, dd
, Flags
,
7245 comm_distance_min
, rconstr
,
7247 sizex
, sizey
, sizez
,
7253 make_dd_communicators(fplog
, cr
, dd
, dd_rank_order
);
7255 if (cr
->duty
& DUTY_PP
)
7257 set_ddgrid_parameters(fplog
, dd
, dlb_scale
, mtop
, ir
, ddbox
);
7259 setup_neighbor_relations(dd
);
7262 /* Set overallocation to avoid frequent reallocation of arrays */
7263 set_over_alloc_dd(TRUE
);
7265 /* Initialize DD paritioning counters */
7266 dd
->comm
->partition_step
= INT_MIN
;
7269 /* We currently don't know the number of threads yet, we set this later */
7272 clear_dd_cycle_counts(dd
);
7277 static gmx_bool
test_dd_cutoff(t_commrec
*cr
,
7278 t_state
*state
, const t_inputrec
*ir
,
7289 set_ddbox(dd
, FALSE
, cr
, ir
, state
->box
,
7290 TRUE
, &dd
->comm
->cgs_gl
, as_rvec_array(state
->x
.data()), &ddbox
);
7294 for (d
= 0; d
< dd
->ndim
; d
++)
7298 inv_cell_size
= DD_CELL_MARGIN
*dd
->nc
[dim
]/ddbox
.box_size
[dim
];
7299 if (dynamic_dd_box(&ddbox
, ir
))
7301 inv_cell_size
*= DD_PRES_SCALE_MARGIN
;
7304 np
= 1 + (int)(cutoff_req
*inv_cell_size
*ddbox
.skew_fac
[dim
]);
7306 if (!isDlbDisabled(dd
->comm
) && (dim
< ddbox
.npbcdim
) && (dd
->comm
->cd
[d
].np_dlb
> 0))
7308 if (np
> dd
->comm
->cd
[d
].np_dlb
)
7313 /* If a current local cell size is smaller than the requested
7314 * cut-off, we could still fix it, but this gets very complicated.
7315 * Without fixing here, we might actually need more checks.
7317 if ((dd
->comm
->cell_x1
[dim
] - dd
->comm
->cell_x0
[dim
])*ddbox
.skew_fac
[dim
]*dd
->comm
->cd
[d
].np_dlb
< cutoff_req
)
7324 if (!isDlbDisabled(dd
->comm
))
7326 /* If DLB is not active yet, we don't need to check the grid jumps.
7327 * Actually we shouldn't, because then the grid jump data is not set.
7329 if (isDlbOn(dd
->comm
) &&
7330 check_grid_jump(0, dd
, cutoff_req
, &ddbox
, FALSE
))
7335 gmx_sumi(1, &LocallyLimited
, cr
);
7337 if (LocallyLimited
> 0)
7346 gmx_bool
change_dd_cutoff(t_commrec
*cr
, t_state
*state
, const t_inputrec
*ir
,
7349 gmx_bool bCutoffAllowed
;
7351 bCutoffAllowed
= test_dd_cutoff(cr
, state
, ir
, cutoff_req
);
7355 cr
->dd
->comm
->cutoff
= cutoff_req
;
7358 return bCutoffAllowed
;
7361 void set_dd_dlb_max_cutoff(t_commrec
*cr
, real cutoff
)
7363 gmx_domdec_comm_t
*comm
;
7365 comm
= cr
->dd
->comm
;
7367 /* Turn on the DLB limiting (might have been on already) */
7368 comm
->bPMELoadBalDLBLimits
= TRUE
;
7370 /* Change the cut-off limit */
7371 comm
->PMELoadBal_max_cutoff
= cutoff
;
7375 fprintf(debug
, "PME load balancing set a limit to the DLB staggering such that a %f cut-off will continue to fit\n",
7376 comm
->PMELoadBal_max_cutoff
);
7380 /* Sets whether we should later check the load imbalance data, so that
7381 * we can trigger dynamic load balancing if enough imbalance has
7384 * Used after PME load balancing unlocks DLB, so that the check
7385 * whether DLB will be useful can happen immediately.
7387 static void dd_dlb_set_should_check_whether_to_turn_dlb_on(gmx_domdec_t
*dd
, gmx_bool bValue
)
7389 if (dd
->comm
->dlbState
== edlbsOffCanTurnOn
)
7391 dd
->comm
->bCheckWhetherToTurnDlbOn
= bValue
;
7395 /* Store the DD partitioning count, so we can ignore cycle counts
7396 * over the next nstlist steps, which are often slower.
7398 dd
->comm
->ddPartioningCountFirstDlbOff
= dd
->ddp_count
;
7403 /* Returns if we should check whether there has been enough load
7404 * imbalance to trigger dynamic load balancing.
7406 static gmx_bool
dd_dlb_get_should_check_whether_to_turn_dlb_on(gmx_domdec_t
*dd
)
7408 if (dd
->comm
->dlbState
!= edlbsOffCanTurnOn
)
7413 if (dd
->ddp_count
<= dd
->comm
->ddPartioningCountFirstDlbOff
)
7415 /* We ignore the first nstlist steps at the start of the run
7416 * or after PME load balancing or after turning DLB off, since
7417 * these often have extra allocation or cache miss overhead.
7422 /* We should check whether we should use DLB directly after
7424 if (dd
->comm
->bCheckWhetherToTurnDlbOn
)
7426 /* This flag was set when the PME load-balancing routines
7427 unlocked DLB, and should now be cleared. */
7428 dd_dlb_set_should_check_whether_to_turn_dlb_on(dd
, FALSE
);
7431 /* We check whether we should use DLB every c_checkTurnDlbOnInterval
7432 * partitionings (we do not do this every partioning, so that we
7433 * avoid excessive communication). */
7434 if (dd
->comm
->n_load_have
% c_checkTurnDlbOnInterval
== c_checkTurnDlbOnInterval
- 1)
7442 gmx_bool
dd_dlb_is_on(const gmx_domdec_t
*dd
)
7444 return isDlbOn(dd
->comm
);
7447 gmx_bool
dd_dlb_is_locked(const gmx_domdec_t
*dd
)
7449 return (dd
->comm
->dlbState
== edlbsOffTemporarilyLocked
);
7452 void dd_dlb_lock(gmx_domdec_t
*dd
)
7454 /* We can only lock the DLB when it is set to auto, otherwise don't do anything */
7455 if (dd
->comm
->dlbState
== edlbsOffCanTurnOn
)
7457 dd
->comm
->dlbState
= edlbsOffTemporarilyLocked
;
7461 void dd_dlb_unlock(gmx_domdec_t
*dd
)
7463 /* We can only lock the DLB when it is set to auto, otherwise don't do anything */
7464 if (dd
->comm
->dlbState
== edlbsOffTemporarilyLocked
)
7466 dd
->comm
->dlbState
= edlbsOffCanTurnOn
;
7467 dd_dlb_set_should_check_whether_to_turn_dlb_on(dd
, TRUE
);
7471 static void merge_cg_buffers(int ncell
,
7472 gmx_domdec_comm_dim_t
*cd
, int pulse
,
7474 int *index_gl
, int *recv_i
,
7475 rvec
*cg_cm
, rvec
*recv_vr
,
7477 cginfo_mb_t
*cginfo_mb
, int *cginfo
)
7479 gmx_domdec_ind_t
*ind
, *ind_p
;
7480 int p
, cell
, c
, cg
, cg0
, cg1
, cg_gl
, nat
;
7481 int shift
, shift_at
;
7483 ind
= &cd
->ind
[pulse
];
7485 /* First correct the already stored data */
7486 shift
= ind
->nrecv
[ncell
];
7487 for (cell
= ncell
-1; cell
>= 0; cell
--)
7489 shift
-= ind
->nrecv
[cell
];
7492 /* Move the cg's present from previous grid pulses */
7493 cg0
= ncg_cell
[ncell
+cell
];
7494 cg1
= ncg_cell
[ncell
+cell
+1];
7495 cgindex
[cg1
+shift
] = cgindex
[cg1
];
7496 for (cg
= cg1
-1; cg
>= cg0
; cg
--)
7498 index_gl
[cg
+shift
] = index_gl
[cg
];
7499 copy_rvec(cg_cm
[cg
], cg_cm
[cg
+shift
]);
7500 cgindex
[cg
+shift
] = cgindex
[cg
];
7501 cginfo
[cg
+shift
] = cginfo
[cg
];
7503 /* Correct the already stored send indices for the shift */
7504 for (p
= 1; p
<= pulse
; p
++)
7506 ind_p
= &cd
->ind
[p
];
7508 for (c
= 0; c
< cell
; c
++)
7510 cg0
+= ind_p
->nsend
[c
];
7512 cg1
= cg0
+ ind_p
->nsend
[cell
];
7513 for (cg
= cg0
; cg
< cg1
; cg
++)
7515 ind_p
->index
[cg
] += shift
;
7521 /* Merge in the communicated buffers */
7525 for (cell
= 0; cell
< ncell
; cell
++)
7527 cg1
= ncg_cell
[ncell
+cell
+1] + shift
;
7530 /* Correct the old cg indices */
7531 for (cg
= ncg_cell
[ncell
+cell
]; cg
< cg1
; cg
++)
7533 cgindex
[cg
+1] += shift_at
;
7536 for (cg
= 0; cg
< ind
->nrecv
[cell
]; cg
++)
7538 /* Copy this charge group from the buffer */
7539 index_gl
[cg1
] = recv_i
[cg0
];
7540 copy_rvec(recv_vr
[cg0
], cg_cm
[cg1
]);
7541 /* Add it to the cgindex */
7542 cg_gl
= index_gl
[cg1
];
7543 cginfo
[cg1
] = ddcginfo(cginfo_mb
, cg_gl
);
7544 nat
= GET_CGINFO_NATOMS(cginfo
[cg1
]);
7545 cgindex
[cg1
+1] = cgindex
[cg1
] + nat
;
7550 shift
+= ind
->nrecv
[cell
];
7551 ncg_cell
[ncell
+cell
+1] = cg1
;
7555 static void make_cell2at_index(gmx_domdec_comm_dim_t
*cd
,
7556 int nzone
, int cg0
, const int *cgindex
)
7560 /* Store the atom block boundaries for easy copying of communication buffers
7563 for (zone
= 0; zone
< nzone
; zone
++)
7565 for (p
= 0; p
< cd
->np
; p
++)
7567 cd
->ind
[p
].cell2at0
[zone
] = cgindex
[cg
];
7568 cg
+= cd
->ind
[p
].nrecv
[zone
];
7569 cd
->ind
[p
].cell2at1
[zone
] = cgindex
[cg
];
7574 static gmx_bool
missing_link(t_blocka
*link
, int cg_gl
, char *bLocalCG
)
7580 for (i
= link
->index
[cg_gl
]; i
< link
->index
[cg_gl
+1]; i
++)
7582 if (!bLocalCG
[link
->a
[i
]])
7591 /* Domain corners for communication, a maximum of 4 i-zones see a j domain */
7593 real c
[DIM
][4]; /* the corners for the non-bonded communication */
7594 real cr0
; /* corner for rounding */
7595 real cr1
[4]; /* corners for rounding */
7596 real bc
[DIM
]; /* corners for bounded communication */
7597 real bcr1
; /* corner for rounding for bonded communication */
7600 /* Determine the corners of the domain(s) we are communicating with */
7602 set_dd_corners(const gmx_domdec_t
*dd
,
7603 int dim0
, int dim1
, int dim2
,
7607 const gmx_domdec_comm_t
*comm
;
7608 const gmx_domdec_zones_t
*zones
;
7613 zones
= &comm
->zones
;
7615 /* Keep the compiler happy */
7619 /* The first dimension is equal for all cells */
7620 c
->c
[0][0] = comm
->cell_x0
[dim0
];
7623 c
->bc
[0] = c
->c
[0][0];
7628 /* This cell row is only seen from the first row */
7629 c
->c
[1][0] = comm
->cell_x0
[dim1
];
7630 /* All rows can see this row */
7631 c
->c
[1][1] = comm
->cell_x0
[dim1
];
7632 if (isDlbOn(dd
->comm
))
7634 c
->c
[1][1] = std::max(comm
->cell_x0
[dim1
], comm
->zone_d1
[1].mch0
);
7637 /* For the multi-body distance we need the maximum */
7638 c
->bc
[1] = std::max(comm
->cell_x0
[dim1
], comm
->zone_d1
[1].p1_0
);
7641 /* Set the upper-right corner for rounding */
7642 c
->cr0
= comm
->cell_x1
[dim0
];
7647 for (j
= 0; j
< 4; j
++)
7649 c
->c
[2][j
] = comm
->cell_x0
[dim2
];
7651 if (isDlbOn(dd
->comm
))
7653 /* Use the maximum of the i-cells that see a j-cell */
7654 for (i
= 0; i
< zones
->nizone
; i
++)
7656 for (j
= zones
->izone
[i
].j0
; j
< zones
->izone
[i
].j1
; j
++)
7661 std::max(c
->c
[2][j
-4],
7662 comm
->zone_d2
[zones
->shift
[i
][dim0
]][zones
->shift
[i
][dim1
]].mch0
);
7668 /* For the multi-body distance we need the maximum */
7669 c
->bc
[2] = comm
->cell_x0
[dim2
];
7670 for (i
= 0; i
< 2; i
++)
7672 for (j
= 0; j
< 2; j
++)
7674 c
->bc
[2] = std::max(c
->bc
[2], comm
->zone_d2
[i
][j
].p1_0
);
7680 /* Set the upper-right corner for rounding */
7681 /* Cell (0,0,0) and cell (1,0,0) can see cell 4 (0,1,1)
7682 * Only cell (0,0,0) can see cell 7 (1,1,1)
7684 c
->cr1
[0] = comm
->cell_x1
[dim1
];
7685 c
->cr1
[3] = comm
->cell_x1
[dim1
];
7686 if (isDlbOn(dd
->comm
))
7688 c
->cr1
[0] = std::max(comm
->cell_x1
[dim1
], comm
->zone_d1
[1].mch1
);
7691 /* For the multi-body distance we need the maximum */
7692 c
->bcr1
= std::max(comm
->cell_x1
[dim1
], comm
->zone_d1
[1].p1_1
);
7699 /* Determine which cg's we need to send in this pulse from this zone */
7701 get_zone_pulse_cgs(gmx_domdec_t
*dd
,
7702 int zonei
, int zone
,
7704 const int *index_gl
,
7706 int dim
, int dim_ind
,
7707 int dim0
, int dim1
, int dim2
,
7708 real r_comm2
, real r_bcomm2
,
7712 real skew_fac2_d
, real skew_fac_01
,
7713 rvec
*v_d
, rvec
*v_0
, rvec
*v_1
,
7714 const dd_corners_t
*c
,
7716 gmx_bool bDistBonded
,
7722 gmx_domdec_ind_t
*ind
,
7723 int **ibuf
, int *ibuf_nalloc
,
7729 gmx_domdec_comm_t
*comm
;
7731 gmx_bool bDistMB_pulse
;
7733 real r2
, rb2
, r
, tric_sh
;
7736 int nsend_z
, nsend
, nat
;
7740 bScrew
= (dd
->bScrewPBC
&& dim
== XX
);
7742 bDistMB_pulse
= (bDistMB
&& bDistBonded
);
7748 for (cg
= cg0
; cg
< cg1
; cg
++)
7752 if (tric_dist
[dim_ind
] == 0)
7754 /* Rectangular direction, easy */
7755 r
= cg_cm
[cg
][dim
] - c
->c
[dim_ind
][zone
];
7762 r
= cg_cm
[cg
][dim
] - c
->bc
[dim_ind
];
7768 /* Rounding gives at most a 16% reduction
7769 * in communicated atoms
7771 if (dim_ind
>= 1 && (zonei
== 1 || zonei
== 2))
7773 r
= cg_cm
[cg
][dim0
] - c
->cr0
;
7774 /* This is the first dimension, so always r >= 0 */
7781 if (dim_ind
== 2 && (zonei
== 2 || zonei
== 3))
7783 r
= cg_cm
[cg
][dim1
] - c
->cr1
[zone
];
7790 r
= cg_cm
[cg
][dim1
] - c
->bcr1
;
7800 /* Triclinic direction, more complicated */
7803 /* Rounding, conservative as the skew_fac multiplication
7804 * will slightly underestimate the distance.
7806 if (dim_ind
>= 1 && (zonei
== 1 || zonei
== 2))
7808 rn
[dim0
] = cg_cm
[cg
][dim0
] - c
->cr0
;
7809 for (i
= dim0
+1; i
< DIM
; i
++)
7811 rn
[dim0
] -= cg_cm
[cg
][i
]*v_0
[i
][dim0
];
7813 r2
= rn
[dim0
]*rn
[dim0
]*sf2_round
[dim0
];
7816 rb
[dim0
] = rn
[dim0
];
7819 /* Take care that the cell planes along dim0 might not
7820 * be orthogonal to those along dim1 and dim2.
7822 for (i
= 1; i
<= dim_ind
; i
++)
7825 if (normal
[dim0
][dimd
] > 0)
7827 rn
[dimd
] -= rn
[dim0
]*normal
[dim0
][dimd
];
7830 rb
[dimd
] -= rb
[dim0
]*normal
[dim0
][dimd
];
7835 if (dim_ind
== 2 && (zonei
== 2 || zonei
== 3))
7837 rn
[dim1
] += cg_cm
[cg
][dim1
] - c
->cr1
[zone
];
7839 for (i
= dim1
+1; i
< DIM
; i
++)
7841 tric_sh
-= cg_cm
[cg
][i
]*v_1
[i
][dim1
];
7843 rn
[dim1
] += tric_sh
;
7846 r2
+= rn
[dim1
]*rn
[dim1
]*sf2_round
[dim1
];
7847 /* Take care of coupling of the distances
7848 * to the planes along dim0 and dim1 through dim2.
7850 r2
-= rn
[dim0
]*rn
[dim1
]*skew_fac_01
;
7851 /* Take care that the cell planes along dim1
7852 * might not be orthogonal to that along dim2.
7854 if (normal
[dim1
][dim2
] > 0)
7856 rn
[dim2
] -= rn
[dim1
]*normal
[dim1
][dim2
];
7862 cg_cm
[cg
][dim1
] - c
->bcr1
+ tric_sh
;
7865 rb2
+= rb
[dim1
]*rb
[dim1
]*sf2_round
[dim1
];
7866 /* Take care of coupling of the distances
7867 * to the planes along dim0 and dim1 through dim2.
7869 rb2
-= rb
[dim0
]*rb
[dim1
]*skew_fac_01
;
7870 /* Take care that the cell planes along dim1
7871 * might not be orthogonal to that along dim2.
7873 if (normal
[dim1
][dim2
] > 0)
7875 rb
[dim2
] -= rb
[dim1
]*normal
[dim1
][dim2
];
7880 /* The distance along the communication direction */
7881 rn
[dim
] += cg_cm
[cg
][dim
] - c
->c
[dim_ind
][zone
];
7883 for (i
= dim
+1; i
< DIM
; i
++)
7885 tric_sh
-= cg_cm
[cg
][i
]*v_d
[i
][dim
];
7890 r2
+= rn
[dim
]*rn
[dim
]*skew_fac2_d
;
7891 /* Take care of coupling of the distances
7892 * to the planes along dim0 and dim1 through dim2.
7894 if (dim_ind
== 1 && zonei
== 1)
7896 r2
-= rn
[dim0
]*rn
[dim
]*skew_fac_01
;
7902 rb
[dim
] += cg_cm
[cg
][dim
] - c
->bc
[dim_ind
] + tric_sh
;
7905 rb2
+= rb
[dim
]*rb
[dim
]*skew_fac2_d
;
7906 /* Take care of coupling of the distances
7907 * to the planes along dim0 and dim1 through dim2.
7909 if (dim_ind
== 1 && zonei
== 1)
7911 rb2
-= rb
[dim0
]*rb
[dim
]*skew_fac_01
;
7919 ((bDistMB
&& rb2
< r_bcomm2
) ||
7920 (bDist2B
&& r2
< r_bcomm2
)) &&
7922 (GET_CGINFO_BOND_INTER(cginfo
[cg
]) &&
7923 missing_link(comm
->cglink
, index_gl
[cg
],
7926 /* Make an index to the local charge groups */
7927 if (nsend
+1 > ind
->nalloc
)
7929 ind
->nalloc
= over_alloc_large(nsend
+1);
7930 srenew(ind
->index
, ind
->nalloc
);
7932 if (nsend
+1 > *ibuf_nalloc
)
7934 *ibuf_nalloc
= over_alloc_large(nsend
+1);
7935 srenew(*ibuf
, *ibuf_nalloc
);
7937 ind
->index
[nsend
] = cg
;
7938 (*ibuf
)[nsend
] = index_gl
[cg
];
7940 vec_rvec_check_alloc(vbuf
, nsend
+1);
7942 if (dd
->ci
[dim
] == 0)
7944 /* Correct cg_cm for pbc */
7945 rvec_add(cg_cm
[cg
], box
[dim
], vbuf
->v
[nsend
]);
7948 vbuf
->v
[nsend
][YY
] = box
[YY
][YY
] - vbuf
->v
[nsend
][YY
];
7949 vbuf
->v
[nsend
][ZZ
] = box
[ZZ
][ZZ
] - vbuf
->v
[nsend
][ZZ
];
7954 copy_rvec(cg_cm
[cg
], vbuf
->v
[nsend
]);
7957 nat
+= cgindex
[cg
+1] - cgindex
[cg
];
7963 *nsend_z_ptr
= nsend_z
;
7966 static void setup_dd_communication(gmx_domdec_t
*dd
,
7967 matrix box
, gmx_ddbox_t
*ddbox
,
7969 t_state
*state
, PaddedRVecVector
*f
)
7971 int dim_ind
, dim
, dim0
, dim1
, dim2
, dimd
, p
, nat_tot
;
7972 int nzone
, nzone_send
, zone
, zonei
, cg0
, cg1
;
7973 int c
, i
, cg
, cg_gl
, nrcg
;
7974 int *zone_cg_range
, pos_cg
, *index_gl
, *cgindex
, *recv_i
;
7975 gmx_domdec_comm_t
*comm
;
7976 gmx_domdec_zones_t
*zones
;
7977 gmx_domdec_comm_dim_t
*cd
;
7978 gmx_domdec_ind_t
*ind
;
7979 cginfo_mb_t
*cginfo_mb
;
7980 gmx_bool bBondComm
, bDist2B
, bDistMB
, bDistBonded
;
7981 real r_comm2
, r_bcomm2
;
7982 dd_corners_t corners
;
7984 rvec
*cg_cm
, *normal
, *v_d
, *v_0
= nullptr, *v_1
= nullptr, *recv_vr
;
7985 real skew_fac2_d
, skew_fac_01
;
7992 fprintf(debug
, "Setting up DD communication\n");
7999 /* Initialize the thread data.
8000 * This can not be done in init_domain_decomposition,
8001 * as the numbers of threads is determined later.
8003 comm
->nth
= gmx_omp_nthreads_get(emntDomdec
);
8006 snew(comm
->dth
, comm
->nth
);
8010 switch (fr
->cutoff_scheme
)
8016 cg_cm
= as_rvec_array(state
->x
.data());
8019 gmx_incons("unimplemented");
8023 for (dim_ind
= 0; dim_ind
< dd
->ndim
; dim_ind
++)
8025 /* Check if we need to use triclinic distances */
8026 tric_dist
[dim_ind
] = 0;
8027 for (i
= 0; i
<= dim_ind
; i
++)
8029 if (ddbox
->tric_dir
[dd
->dim
[i
]])
8031 tric_dist
[dim_ind
] = 1;
8036 bBondComm
= comm
->bBondComm
;
8038 /* Do we need to determine extra distances for multi-body bondeds? */
8039 bDistMB
= (comm
->bInterCGMultiBody
&& isDlbOn(dd
->comm
) && dd
->ndim
> 1);
8041 /* Do we need to determine extra distances for only two-body bondeds? */
8042 bDist2B
= (bBondComm
&& !bDistMB
);
8044 r_comm2
= gmx::square(comm
->cutoff
);
8045 r_bcomm2
= gmx::square(comm
->cutoff_mbody
);
8049 fprintf(debug
, "bBondComm %d, r_bc %f\n", bBondComm
, std::sqrt(r_bcomm2
));
8052 zones
= &comm
->zones
;
8055 dim1
= (dd
->ndim
>= 2 ? dd
->dim
[1] : -1);
8056 dim2
= (dd
->ndim
>= 3 ? dd
->dim
[2] : -1);
8058 set_dd_corners(dd
, dim0
, dim1
, dim2
, bDistMB
, &corners
);
8060 /* Triclinic stuff */
8061 normal
= ddbox
->normal
;
8065 v_0
= ddbox
->v
[dim0
];
8066 if (ddbox
->tric_dir
[dim0
] && ddbox
->tric_dir
[dim1
])
8068 /* Determine the coupling coefficient for the distances
8069 * to the cell planes along dim0 and dim1 through dim2.
8070 * This is required for correct rounding.
8073 ddbox
->v
[dim0
][dim1
+1][dim0
]*ddbox
->v
[dim1
][dim1
+1][dim1
];
8076 fprintf(debug
, "\nskew_fac_01 %f\n", skew_fac_01
);
8082 v_1
= ddbox
->v
[dim1
];
8085 zone_cg_range
= zones
->cg_range
;
8086 index_gl
= dd
->index_gl
;
8087 cgindex
= dd
->cgindex
;
8088 cginfo_mb
= fr
->cginfo_mb
;
8090 zone_cg_range
[0] = 0;
8091 zone_cg_range
[1] = dd
->ncg_home
;
8092 comm
->zone_ncg1
[0] = dd
->ncg_home
;
8093 pos_cg
= dd
->ncg_home
;
8095 nat_tot
= dd
->nat_home
;
8097 for (dim_ind
= 0; dim_ind
< dd
->ndim
; dim_ind
++)
8099 dim
= dd
->dim
[dim_ind
];
8100 cd
= &comm
->cd
[dim_ind
];
8102 if (dim
>= ddbox
->npbcdim
&& dd
->ci
[dim
] == 0)
8104 /* No pbc in this dimension, the first node should not comm. */
8112 v_d
= ddbox
->v
[dim
];
8113 skew_fac2_d
= gmx::square(ddbox
->skew_fac
[dim
]);
8115 cd
->bInPlace
= TRUE
;
8116 for (p
= 0; p
< cd
->np
; p
++)
8118 /* Only atoms communicated in the first pulse are used
8119 * for multi-body bonded interactions or for bBondComm.
8121 bDistBonded
= ((bDistMB
|| bDist2B
) && p
== 0);
8126 for (zone
= 0; zone
< nzone_send
; zone
++)
8128 if (tric_dist
[dim_ind
] && dim_ind
> 0)
8130 /* Determine slightly more optimized skew_fac's
8132 * This reduces the number of communicated atoms
8133 * by about 10% for 3D DD of rhombic dodecahedra.
8135 for (dimd
= 0; dimd
< dim
; dimd
++)
8137 sf2_round
[dimd
] = 1;
8138 if (ddbox
->tric_dir
[dimd
])
8140 for (i
= dd
->dim
[dimd
]+1; i
< DIM
; i
++)
8142 /* If we are shifted in dimension i
8143 * and the cell plane is tilted forward
8144 * in dimension i, skip this coupling.
8146 if (!(zones
->shift
[nzone
+zone
][i
] &&
8147 ddbox
->v
[dimd
][i
][dimd
] >= 0))
8150 gmx::square(ddbox
->v
[dimd
][i
][dimd
]);
8153 sf2_round
[dimd
] = 1/sf2_round
[dimd
];
8158 zonei
= zone_perm
[dim_ind
][zone
];
8161 /* Here we permutate the zones to obtain a convenient order
8162 * for neighbor searching
8164 cg0
= zone_cg_range
[zonei
];
8165 cg1
= zone_cg_range
[zonei
+1];
8169 /* Look only at the cg's received in the previous grid pulse
8171 cg1
= zone_cg_range
[nzone
+zone
+1];
8172 cg0
= cg1
- cd
->ind
[p
-1].nrecv
[zone
];
8175 #pragma omp parallel for num_threads(comm->nth) schedule(static)
8176 for (th
= 0; th
< comm
->nth
; th
++)
8180 gmx_domdec_ind_t
*ind_p
;
8181 int **ibuf_p
, *ibuf_nalloc_p
;
8183 int *nsend_p
, *nat_p
;
8189 /* Thread 0 writes in the comm buffers */
8191 ibuf_p
= &comm
->buf_int
;
8192 ibuf_nalloc_p
= &comm
->nalloc_int
;
8193 vbuf_p
= &comm
->vbuf
;
8196 nsend_zone_p
= &ind
->nsend
[zone
];
8200 /* Other threads write into temp buffers */
8201 ind_p
= &comm
->dth
[th
].ind
;
8202 ibuf_p
= &comm
->dth
[th
].ibuf
;
8203 ibuf_nalloc_p
= &comm
->dth
[th
].ibuf_nalloc
;
8204 vbuf_p
= &comm
->dth
[th
].vbuf
;
8205 nsend_p
= &comm
->dth
[th
].nsend
;
8206 nat_p
= &comm
->dth
[th
].nat
;
8207 nsend_zone_p
= &comm
->dth
[th
].nsend_zone
;
8209 comm
->dth
[th
].nsend
= 0;
8210 comm
->dth
[th
].nat
= 0;
8211 comm
->dth
[th
].nsend_zone
= 0;
8221 cg0_th
= cg0
+ ((cg1
- cg0
)* th
)/comm
->nth
;
8222 cg1_th
= cg0
+ ((cg1
- cg0
)*(th
+1))/comm
->nth
;
8225 /* Get the cg's for this pulse in this zone */
8226 get_zone_pulse_cgs(dd
, zonei
, zone
, cg0_th
, cg1_th
,
8228 dim
, dim_ind
, dim0
, dim1
, dim2
,
8231 normal
, skew_fac2_d
, skew_fac_01
,
8232 v_d
, v_0
, v_1
, &corners
, sf2_round
,
8233 bDistBonded
, bBondComm
,
8237 ibuf_p
, ibuf_nalloc_p
,
8242 GMX_CATCH_ALL_AND_EXIT_WITH_FATAL_ERROR
;
8245 /* Append data of threads>=1 to the communication buffers */
8246 for (th
= 1; th
< comm
->nth
; th
++)
8248 dd_comm_setup_work_t
*dth
;
8251 dth
= &comm
->dth
[th
];
8253 ns1
= nsend
+ dth
->nsend_zone
;
8254 if (ns1
> ind
->nalloc
)
8256 ind
->nalloc
= over_alloc_dd(ns1
);
8257 srenew(ind
->index
, ind
->nalloc
);
8259 if (ns1
> comm
->nalloc_int
)
8261 comm
->nalloc_int
= over_alloc_dd(ns1
);
8262 srenew(comm
->buf_int
, comm
->nalloc_int
);
8264 if (ns1
> comm
->vbuf
.nalloc
)
8266 comm
->vbuf
.nalloc
= over_alloc_dd(ns1
);
8267 srenew(comm
->vbuf
.v
, comm
->vbuf
.nalloc
);
8270 for (i
= 0; i
< dth
->nsend_zone
; i
++)
8272 ind
->index
[nsend
] = dth
->ind
.index
[i
];
8273 comm
->buf_int
[nsend
] = dth
->ibuf
[i
];
8274 copy_rvec(dth
->vbuf
.v
[i
],
8275 comm
->vbuf
.v
[nsend
]);
8279 ind
->nsend
[zone
] += dth
->nsend_zone
;
8282 /* Clear the counts in case we do not have pbc */
8283 for (zone
= nzone_send
; zone
< nzone
; zone
++)
8285 ind
->nsend
[zone
] = 0;
8287 ind
->nsend
[nzone
] = nsend
;
8288 ind
->nsend
[nzone
+1] = nat
;
8289 /* Communicate the number of cg's and atoms to receive */
8290 dd_sendrecv_int(dd
, dim_ind
, dddirBackward
,
8291 ind
->nsend
, nzone
+2,
8292 ind
->nrecv
, nzone
+2);
8294 /* The rvec buffer is also required for atom buffers of size nsend
8295 * in dd_move_x and dd_move_f.
8297 vec_rvec_check_alloc(&comm
->vbuf
, ind
->nsend
[nzone
+1]);
8301 /* We can receive in place if only the last zone is not empty */
8302 for (zone
= 0; zone
< nzone
-1; zone
++)
8304 if (ind
->nrecv
[zone
] > 0)
8306 cd
->bInPlace
= FALSE
;
8311 /* The int buffer is only required here for the cg indices */
8312 if (ind
->nrecv
[nzone
] > comm
->nalloc_int2
)
8314 comm
->nalloc_int2
= over_alloc_dd(ind
->nrecv
[nzone
]);
8315 srenew(comm
->buf_int2
, comm
->nalloc_int2
);
8317 /* The rvec buffer is also required for atom buffers
8318 * of size nrecv in dd_move_x and dd_move_f.
8320 i
= std::max(cd
->ind
[0].nrecv
[nzone
+1], ind
->nrecv
[nzone
+1]);
8321 vec_rvec_check_alloc(&comm
->vbuf2
, i
);
8325 /* Make space for the global cg indices */
8326 if (pos_cg
+ ind
->nrecv
[nzone
] > dd
->cg_nalloc
8327 || dd
->cg_nalloc
== 0)
8329 dd
->cg_nalloc
= over_alloc_dd(pos_cg
+ ind
->nrecv
[nzone
]);
8330 srenew(index_gl
, dd
->cg_nalloc
);
8331 srenew(cgindex
, dd
->cg_nalloc
+1);
8333 /* Communicate the global cg indices */
8336 recv_i
= index_gl
+ pos_cg
;
8340 recv_i
= comm
->buf_int2
;
8342 dd_sendrecv_int(dd
, dim_ind
, dddirBackward
,
8343 comm
->buf_int
, nsend
,
8344 recv_i
, ind
->nrecv
[nzone
]);
8346 /* Make space for cg_cm */
8347 dd_check_alloc_ncg(fr
, state
, f
, pos_cg
+ ind
->nrecv
[nzone
]);
8348 if (fr
->cutoff_scheme
== ecutsGROUP
)
8354 cg_cm
= as_rvec_array(state
->x
.data());
8356 /* Communicate cg_cm */
8359 recv_vr
= cg_cm
+ pos_cg
;
8363 recv_vr
= comm
->vbuf2
.v
;
8365 dd_sendrecv_rvec(dd
, dim_ind
, dddirBackward
,
8366 comm
->vbuf
.v
, nsend
,
8367 recv_vr
, ind
->nrecv
[nzone
]);
8369 /* Make the charge group index */
8372 zone
= (p
== 0 ? 0 : nzone
- 1);
8373 while (zone
< nzone
)
8375 for (cg
= 0; cg
< ind
->nrecv
[zone
]; cg
++)
8377 cg_gl
= index_gl
[pos_cg
];
8378 fr
->cginfo
[pos_cg
] = ddcginfo(cginfo_mb
, cg_gl
);
8379 nrcg
= GET_CGINFO_NATOMS(fr
->cginfo
[pos_cg
]);
8380 cgindex
[pos_cg
+1] = cgindex
[pos_cg
] + nrcg
;
8383 /* Update the charge group presence,
8384 * so we can use it in the next pass of the loop.
8386 comm
->bLocalCG
[cg_gl
] = TRUE
;
8392 comm
->zone_ncg1
[nzone
+zone
] = ind
->nrecv
[zone
];
8395 zone_cg_range
[nzone
+zone
] = pos_cg
;
8400 /* This part of the code is never executed with bBondComm. */
8401 merge_cg_buffers(nzone
, cd
, p
, zone_cg_range
,
8402 index_gl
, recv_i
, cg_cm
, recv_vr
,
8403 cgindex
, fr
->cginfo_mb
, fr
->cginfo
);
8404 pos_cg
+= ind
->nrecv
[nzone
];
8406 nat_tot
+= ind
->nrecv
[nzone
+1];
8410 /* Store the atom block for easy copying of communication buffers */
8411 make_cell2at_index(cd
, nzone
, zone_cg_range
[nzone
], cgindex
);
8415 dd
->index_gl
= index_gl
;
8416 dd
->cgindex
= cgindex
;
8418 dd
->ncg_tot
= zone_cg_range
[zones
->n
];
8419 dd
->nat_tot
= nat_tot
;
8420 comm
->nat
[ddnatHOME
] = dd
->nat_home
;
8421 for (i
= ddnatZONE
; i
< ddnatNR
; i
++)
8423 comm
->nat
[i
] = dd
->nat_tot
;
8428 /* We don't need to update cginfo, since that was alrady done above.
8429 * So we pass NULL for the forcerec.
8431 dd_set_cginfo(dd
->index_gl
, dd
->ncg_home
, dd
->ncg_tot
,
8432 nullptr, comm
->bLocalCG
);
8437 fprintf(debug
, "Finished setting up DD communication, zones:");
8438 for (c
= 0; c
< zones
->n
; c
++)
8440 fprintf(debug
, " %d", zones
->cg_range
[c
+1]-zones
->cg_range
[c
]);
8442 fprintf(debug
, "\n");
8446 static void set_cg_boundaries(gmx_domdec_zones_t
*zones
)
8450 for (c
= 0; c
< zones
->nizone
; c
++)
8452 zones
->izone
[c
].cg1
= zones
->cg_range
[c
+1];
8453 zones
->izone
[c
].jcg0
= zones
->cg_range
[zones
->izone
[c
].j0
];
8454 zones
->izone
[c
].jcg1
= zones
->cg_range
[zones
->izone
[c
].j1
];
8458 static void set_zones_size(gmx_domdec_t
*dd
,
8459 matrix box
, const gmx_ddbox_t
*ddbox
,
8460 int zone_start
, int zone_end
)
8462 gmx_domdec_comm_t
*comm
;
8463 gmx_domdec_zones_t
*zones
;
8472 zones
= &comm
->zones
;
8474 /* Do we need to determine extra distances for multi-body bondeds? */
8475 bDistMB
= (comm
->bInterCGMultiBody
&& isDlbOn(dd
->comm
) && dd
->ndim
> 1);
8477 for (z
= zone_start
; z
< zone_end
; z
++)
8479 /* Copy cell limits to zone limits.
8480 * Valid for non-DD dims and non-shifted dims.
8482 copy_rvec(comm
->cell_x0
, zones
->size
[z
].x0
);
8483 copy_rvec(comm
->cell_x1
, zones
->size
[z
].x1
);
8486 for (d
= 0; d
< dd
->ndim
; d
++)
8490 for (z
= 0; z
< zones
->n
; z
++)
8492 /* With a staggered grid we have different sizes
8493 * for non-shifted dimensions.
8495 if (isDlbOn(dd
->comm
) && zones
->shift
[z
][dim
] == 0)
8499 zones
->size
[z
].x0
[dim
] = comm
->zone_d1
[zones
->shift
[z
][dd
->dim
[d
-1]]].min0
;
8500 zones
->size
[z
].x1
[dim
] = comm
->zone_d1
[zones
->shift
[z
][dd
->dim
[d
-1]]].max1
;
8504 zones
->size
[z
].x0
[dim
] = comm
->zone_d2
[zones
->shift
[z
][dd
->dim
[d
-2]]][zones
->shift
[z
][dd
->dim
[d
-1]]].min0
;
8505 zones
->size
[z
].x1
[dim
] = comm
->zone_d2
[zones
->shift
[z
][dd
->dim
[d
-2]]][zones
->shift
[z
][dd
->dim
[d
-1]]].max1
;
8511 rcmbs
= comm
->cutoff_mbody
;
8512 if (ddbox
->tric_dir
[dim
])
8514 rcs
/= ddbox
->skew_fac
[dim
];
8515 rcmbs
/= ddbox
->skew_fac
[dim
];
8518 /* Set the lower limit for the shifted zone dimensions */
8519 for (z
= zone_start
; z
< zone_end
; z
++)
8521 if (zones
->shift
[z
][dim
] > 0)
8524 if (!isDlbOn(dd
->comm
) || d
== 0)
8526 zones
->size
[z
].x0
[dim
] = comm
->cell_x1
[dim
];
8527 zones
->size
[z
].x1
[dim
] = comm
->cell_x1
[dim
] + rcs
;
8531 /* Here we take the lower limit of the zone from
8532 * the lowest domain of the zone below.
8536 zones
->size
[z
].x0
[dim
] =
8537 comm
->zone_d1
[zones
->shift
[z
][dd
->dim
[d
-1]]].min1
;
8543 zones
->size
[z
].x0
[dim
] =
8544 zones
->size
[zone_perm
[2][z
-4]].x0
[dim
];
8548 zones
->size
[z
].x0
[dim
] =
8549 comm
->zone_d2
[zones
->shift
[z
][dd
->dim
[d
-2]]][zones
->shift
[z
][dd
->dim
[d
-1]]].min1
;
8552 /* A temporary limit, is updated below */
8553 zones
->size
[z
].x1
[dim
] = zones
->size
[z
].x0
[dim
];
8557 for (zi
= 0; zi
< zones
->nizone
; zi
++)
8559 if (zones
->shift
[zi
][dim
] == 0)
8561 /* This takes the whole zone into account.
8562 * With multiple pulses this will lead
8563 * to a larger zone then strictly necessary.
8565 zones
->size
[z
].x1
[dim
] = std::max(zones
->size
[z
].x1
[dim
],
8566 zones
->size
[zi
].x1
[dim
]+rcmbs
);
8574 /* Loop over the i-zones to set the upper limit of each
8577 for (zi
= 0; zi
< zones
->nizone
; zi
++)
8579 if (zones
->shift
[zi
][dim
] == 0)
8581 for (z
= zones
->izone
[zi
].j0
; z
< zones
->izone
[zi
].j1
; z
++)
8583 if (zones
->shift
[z
][dim
] > 0)
8585 zones
->size
[z
].x1
[dim
] = std::max(zones
->size
[z
].x1
[dim
],
8586 zones
->size
[zi
].x1
[dim
]+rcs
);
8593 for (z
= zone_start
; z
< zone_end
; z
++)
8595 /* Initialization only required to keep the compiler happy */
8596 rvec corner_min
= {0, 0, 0}, corner_max
= {0, 0, 0}, corner
;
8599 /* To determine the bounding box for a zone we need to find
8600 * the extreme corners of 4, 2 or 1 corners.
8602 nc
= 1 << (ddbox
->nboundeddim
- 1);
8604 for (c
= 0; c
< nc
; c
++)
8606 /* Set up a zone corner at x=0, ignoring trilinic couplings */
8610 corner
[YY
] = zones
->size
[z
].x0
[YY
];
8614 corner
[YY
] = zones
->size
[z
].x1
[YY
];
8618 corner
[ZZ
] = zones
->size
[z
].x0
[ZZ
];
8622 corner
[ZZ
] = zones
->size
[z
].x1
[ZZ
];
8624 if (dd
->ndim
== 1 && dd
->dim
[0] < ZZ
&& ZZ
< dd
->npbcdim
&&
8625 box
[ZZ
][1 - dd
->dim
[0]] != 0)
8627 /* With 1D domain decomposition the cg's are not in
8628 * the triclinic box, but triclinic x-y and rectangular y/x-z.
8629 * Shift the corner of the z-vector back to along the box
8630 * vector of dimension d, so it will later end up at 0 along d.
8631 * This can affect the location of this corner along dd->dim[0]
8632 * through the matrix operation below if box[d][dd->dim[0]]!=0.
8634 int d
= 1 - dd
->dim
[0];
8636 corner
[d
] -= corner
[ZZ
]*box
[ZZ
][d
]/box
[ZZ
][ZZ
];
8638 /* Apply the triclinic couplings */
8639 assert(ddbox
->npbcdim
<= DIM
);
8640 for (i
= YY
; i
< ddbox
->npbcdim
; i
++)
8642 for (j
= XX
; j
< i
; j
++)
8644 corner
[j
] += corner
[i
]*box
[i
][j
]/box
[i
][i
];
8649 copy_rvec(corner
, corner_min
);
8650 copy_rvec(corner
, corner_max
);
8654 for (i
= 0; i
< DIM
; i
++)
8656 corner_min
[i
] = std::min(corner_min
[i
], corner
[i
]);
8657 corner_max
[i
] = std::max(corner_max
[i
], corner
[i
]);
8661 /* Copy the extreme cornes without offset along x */
8662 for (i
= 0; i
< DIM
; i
++)
8664 zones
->size
[z
].bb_x0
[i
] = corner_min
[i
];
8665 zones
->size
[z
].bb_x1
[i
] = corner_max
[i
];
8667 /* Add the offset along x */
8668 zones
->size
[z
].bb_x0
[XX
] += zones
->size
[z
].x0
[XX
];
8669 zones
->size
[z
].bb_x1
[XX
] += zones
->size
[z
].x1
[XX
];
8672 if (zone_start
== 0)
8675 for (dim
= 0; dim
< DIM
; dim
++)
8677 vol
*= zones
->size
[0].x1
[dim
] - zones
->size
[0].x0
[dim
];
8679 zones
->dens_zone0
= (zones
->cg_range
[1] - zones
->cg_range
[0])/vol
;
8684 for (z
= zone_start
; z
< zone_end
; z
++)
8686 fprintf(debug
, "zone %d %6.3f - %6.3f %6.3f - %6.3f %6.3f - %6.3f\n",
8688 zones
->size
[z
].x0
[XX
], zones
->size
[z
].x1
[XX
],
8689 zones
->size
[z
].x0
[YY
], zones
->size
[z
].x1
[YY
],
8690 zones
->size
[z
].x0
[ZZ
], zones
->size
[z
].x1
[ZZ
]);
8691 fprintf(debug
, "zone %d bb %6.3f - %6.3f %6.3f - %6.3f %6.3f - %6.3f\n",
8693 zones
->size
[z
].bb_x0
[XX
], zones
->size
[z
].bb_x1
[XX
],
8694 zones
->size
[z
].bb_x0
[YY
], zones
->size
[z
].bb_x1
[YY
],
8695 zones
->size
[z
].bb_x0
[ZZ
], zones
->size
[z
].bb_x1
[ZZ
]);
8700 static int comp_cgsort(const void *a
, const void *b
)
8704 gmx_cgsort_t
*cga
, *cgb
;
8705 cga
= (gmx_cgsort_t
*)a
;
8706 cgb
= (gmx_cgsort_t
*)b
;
8708 comp
= cga
->nsc
- cgb
->nsc
;
8711 comp
= cga
->ind_gl
- cgb
->ind_gl
;
8717 static void order_int_cg(int n
, const gmx_cgsort_t
*sort
,
8722 /* Order the data */
8723 for (i
= 0; i
< n
; i
++)
8725 buf
[i
] = a
[sort
[i
].ind
];
8728 /* Copy back to the original array */
8729 for (i
= 0; i
< n
; i
++)
8735 static void order_vec_cg(int n
, const gmx_cgsort_t
*sort
,
8740 /* Order the data */
8741 for (i
= 0; i
< n
; i
++)
8743 copy_rvec(v
[sort
[i
].ind
], buf
[i
]);
8746 /* Copy back to the original array */
8747 for (i
= 0; i
< n
; i
++)
8749 copy_rvec(buf
[i
], v
[i
]);
8753 static void order_vec_atom(int ncg
, const int *cgindex
, const gmx_cgsort_t
*sort
,
8756 int a
, atot
, cg
, cg0
, cg1
, i
;
8758 if (cgindex
== nullptr)
8760 /* Avoid the useless loop of the atoms within a cg */
8761 order_vec_cg(ncg
, sort
, v
, buf
);
8766 /* Order the data */
8768 for (cg
= 0; cg
< ncg
; cg
++)
8770 cg0
= cgindex
[sort
[cg
].ind
];
8771 cg1
= cgindex
[sort
[cg
].ind
+1];
8772 for (i
= cg0
; i
< cg1
; i
++)
8774 copy_rvec(v
[i
], buf
[a
]);
8780 /* Copy back to the original array */
8781 for (a
= 0; a
< atot
; a
++)
8783 copy_rvec(buf
[a
], v
[a
]);
8787 static void ordered_sort(int nsort2
, gmx_cgsort_t
*sort2
,
8788 int nsort_new
, gmx_cgsort_t
*sort_new
,
8789 gmx_cgsort_t
*sort1
)
8793 /* The new indices are not very ordered, so we qsort them */
8794 gmx_qsort_threadsafe(sort_new
, nsort_new
, sizeof(sort_new
[0]), comp_cgsort
);
8796 /* sort2 is already ordered, so now we can merge the two arrays */
8800 while (i2
< nsort2
|| i_new
< nsort_new
)
8804 sort1
[i1
++] = sort_new
[i_new
++];
8806 else if (i_new
== nsort_new
)
8808 sort1
[i1
++] = sort2
[i2
++];
8810 else if (sort2
[i2
].nsc
< sort_new
[i_new
].nsc
||
8811 (sort2
[i2
].nsc
== sort_new
[i_new
].nsc
&&
8812 sort2
[i2
].ind_gl
< sort_new
[i_new
].ind_gl
))
8814 sort1
[i1
++] = sort2
[i2
++];
8818 sort1
[i1
++] = sort_new
[i_new
++];
8823 static int dd_sort_order(gmx_domdec_t
*dd
, t_forcerec
*fr
, int ncg_home_old
)
8825 gmx_domdec_sort_t
*sort
;
8826 gmx_cgsort_t
*cgsort
, *sort_i
;
8827 int ncg_new
, nsort2
, nsort_new
, i
, *a
, moved
;
8829 sort
= dd
->comm
->sort
;
8831 a
= fr
->ns
->grid
->cell_index
;
8833 moved
= NSGRID_SIGNAL_MOVED_FAC
*fr
->ns
->grid
->ncells
;
8835 if (ncg_home_old
>= 0)
8837 /* The charge groups that remained in the same ns grid cell
8838 * are completely ordered. So we can sort efficiently by sorting
8839 * the charge groups that did move into the stationary list.
8844 for (i
= 0; i
< dd
->ncg_home
; i
++)
8846 /* Check if this cg did not move to another node */
8849 if (i
>= ncg_home_old
|| a
[i
] != sort
->sort
[i
].nsc
)
8851 /* This cg is new on this node or moved ns grid cell */
8852 if (nsort_new
>= sort
->sort_new_nalloc
)
8854 sort
->sort_new_nalloc
= over_alloc_dd(nsort_new
+1);
8855 srenew(sort
->sort_new
, sort
->sort_new_nalloc
);
8857 sort_i
= &(sort
->sort_new
[nsort_new
++]);
8861 /* This cg did not move */
8862 sort_i
= &(sort
->sort2
[nsort2
++]);
8864 /* Sort on the ns grid cell indices
8865 * and the global topology index.
8866 * index_gl is irrelevant with cell ns,
8867 * but we set it here anyhow to avoid a conditional.
8870 sort_i
->ind_gl
= dd
->index_gl
[i
];
8877 fprintf(debug
, "ordered sort cgs: stationary %d moved %d\n",
8880 /* Sort efficiently */
8881 ordered_sort(nsort2
, sort
->sort2
, nsort_new
, sort
->sort_new
,
8886 cgsort
= sort
->sort
;
8888 for (i
= 0; i
< dd
->ncg_home
; i
++)
8890 /* Sort on the ns grid cell indices
8891 * and the global topology index
8893 cgsort
[i
].nsc
= a
[i
];
8894 cgsort
[i
].ind_gl
= dd
->index_gl
[i
];
8896 if (cgsort
[i
].nsc
< moved
)
8903 fprintf(debug
, "qsort cgs: %d new home %d\n", dd
->ncg_home
, ncg_new
);
8905 /* Determine the order of the charge groups using qsort */
8906 gmx_qsort_threadsafe(cgsort
, dd
->ncg_home
, sizeof(cgsort
[0]), comp_cgsort
);
8912 static int dd_sort_order_nbnxn(gmx_domdec_t
*dd
, t_forcerec
*fr
)
8918 sort
= dd
->comm
->sort
->sort
;
8920 nbnxn_get_atomorder(fr
->nbv
->nbs
, &a
, &na
);
8923 for (i
= 0; i
< na
; i
++)
8927 sort
[ncg_new
].ind
= a
[i
];
8935 static void dd_sort_state(gmx_domdec_t
*dd
, rvec
*cgcm
, t_forcerec
*fr
, t_state
*state
,
8938 gmx_domdec_sort_t
*sort
;
8939 gmx_cgsort_t
*cgsort
;
8941 int ncg_new
, i
, *ibuf
, cgsize
;
8944 sort
= dd
->comm
->sort
;
8946 if (dd
->ncg_home
> sort
->sort_nalloc
)
8948 sort
->sort_nalloc
= over_alloc_dd(dd
->ncg_home
);
8949 srenew(sort
->sort
, sort
->sort_nalloc
);
8950 srenew(sort
->sort2
, sort
->sort_nalloc
);
8952 cgsort
= sort
->sort
;
8954 switch (fr
->cutoff_scheme
)
8957 ncg_new
= dd_sort_order(dd
, fr
, ncg_home_old
);
8960 ncg_new
= dd_sort_order_nbnxn(dd
, fr
);
8963 gmx_incons("unimplemented");
8967 /* We alloc with the old size, since cgindex is still old */
8968 vec_rvec_check_alloc(&dd
->comm
->vbuf
, dd
->cgindex
[dd
->ncg_home
]);
8969 vbuf
= dd
->comm
->vbuf
.v
;
8973 cgindex
= dd
->cgindex
;
8980 /* Remove the charge groups which are no longer at home here */
8981 dd
->ncg_home
= ncg_new
;
8984 fprintf(debug
, "Set the new home charge group count to %d\n",
8988 /* Reorder the state */
8989 if (state
->flags
& (1 << estX
))
8991 order_vec_atom(dd
->ncg_home
, cgindex
, cgsort
, as_rvec_array(state
->x
.data()), vbuf
);
8993 if (state
->flags
& (1 << estV
))
8995 order_vec_atom(dd
->ncg_home
, cgindex
, cgsort
, as_rvec_array(state
->v
.data()), vbuf
);
8997 if (state
->flags
& (1 << estCGP
))
8999 order_vec_atom(dd
->ncg_home
, cgindex
, cgsort
, as_rvec_array(state
->cg_p
.data()), vbuf
);
9002 if (fr
->cutoff_scheme
== ecutsGROUP
)
9005 order_vec_cg(dd
->ncg_home
, cgsort
, cgcm
, vbuf
);
9008 if (dd
->ncg_home
+1 > sort
->ibuf_nalloc
)
9010 sort
->ibuf_nalloc
= over_alloc_dd(dd
->ncg_home
+1);
9011 srenew(sort
->ibuf
, sort
->ibuf_nalloc
);
9014 /* Reorder the global cg index */
9015 order_int_cg(dd
->ncg_home
, cgsort
, dd
->index_gl
, ibuf
);
9016 /* Reorder the cginfo */
9017 order_int_cg(dd
->ncg_home
, cgsort
, fr
->cginfo
, ibuf
);
9018 /* Rebuild the local cg index */
9022 for (i
= 0; i
< dd
->ncg_home
; i
++)
9024 cgsize
= dd
->cgindex
[cgsort
[i
].ind
+1] - dd
->cgindex
[cgsort
[i
].ind
];
9025 ibuf
[i
+1] = ibuf
[i
] + cgsize
;
9027 for (i
= 0; i
< dd
->ncg_home
+1; i
++)
9029 dd
->cgindex
[i
] = ibuf
[i
];
9034 for (i
= 0; i
< dd
->ncg_home
+1; i
++)
9039 /* Set the home atom number */
9040 dd
->nat_home
= dd
->cgindex
[dd
->ncg_home
];
9042 if (fr
->cutoff_scheme
== ecutsVERLET
)
9044 /* The atoms are now exactly in grid order, update the grid order */
9045 nbnxn_set_atomorder(fr
->nbv
->nbs
);
9049 /* Copy the sorted ns cell indices back to the ns grid struct */
9050 for (i
= 0; i
< dd
->ncg_home
; i
++)
9052 fr
->ns
->grid
->cell_index
[i
] = cgsort
[i
].nsc
;
9054 fr
->ns
->grid
->nr
= dd
->ncg_home
;
9058 static void add_dd_statistics(gmx_domdec_t
*dd
)
9060 gmx_domdec_comm_t
*comm
;
9065 for (ddnat
= ddnatZONE
; ddnat
< ddnatNR
; ddnat
++)
9067 comm
->sum_nat
[ddnat
-ddnatZONE
] +=
9068 comm
->nat
[ddnat
] - comm
->nat
[ddnat
-1];
9073 void reset_dd_statistics_counters(gmx_domdec_t
*dd
)
9075 gmx_domdec_comm_t
*comm
;
9080 /* Reset all the statistics and counters for total run counting */
9081 for (ddnat
= ddnatZONE
; ddnat
< ddnatNR
; ddnat
++)
9083 comm
->sum_nat
[ddnat
-ddnatZONE
] = 0;
9087 comm
->load_step
= 0;
9090 clear_ivec(comm
->load_lim
);
9095 void print_dd_statistics(t_commrec
*cr
, const t_inputrec
*ir
, FILE *fplog
)
9097 gmx_domdec_comm_t
*comm
;
9101 comm
= cr
->dd
->comm
;
9103 gmx_sumd(ddnatNR
-ddnatZONE
, comm
->sum_nat
, cr
);
9105 if (fplog
== nullptr)
9110 fprintf(fplog
, "\n D O M A I N D E C O M P O S I T I O N S T A T I S T I C S\n\n");
9112 for (ddnat
= ddnatZONE
; ddnat
< ddnatNR
; ddnat
++)
9114 av
= comm
->sum_nat
[ddnat
-ddnatZONE
]/comm
->ndecomp
;
9119 " av. #atoms communicated per step for force: %d x %.1f\n",
9123 if (cr
->dd
->vsite_comm
)
9126 " av. #atoms communicated per step for vsites: %d x %.1f\n",
9127 (EEL_PME(ir
->coulombtype
) || ir
->coulombtype
== eelEWALD
) ? 3 : 2,
9132 if (cr
->dd
->constraint_comm
)
9135 " av. #atoms communicated per step for LINCS: %d x %.1f\n",
9136 1 + ir
->nLincsIter
, av
);
9140 gmx_incons(" Unknown type for DD statistics");
9143 fprintf(fplog
, "\n");
9145 if (comm
->bRecordLoad
&& EI_DYNAMICS(ir
->eI
))
9147 print_dd_load_av(fplog
, cr
->dd
);
9151 void dd_partition_system(FILE *fplog
,
9154 gmx_bool bMasterState
,
9156 t_state
*state_global
,
9157 const gmx_mtop_t
*top_global
,
9158 const t_inputrec
*ir
,
9159 t_state
*state_local
,
9160 PaddedRVecVector
*f
,
9162 gmx_localtop_t
*top_local
,
9165 gmx_constr_t constr
,
9167 gmx_wallcycle_t wcycle
,
9171 gmx_domdec_comm_t
*comm
;
9172 gmx_ddbox_t ddbox
= {0};
9174 gmx_int64_t step_pcoupl
;
9175 rvec cell_ns_x0
, cell_ns_x1
;
9176 int i
, n
, ncgindex_set
, ncg_home_old
= -1, ncg_moved
, nat_f_novirsum
;
9177 gmx_bool bBoxChanged
, bNStGlobalComm
, bDoDLB
, bCheckWhetherToTurnDlbOn
, bLogLoad
;
9178 gmx_bool bRedist
, bSortCG
, bResortAll
;
9179 ivec ncells_old
= {0, 0, 0}, ncells_new
= {0, 0, 0}, np
;
9183 wallcycle_start(wcycle
, ewcDOMDEC
);
9188 bBoxChanged
= (bMasterState
|| inputrecDeform(ir
));
9189 if (ir
->epc
!= epcNO
)
9191 /* With nstpcouple > 1 pressure coupling happens.
9192 * one step after calculating the pressure.
9193 * Box scaling happens at the end of the MD step,
9194 * after the DD partitioning.
9195 * We therefore have to do DLB in the first partitioning
9196 * after an MD step where P-coupling occurred.
9197 * We need to determine the last step in which p-coupling occurred.
9198 * MRS -- need to validate this for vv?
9203 step_pcoupl
= step
- 1;
9207 step_pcoupl
= ((step
- 1)/n
)*n
+ 1;
9209 if (step_pcoupl
>= comm
->partition_step
)
9215 bNStGlobalComm
= (step
% nstglobalcomm
== 0);
9223 /* Should we do dynamic load balacing this step?
9224 * Since it requires (possibly expensive) global communication,
9225 * we might want to do DLB less frequently.
9227 if (bBoxChanged
|| ir
->epc
!= epcNO
)
9229 bDoDLB
= bBoxChanged
;
9233 bDoDLB
= bNStGlobalComm
;
9237 /* Check if we have recorded loads on the nodes */
9238 if (comm
->bRecordLoad
&& dd_load_count(comm
) > 0)
9240 bCheckWhetherToTurnDlbOn
= dd_dlb_get_should_check_whether_to_turn_dlb_on(dd
);
9242 /* Print load every nstlog, first and last step to the log file */
9243 bLogLoad
= ((ir
->nstlog
> 0 && step
% ir
->nstlog
== 0) ||
9244 comm
->n_load_collect
== 0 ||
9246 (step
+ ir
->nstlist
> ir
->init_step
+ ir
->nsteps
)));
9248 /* Avoid extra communication due to verbose screen output
9249 * when nstglobalcomm is set.
9251 if (bDoDLB
|| bLogLoad
|| bCheckWhetherToTurnDlbOn
||
9252 (bVerbose
&& (ir
->nstlist
== 0 || nstglobalcomm
<= ir
->nstlist
)))
9254 get_load_distribution(dd
, wcycle
);
9259 dd_print_load(fplog
, dd
, step
-1);
9263 dd_print_load_verbose(dd
);
9266 comm
->n_load_collect
++;
9272 /* Add the measured cycles to the running average */
9273 const float averageFactor
= 0.1f
;
9274 comm
->cyclesPerStepDlbExpAverage
=
9275 (1 - averageFactor
)*comm
->cyclesPerStepDlbExpAverage
+
9276 averageFactor
*comm
->cycl
[ddCyclStep
]/comm
->cycl_n
[ddCyclStep
];
9278 if (comm
->dlbState
== edlbsOnCanTurnOff
&&
9279 dd
->comm
->n_load_have
% c_checkTurnDlbOffInterval
== c_checkTurnDlbOffInterval
- 1)
9281 gmx_bool turnOffDlb
;
9284 /* If the running averaged cycles with DLB are more
9285 * than before we turned on DLB, turn off DLB.
9286 * We will again run and check the cycles without DLB
9287 * and we can then decide if to turn off DLB forever.
9289 turnOffDlb
= (comm
->cyclesPerStepDlbExpAverage
>
9290 comm
->cyclesPerStepBeforeDLB
);
9292 dd_bcast(dd
, sizeof(turnOffDlb
), &turnOffDlb
);
9295 /* To turn off DLB, we need to redistribute the atoms */
9296 dd_collect_state(dd
, state_local
, state_global
);
9297 bMasterState
= TRUE
;
9298 turn_off_dlb(fplog
, cr
, step
);
9302 else if (bCheckWhetherToTurnDlbOn
)
9304 gmx_bool turnOffDlbForever
= FALSE
;
9305 gmx_bool turnOnDlb
= FALSE
;
9307 /* Since the timings are node dependent, the master decides */
9310 /* If we recently turned off DLB, we want to check if
9311 * performance is better without DLB. We want to do this
9312 * ASAP to minimize the chance that external factors
9313 * slowed down the DLB step are gone here and we
9314 * incorrectly conclude that DLB was causing the slowdown.
9315 * So we measure one nstlist block, no running average.
9317 if (comm
->haveTurnedOffDlb
&&
9318 comm
->cycl
[ddCyclStep
]/comm
->cycl_n
[ddCyclStep
] <
9319 comm
->cyclesPerStepDlbExpAverage
)
9321 /* After turning off DLB we ran nstlist steps in fewer
9322 * cycles than with DLB. This likely means that DLB
9323 * in not benefical, but this could be due to a one
9324 * time unlucky fluctuation, so we require two such
9325 * observations in close succession to turn off DLB
9328 if (comm
->dlbSlowerPartitioningCount
> 0 &&
9329 dd
->ddp_count
< comm
->dlbSlowerPartitioningCount
+ 10*c_checkTurnDlbOnInterval
)
9331 turnOffDlbForever
= TRUE
;
9333 comm
->haveTurnedOffDlb
= false;
9334 /* Register when we last measured DLB slowdown */
9335 comm
->dlbSlowerPartitioningCount
= dd
->ddp_count
;
9339 /* Here we check if the max PME rank load is more than 0.98
9340 * the max PP force load. If so, PP DLB will not help,
9341 * since we are (almost) limited by PME. Furthermore,
9342 * DLB will cause a significant extra x/f redistribution
9343 * cost on the PME ranks, which will then surely result
9344 * in lower total performance.
9346 if (cr
->npmenodes
> 0 &&
9347 dd_pme_f_ratio(dd
) > 1 - DD_PERF_LOSS_DLB_ON
)
9353 turnOnDlb
= (dd_force_imb_perf_loss(dd
) >= DD_PERF_LOSS_DLB_ON
);
9359 gmx_bool turnOffDlbForever
;
9363 turnOffDlbForever
, turnOnDlb
9365 dd_bcast(dd
, sizeof(bools
), &bools
);
9366 if (bools
.turnOffDlbForever
)
9368 turn_off_dlb_forever(fplog
, cr
, step
);
9370 else if (bools
.turnOnDlb
)
9372 turn_on_dlb(fplog
, cr
, step
);
9377 comm
->n_load_have
++;
9380 cgs_gl
= &comm
->cgs_gl
;
9385 /* Clear the old state */
9386 clear_dd_indices(dd
, 0, 0);
9389 set_ddbox(dd
, bMasterState
, cr
, ir
, state_global
->box
,
9390 TRUE
, cgs_gl
, as_rvec_array(state_global
->x
.data()), &ddbox
);
9392 get_cg_distribution(fplog
, dd
, cgs_gl
,
9393 state_global
->box
, &ddbox
, as_rvec_array(state_global
->x
.data()));
9395 dd_distribute_state(dd
, cgs_gl
,
9396 state_global
, state_local
, f
);
9398 dd_make_local_cgs(dd
, &top_local
->cgs
);
9400 /* Ensure that we have space for the new distribution */
9401 dd_check_alloc_ncg(fr
, state_local
, f
, dd
->ncg_home
);
9403 if (fr
->cutoff_scheme
== ecutsGROUP
)
9405 calc_cgcm(fplog
, 0, dd
->ncg_home
,
9406 &top_local
->cgs
, as_rvec_array(state_local
->x
.data()), fr
->cg_cm
);
9409 inc_nrnb(nrnb
, eNR_CGCM
, dd
->nat_home
);
9411 dd_set_cginfo(dd
->index_gl
, 0, dd
->ncg_home
, fr
, comm
->bLocalCG
);
9413 else if (state_local
->ddp_count
!= dd
->ddp_count
)
9415 if (state_local
->ddp_count
> dd
->ddp_count
)
9417 gmx_fatal(FARGS
, "Internal inconsistency state_local->ddp_count (%d) > dd->ddp_count (%d)", state_local
->ddp_count
, dd
->ddp_count
);
9420 if (state_local
->ddp_count_cg_gl
!= state_local
->ddp_count
)
9422 gmx_fatal(FARGS
, "Internal inconsistency state_local->ddp_count_cg_gl (%d) != state_local->ddp_count (%d)", state_local
->ddp_count_cg_gl
, state_local
->ddp_count
);
9425 /* Clear the old state */
9426 clear_dd_indices(dd
, 0, 0);
9428 /* Build the new indices */
9429 rebuild_cgindex(dd
, cgs_gl
->index
, state_local
);
9430 make_dd_indices(dd
, cgs_gl
->index
, 0);
9431 ncgindex_set
= dd
->ncg_home
;
9433 if (fr
->cutoff_scheme
== ecutsGROUP
)
9435 /* Redetermine the cg COMs */
9436 calc_cgcm(fplog
, 0, dd
->ncg_home
,
9437 &top_local
->cgs
, as_rvec_array(state_local
->x
.data()), fr
->cg_cm
);
9440 inc_nrnb(nrnb
, eNR_CGCM
, dd
->nat_home
);
9442 dd_set_cginfo(dd
->index_gl
, 0, dd
->ncg_home
, fr
, comm
->bLocalCG
);
9444 set_ddbox(dd
, bMasterState
, cr
, ir
, state_local
->box
,
9445 TRUE
, &top_local
->cgs
, as_rvec_array(state_local
->x
.data()), &ddbox
);
9447 bRedist
= isDlbOn(comm
);
9451 /* We have the full state, only redistribute the cgs */
9453 /* Clear the non-home indices */
9454 clear_dd_indices(dd
, dd
->ncg_home
, dd
->nat_home
);
9457 /* Avoid global communication for dim's without pbc and -gcom */
9458 if (!bNStGlobalComm
)
9460 copy_rvec(comm
->box0
, ddbox
.box0
);
9461 copy_rvec(comm
->box_size
, ddbox
.box_size
);
9463 set_ddbox(dd
, bMasterState
, cr
, ir
, state_local
->box
,
9464 bNStGlobalComm
, &top_local
->cgs
, as_rvec_array(state_local
->x
.data()), &ddbox
);
9469 /* For dim's without pbc and -gcom */
9470 copy_rvec(ddbox
.box0
, comm
->box0
);
9471 copy_rvec(ddbox
.box_size
, comm
->box_size
);
9473 set_dd_cell_sizes(dd
, &ddbox
, dynamic_dd_box(&ddbox
, ir
), bMasterState
, bDoDLB
,
9476 if (comm
->nstDDDumpGrid
> 0 && step
% comm
->nstDDDumpGrid
== 0)
9478 write_dd_grid_pdb("dd_grid", step
, dd
, state_local
->box
, &ddbox
);
9481 /* Check if we should sort the charge groups */
9482 bSortCG
= (bMasterState
|| bRedist
);
9484 ncg_home_old
= dd
->ncg_home
;
9489 wallcycle_sub_start(wcycle
, ewcsDD_REDIST
);
9491 dd_redistribute_cg(fplog
, step
, dd
, ddbox
.tric_dir
,
9493 !bSortCG
, nrnb
, &ncgindex_set
, &ncg_moved
);
9495 wallcycle_sub_stop(wcycle
, ewcsDD_REDIST
);
9498 get_nsgrid_boundaries(ddbox
.nboundeddim
, state_local
->box
,
9500 &comm
->cell_x0
, &comm
->cell_x1
,
9501 dd
->ncg_home
, fr
->cg_cm
,
9502 cell_ns_x0
, cell_ns_x1
, &grid_density
);
9506 comm_dd_ns_cell_sizes(dd
, &ddbox
, cell_ns_x0
, cell_ns_x1
, step
);
9509 switch (fr
->cutoff_scheme
)
9512 copy_ivec(fr
->ns
->grid
->n
, ncells_old
);
9513 grid_first(fplog
, fr
->ns
->grid
, dd
, &ddbox
,
9514 state_local
->box
, cell_ns_x0
, cell_ns_x1
,
9515 fr
->rlist
, grid_density
);
9518 nbnxn_get_ncells(fr
->nbv
->nbs
, &ncells_old
[XX
], &ncells_old
[YY
]);
9521 gmx_incons("unimplemented");
9523 /* We need to store tric_dir for dd_get_ns_ranges called from ns.c */
9524 copy_ivec(ddbox
.tric_dir
, comm
->tric_dir
);
9528 wallcycle_sub_start(wcycle
, ewcsDD_GRID
);
9530 /* Sort the state on charge group position.
9531 * This enables exact restarts from this step.
9532 * It also improves performance by about 15% with larger numbers
9533 * of atoms per node.
9536 /* Fill the ns grid with the home cell,
9537 * so we can sort with the indices.
9539 set_zones_ncg_home(dd
);
9541 switch (fr
->cutoff_scheme
)
9544 set_zones_size(dd
, state_local
->box
, &ddbox
, 0, 1);
9546 nbnxn_put_on_grid(fr
->nbv
->nbs
, fr
->ePBC
, state_local
->box
,
9548 comm
->zones
.size
[0].bb_x0
,
9549 comm
->zones
.size
[0].bb_x1
,
9551 comm
->zones
.dens_zone0
,
9553 as_rvec_array(state_local
->x
.data()),
9554 ncg_moved
, bRedist
? comm
->moved
: nullptr,
9555 fr
->nbv
->grp
[eintLocal
].kernel_type
,
9556 fr
->nbv
->grp
[eintLocal
].nbat
);
9558 nbnxn_get_ncells(fr
->nbv
->nbs
, &ncells_new
[XX
], &ncells_new
[YY
]);
9561 fill_grid(&comm
->zones
, fr
->ns
->grid
, dd
->ncg_home
,
9562 0, dd
->ncg_home
, fr
->cg_cm
);
9564 copy_ivec(fr
->ns
->grid
->n
, ncells_new
);
9567 gmx_incons("unimplemented");
9570 bResortAll
= bMasterState
;
9572 /* Check if we can user the old order and ns grid cell indices
9573 * of the charge groups to sort the charge groups efficiently.
9575 if (ncells_new
[XX
] != ncells_old
[XX
] ||
9576 ncells_new
[YY
] != ncells_old
[YY
] ||
9577 ncells_new
[ZZ
] != ncells_old
[ZZ
])
9584 fprintf(debug
, "Step %s, sorting the %d home charge groups\n",
9585 gmx_step_str(step
, sbuf
), dd
->ncg_home
);
9587 dd_sort_state(dd
, fr
->cg_cm
, fr
, state_local
,
9588 bResortAll
? -1 : ncg_home_old
);
9590 /* After sorting and compacting we set the correct size */
9591 dd_resize_state(state_local
, f
, dd
->nat_home
);
9593 /* Rebuild all the indices */
9594 ga2la_clear(dd
->ga2la
);
9597 wallcycle_sub_stop(wcycle
, ewcsDD_GRID
);
9600 wallcycle_sub_start(wcycle
, ewcsDD_SETUPCOMM
);
9602 /* Setup up the communication and communicate the coordinates */
9603 setup_dd_communication(dd
, state_local
->box
, &ddbox
, fr
, state_local
, f
);
9605 /* Set the indices */
9606 make_dd_indices(dd
, cgs_gl
->index
, ncgindex_set
);
9608 /* Set the charge group boundaries for neighbor searching */
9609 set_cg_boundaries(&comm
->zones
);
9611 if (fr
->cutoff_scheme
== ecutsVERLET
)
9613 set_zones_size(dd
, state_local
->box
, &ddbox
,
9614 bSortCG
? 1 : 0, comm
->zones
.n
);
9617 wallcycle_sub_stop(wcycle
, ewcsDD_SETUPCOMM
);
9620 write_dd_pdb("dd_home",step,"dump",top_global,cr,
9621 -1,as_rvec_array(state_local->x.data()),state_local->box);
9624 wallcycle_sub_start(wcycle
, ewcsDD_MAKETOP
);
9626 /* Extract a local topology from the global topology */
9627 for (i
= 0; i
< dd
->ndim
; i
++)
9629 np
[dd
->dim
[i
]] = comm
->cd
[i
].np
;
9631 dd_make_local_top(dd
, &comm
->zones
, dd
->npbcdim
, state_local
->box
,
9632 comm
->cellsize_min
, np
,
9634 fr
->cutoff_scheme
== ecutsGROUP
? fr
->cg_cm
: as_rvec_array(state_local
->x
.data()),
9635 vsite
, top_global
, top_local
);
9637 wallcycle_sub_stop(wcycle
, ewcsDD_MAKETOP
);
9639 wallcycle_sub_start(wcycle
, ewcsDD_MAKECONSTR
);
9641 /* Set up the special atom communication */
9642 n
= comm
->nat
[ddnatZONE
];
9643 for (i
= ddnatZONE
+1; i
< ddnatNR
; i
++)
9648 if (vsite
&& vsite
->n_intercg_vsite
)
9650 n
= dd_make_local_vsites(dd
, n
, top_local
->idef
.il
);
9654 if (dd
->bInterCGcons
|| dd
->bInterCGsettles
)
9656 /* Only for inter-cg constraints we need special code */
9657 n
= dd_make_local_constraints(dd
, n
, top_global
, fr
->cginfo
,
9658 constr
, ir
->nProjOrder
,
9659 top_local
->idef
.il
);
9663 gmx_incons("Unknown special atom type setup");
9668 wallcycle_sub_stop(wcycle
, ewcsDD_MAKECONSTR
);
9670 wallcycle_sub_start(wcycle
, ewcsDD_TOPOTHER
);
9672 /* Make space for the extra coordinates for virtual site
9673 * or constraint communication.
9675 state_local
->natoms
= comm
->nat
[ddnatNR
-1];
9677 dd_resize_state(state_local
, f
, state_local
->natoms
);
9679 if (fr
->bF_NoVirSum
)
9681 if (vsite
&& vsite
->n_intercg_vsite
)
9683 nat_f_novirsum
= comm
->nat
[ddnatVSITE
];
9687 if (EEL_FULL(ir
->coulombtype
) && dd
->n_intercg_excl
> 0)
9689 nat_f_novirsum
= dd
->nat_tot
;
9693 nat_f_novirsum
= dd
->nat_home
;
9702 /* Set the number of atoms required for the force calculation.
9703 * Forces need to be constrained when doing energy
9704 * minimization. For simple simulations we could avoid some
9705 * allocation, zeroing and copying, but this is probably not worth
9706 * the complications and checking.
9708 forcerec_set_ranges(fr
, dd
->ncg_home
, dd
->ncg_tot
,
9709 dd
->nat_tot
, comm
->nat
[ddnatCON
], nat_f_novirsum
);
9711 /* Update atom data for mdatoms and several algorithms */
9712 mdAlgorithmsSetupAtomData(cr
, ir
, top_global
, top_local
, fr
,
9713 nullptr, mdatoms
, vsite
, nullptr);
9715 if (ir
->implicit_solvent
)
9717 make_local_gb(cr
, fr
->born
, ir
->gb_algorithm
);
9720 if (!(cr
->duty
& DUTY_PME
))
9722 /* Send the charges and/or c6/sigmas to our PME only node */
9723 gmx_pme_send_parameters(cr
,
9725 mdatoms
->nChargePerturbed
, mdatoms
->nTypePerturbed
,
9726 mdatoms
->chargeA
, mdatoms
->chargeB
,
9727 mdatoms
->sqrt_c6A
, mdatoms
->sqrt_c6B
,
9728 mdatoms
->sigmaA
, mdatoms
->sigmaB
,
9729 dd_pme_maxshift_x(dd
), dd_pme_maxshift_y(dd
));
9734 set_constraints(constr
, top_local
, ir
, mdatoms
, cr
);
9739 /* Update the local pull groups */
9740 dd_make_local_pull_groups(cr
, ir
->pull_work
, mdatoms
);
9745 /* Update the local rotation groups */
9746 dd_make_local_rotation_groups(dd
, ir
->rot
);
9749 if (ir
->eSwapCoords
!= eswapNO
)
9751 /* Update the local groups needed for ion swapping */
9752 dd_make_local_swap_groups(dd
, ir
->swap
);
9755 /* Update the local atoms to be communicated via the IMD protocol if bIMD is TRUE. */
9756 dd_make_local_IMD_atoms(ir
->bIMD
, dd
, ir
->imd
);
9758 add_dd_statistics(dd
);
9760 /* Make sure we only count the cycles for this DD partitioning */
9761 clear_dd_cycle_counts(dd
);
9763 /* Because the order of the atoms might have changed since
9764 * the last vsite construction, we need to communicate the constructing
9765 * atom coordinates again (for spreading the forces this MD step).
9767 dd_move_x_vsites(dd
, state_local
->box
, as_rvec_array(state_local
->x
.data()));
9769 wallcycle_sub_stop(wcycle
, ewcsDD_TOPOTHER
);
9771 if (comm
->nstDDDump
> 0 && step
% comm
->nstDDDump
== 0)
9773 dd_move_x(dd
, state_local
->box
, as_rvec_array(state_local
->x
.data()));
9774 write_dd_pdb("dd_dump", step
, "dump", top_global
, cr
,
9775 -1, as_rvec_array(state_local
->x
.data()), state_local
->box
);
9778 /* Store the partitioning step */
9779 comm
->partition_step
= step
;
9781 /* Increase the DD partitioning counter */
9783 /* The state currently matches this DD partitioning count, store it */
9784 state_local
->ddp_count
= dd
->ddp_count
;
9787 /* The DD master node knows the complete cg distribution,
9788 * store the count so we can possibly skip the cg info communication.
9790 comm
->master_cg_ddp_count
= (bSortCG
? 0 : dd
->ddp_count
);
9793 if (comm
->DD_debug
> 0)
9795 /* Set the env var GMX_DD_DEBUG if you suspect corrupted indices */
9796 check_index_consistency(dd
, top_global
->natoms
, ncg_mtop(top_global
),
9797 "after partitioning");
9800 wallcycle_stop(wcycle
, ewcDOMDEC
);