Original WRF subgrid support version from John Michalakes without fire
[wrffire.git] / wrfv2_fire / external / RSL / RSL / exch_period.c
blob8787846e76b4c344583b13f85991231726b5d8c2
1 /***********************************************************************
3 COPYRIGHT
5 The following is a notice of limited availability of the code and
6 Government license and disclaimer which must be included in the
7 prologue of the code and in all source listings of the code.
9 Copyright notice
10 (c) 1977 University of Chicago
12 Permission is hereby granted to use, reproduce, prepare
13 derivative works, and to redistribute to others at no charge. If
14 you distribute a copy or copies of the Software, or you modify a
15 copy or copies of the Software or any portion of it, thus forming
16 a work based on the Software and make and/or distribute copies of
17 such work, you must meet the following conditions:
19 a) If you make a copy of the Software (modified or verbatim)
20 it must include the copyright notice and Government
21 license and disclaimer.
23 b) You must cause the modified Software to carry prominent
24 notices stating that you changed specified portions of
25 the Software.
27 This software was authored by:
29 Argonne National Laboratory
30 J. Michalakes: (630) 252-6646; email: michalak@mcs.anl.gov
31 Mathematics and Computer Science Division
32 Argonne National Laboratory, Argonne, IL 60439
34 ARGONNE NATIONAL LABORATORY (ANL), WITH FACILITIES IN THE STATES
35 OF ILLINOIS AND IDAHO, IS OWNED BY THE UNITED STATES GOVERNMENT,
36 AND OPERATED BY THE UNIVERSITY OF CHICAGO UNDER PROVISION OF A
37 CONTRACT WITH THE DEPARTMENT OF ENERGY.
39 GOVERNMENT LICENSE AND DISCLAIMER
41 This computer code material was prepared, in part, as an account
42 of work sponsored by an agency of the United States Government.
43 The Government is granted for itself and others acting on its
44 behalf a paid-up, nonexclusive, irrevocable worldwide license in
45 this data to reproduce, prepare derivative works, distribute
46 copies to the public, perform publicly and display publicly, and
47 to permit others to do so. NEITHER THE UNITED STATES GOVERNMENT
48 NOR ANY AGENCY THEREOF, NOR THE UNIVERSITY OF CHICAGO, NOR ANY OF
49 THEIR EMPLOYEES, MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
50 ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE ACCURACY,
51 COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, APPARATUS,
52 PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD
53 NOT INFRINGE PRIVATELY OWNED RIGHTS.
55 ***************************************************************************/
57 #include <stdio.h>
58 #include <stdlib.h>
59 #include "rsl.h"
61 /*@
62 RSL_EXCH_PERIOD - Exchange data on an RSL periodic boundary
64 Notes:
66 See also:
68 @*/
70 RSL_EXCH_PERIOD ( d_p, s_p, dir_p )
71 int_p
72 d_p /* (I) Domain descriptor. */
73 ,s_p /* (I) Period descriptor. */
74 ,dir_p ; /* (I) RSL_M or RSL_N. */
76 int d, s, dir ;
77 period_desc_t *per ;
78 message_desc_t *msg ;
79 rsl_procrec_t *procrec ;
80 rsl_ptrec_t *ptrec ;
81 rsl_list_t *lp, *lp1 ;
82 rsl_index_t ig, jg ;
83 rsl_point_hdr_t point_hdr ;
84 int i, ipt, sp, j ;
85 int curs ;
86 int nprocs, npts ;
87 int retval ;
88 int mtype, mdest ;
89 char * pbuf ;
90 int P ;
91 int Pque[RSL_MAXPROC] ;
92 rsl_procrec_t *procrecque[RSL_MAXPROC ] ;
93 int typeque[RSL_MAXPROC] ;
94 int tqp, ndone ;
95 packrec_t * pr ;
96 void * base ;
98 int ts, te ;
100 d = *d_p ; s = *s_p ;
102 RSL_TEST_ERR(d < 0 || d >= RSL_MAXDOMAINS,
103 "bad domain descriptor" ) ;
104 RSL_TEST_ERR( domain_info[d].valid != RSL_VALID,
105 "descriptor for invalid domain" ) ;
106 if ( *dir_p == RSL_M )
108 dir = 0 ;
110 else if ( *dir_p == RSL_N )
112 dir = 1 ;
114 else
116 RSL_TEST_ERR( 1 , "invalid direction: must be RSL_M or RSL_N" ) ;
120 #ifdef UPSHOT
121 MPE_Log_event( 15, s, "period begin" ) ;
122 #endif
123 #if 0
124 fprintf(stderr,"debug called RSL_EXCH_PERIOD %d\n",s ) ;
125 #endif
127 per = (period_desc_t *) pr_descriptors[ s ] ;
129 /* if period has not been compiled, compile it now! */
130 if ( per->compiled[d] == 0 )
132 rsl_compile_period( d_p, s_p ) ;
135 /* post receives */
136 /* iterate over procrecs for domain and post buffers */
138 tqp = 0 ;
139 for ( procrec = per->procs[dir][d] ; procrec != NULL ; procrec = procrec->next )
141 #if 0
142 fprintf(stderr,"A procrec->P %d\n",procrec->P) ;
143 fprintf(stderr," procrec->npts %d\n",procrec->npts) ;
144 fprintf(stderr," procrec->recv_npts %d\n",procrec->recv_npts) ;
145 fprintf(stderr," procrec->pack_table_size %d\n",procrec->pack_table_size) ;
146 fprintf(stderr," procrec->unpack_table_size %d\n",procrec->unpack_table_size) ;
147 fprintf(stderr," procrec->pack_table_nbytes %d\n",procrec->pack_table_nbytes) ;
148 fprintf(stderr," procrec->unpack_table_nbytes %d\n",procrec->unpack_table_nbytes) ;
149 #endif
150 if ( procrec->unpack_table_nbytes > 0 )
152 P = procrec->P ;
153 Pque[tqp] = P ;
154 procrecque[tqp] = procrec ;
155 pbuf = buffer_for_proc( P, procrec->unpack_table_nbytes, RSL_RECVBUF ) ;
156 mtype = MTYPE_FROMTO( MSG_PERCOM,
157 rsl_c_comp2phys_proc (procrec->P),
158 rsl_myproc ) ;
159 typeque[tqp] = mtype ;
160 procrec->nrecvs++ ; /* diagnostic */
161 #if 0
162 fprintf(stderr,"debug posting async recv for %d bytes from %d\n", procrec->unpack_table_nbytes, rsl_c_comp2phys_proc (procrec->P) ) ;
163 #endif
164 RSL_RECVBEGIN ( pbuf, procrec->unpack_table_nbytes, mtype ) ;
165 tqp++ ;
168 nprocs = tqp ;
170 /* pack buffers and issue sends */
172 for ( procrec = per->procs[dir][d] ; procrec != NULL ; procrec = procrec->next )
174 pbuf=buffer_for_proc(procrec->P, procrec->pack_table_nbytes, RSL_SENDBUF) ;
175 pr = procrec->pack_table ;
176 for ( curs = 0, i = 0 ; i < procrec->pack_table_size ; i++, pr++ )
178 if ( per->has_f90_fields && procrec->pack_table_size > 0 )
179 base = (void *) get_base_for_index ( pr->f90_table_index ) ;
180 else
181 base = pr->base ;
182 for ( j = 0 ; j < pr->nelems ; j++ )
184 #if 0
185 fprintf(stderr,"pck base %08x, %08x, buf %08x, curs %5d, n %5d, off %5d, j %5d, s %5d\n",
186 (char *)(base),
187 (char *)(base) + pr->offset + j * pr->stride,
188 &(pbuf[curs]), curs, pr->n,
189 pr->offset, j, pr->stride ) ;
190 #endif
191 bcopy((char *)(base) + pr->offset + j * pr->stride,
192 &(pbuf[curs]),pr->n) ;
193 curs += pr->n ;
196 if ( curs > 0 )
198 mdest = rsl_c_comp2phys_proc (procrec->P) ;
199 mtype = MTYPE_FROMTO( MSG_PERCOM, rsl_myproc, mdest ) ;
200 procrec->nsends++ ;
201 if ( curs > procrec->pack_table_nbytes )
203 sprintf(mess,"pack buffer overflow %d > %d\n",curs,procrec->pack_table_nbytes) ;
204 RSL_TEST_ERR(1,mess) ;
206 #if 0
207 fprintf(stderr,"debug sending %d bytes to %d\n", curs, mdest ) ;
208 #endif
209 #if 1
210 RSL_SEND ( pbuf, curs, mtype, mdest ) ;
211 #else
214 MPI_Request waitHandle ;
215 MPI_Isend (pbuf,
216 curs,
217 MPI_BYTE,
218 mdest,
219 mtype,
220 rsl_mpi_communicator,
221 &waitHandle);
224 #endif
226 else if ( curs == 0 && procrec->pack_table_nbytes != 0 )
228 RSL_TEST_ERR(1,"internal error") ;
233 /* wait on receives and unpack messages as they come in */
234 ndone = 0 ;
235 tqp = 0 ;
236 retval = 1 ;
238 while( ndone < nprocs )
240 if (tqp >= nprocs ) tqp = 0 ;
241 if (typeque[tqp] != RSL_INVALID)
243 mtype = typeque[tqp] ;
244 if ( rsl_noprobe == NULL )
245 RSL_PROBE ( mtype, &retval ) ;
246 /* else, retval will always be 1 */
248 if ( retval )
250 #ifdef PGON
251 /* on the Paragon, calling RSL_PROBE clears the message so this
252 would bomb on an unknown message id. Don't call unless the probe
253 is disabled (rsl_noprobe != NULL). */
254 if ( rsl_noprobe != NULL ) RSL_RECVEND ( mtype ) ;
255 #else
256 RSL_RECVEND ( mtype ) ;
257 #endif
259 curs = 0 ;
260 pbuf = buffer_for_proc( Pque[tqp], 0, RSL_RECVBUF ) ;
261 procrec = procrecque[tqp] ;
262 pr = procrec->unpack_table ;
263 for ( curs = 0, i = 0 ; i < procrec->unpack_table_size ; i++, pr++ )
265 if ( per->has_f90_fields && procrec->unpack_table_size > 0 )
266 base = (void *) get_base_for_index ( pr->f90_table_index ) ;
267 else
268 base = pr->base ;
269 for ( j = 0 ; j < pr->nelems ; j++ )
271 #if 0
272 fprintf(stderr,"uck base %08x, %08x, buf %08x, curs %5d, n %5d, off %5d, j %5d, s %5d\n",
273 (char *)(base),
274 (char *)(base) + pr->offset + j * pr->stride,
275 &(pbuf[curs]), curs, pr->n,
276 pr->offset, j, pr->stride ) ;
277 #endif
278 bcopy(&(pbuf[curs]),
279 (char *)(base) + pr->offset + j * pr->stride, pr->n) ;
280 curs += pr->n ;
283 if ( curs == 0 )
285 RSL_TEST_ERR(1,"internal error") ;
287 if ( curs > procrec->unpack_table_nbytes )
289 sprintf(mess,"unpack buffer overflow %d > %d\n",curs,procrec->pack_table_nbytes) ;
290 RSL_TEST_ERR(1,mess) ;
292 #if 0
293 fprintf(stderr,"debug got message from %d and unpacked %d bytes\n", Pque[tqp], curs ) ;
294 #endif
295 typeque[tqp] = RSL_INVALID ;
296 ndone++ ;
299 tqp++ ;
301 #ifdef UPSHOT
302 MPE_Log_event( 16, s, "per end" ) ;
303 #endif