wrfv2_fire/external/RSL/RSL/exch_period.c

   1 /***********************************************************************
   2
   3                               COPYRIGHT
   4
   5      The following is a notice of limited availability of the code and
   6      Government license and disclaimer which must be included in the
   7      prologue of the code and in all source listings of the code.
   8
   9      Copyright notice
  10        (c) 1977  University of Chicago
  11
  12      Permission is hereby granted to use, reproduce, prepare
  13      derivative works, and to redistribute to others at no charge.  If
  14      you distribute a copy or copies of the Software, or you modify a
  15      copy or copies of the Software or any portion of it, thus forming
  16      a work based on the Software and make and/or distribute copies of
  17      such work, you must meet the following conditions:
  18
  19           a) If you make a copy of the Software (modified or verbatim)
  20              it must include the copyright notice and Government
  21              license and disclaimer.
  22
  23           b) You must cause the modified Software to carry prominent
  24              notices stating that you changed specified portions of
  25              the Software.
  26
  27      This software was authored by:
  28
  29      Argonne National Laboratory
  30      J. Michalakes: (630) 252-6646; email: michalak@mcs.anl.gov
  31      Mathematics and Computer Science Division
  32      Argonne National Laboratory, Argonne, IL  60439
  33
  34      ARGONNE NATIONAL LABORATORY (ANL), WITH FACILITIES IN THE STATES
  35      OF ILLINOIS AND IDAHO, IS OWNED BY THE UNITED STATES GOVERNMENT,
  36      AND OPERATED BY THE UNIVERSITY OF CHICAGO UNDER PROVISION OF A
  37      CONTRACT WITH THE DEPARTMENT OF ENERGY.
  38
  39                       GOVERNMENT LICENSE AND DISCLAIMER
  40
  41      This computer code material was prepared, in part, as an account
  42      of work sponsored by an agency of the United States Government.
  43      The Government is granted for itself and others acting on its
  44      behalf a paid-up, nonexclusive, irrevocable worldwide license in
  45      this data to reproduce, prepare derivative works, distribute
  46      copies to the public, perform publicly and display publicly, and
  47      to permit others to do so.  NEITHER THE UNITED STATES GOVERNMENT
  48      NOR ANY AGENCY THEREOF, NOR THE UNIVERSITY OF CHICAGO, NOR ANY OF
  49      THEIR EMPLOYEES, MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR
  50      ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE ACCURACY,
  51      COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, APPARATUS,
  52      PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD
  53      NOT INFRINGE PRIVATELY OWNED RIGHTS.
  54
  55 ***************************************************************************/
  56
  57 #include <stdio.h>
  58 #include <stdlib.h>
  59 #include "rsl.h"
  60
  61 /*@
  62   RSL_EXCH_PERIOD - Exchange data on an RSL periodic boundary
  63
  64   Notes:
  65
  66   See also:
  67
  68 @*/
  69
  70 RSL_EXCH_PERIOD ( d_p, s_p, dir_p )
  71   int_p
  72     d_p             /* (I) Domain descriptor. */
  73    ,s_p             /* (I) Period descriptor. */
  74    ,dir_p ;         /* (I) RSL_M or RSL_N. */
  75 {
  76   int d, s, dir ;
  77   period_desc_t *per ;
  78   message_desc_t *msg ;
  79   rsl_procrec_t *procrec ;
  80   rsl_ptrec_t *ptrec ;
  81   rsl_list_t *lp, *lp1 ;
  82   rsl_index_t ig, jg ;
  83   rsl_point_hdr_t point_hdr ;
  84   int i, ipt, sp, j ;
  85   int curs ;
  86   int nprocs, npts ;
  87   int retval ;
  88   int mtype, mdest ;
  89   char * pbuf ;
  90   int P ;
  91   int Pque[RSL_MAXPROC] ;
  92   rsl_procrec_t *procrecque[RSL_MAXPROC ] ;
  93   int typeque[RSL_MAXPROC] ;
  94   int tqp, ndone ;
  95   packrec_t * pr ;
  96   void * base ;
  97
  98 int ts, te ;
  99
 100   d = *d_p ; s = *s_p ;
 101
 102   RSL_TEST_ERR(d < 0 || d >= RSL_MAXDOMAINS,
 103           "bad domain descriptor" ) ;
 104   RSL_TEST_ERR( domain_info[d].valid != RSL_VALID,
 105         "descriptor for invalid domain" ) ;
 106   if      ( *dir_p == RSL_M )
 107   {
 108     dir = 0 ;
 109   }
 110   else if ( *dir_p == RSL_N )
 111   {
 112     dir = 1 ;
 113   }
 114   else
 115   {
 116     RSL_TEST_ERR( 1 , "invalid direction: must be RSL_M or RSL_N" ) ;
 117   }
 118
 119
 120 #ifdef UPSHOT
 121 MPE_Log_event( 15, s, "period begin" ) ;
 122 #endif
 123 #if 0
 124 fprintf(stderr,"debug called RSL_EXCH_PERIOD %d\n",s ) ;
 125 #endif
 126
 127   per = (period_desc_t *) pr_descriptors[ s ] ;
 128
 129   /* if period has not been compiled, compile it now! */
 130   if ( per->compiled[d] == 0 )
 131   {
 132     rsl_compile_period( d_p, s_p ) ;
 133   }
 134
 135   /* post receives */
 136   /* iterate over procrecs for domain and post buffers */
 137
 138   tqp = 0 ;
 139   for ( procrec = per->procs[dir][d] ; procrec != NULL ; procrec = procrec->next )
 140   {
 141 #if 0
 142  fprintf(stderr,"A procrec->P %d\n",procrec->P) ;
 143  fprintf(stderr,"  procrec->npts %d\n",procrec->npts) ;
 144  fprintf(stderr,"  procrec->recv_npts %d\n",procrec->recv_npts) ;
 145  fprintf(stderr,"  procrec->pack_table_size %d\n",procrec->pack_table_size) ;
 146  fprintf(stderr,"  procrec->unpack_table_size %d\n",procrec->unpack_table_size) ;
 147  fprintf(stderr,"  procrec->pack_table_nbytes %d\n",procrec->pack_table_nbytes) ;
 148  fprintf(stderr,"  procrec->unpack_table_nbytes %d\n",procrec->unpack_table_nbytes) ;
 149 #endif
 150     if ( procrec->unpack_table_nbytes > 0 )
 151     {
 152       P = procrec->P ;
 153       Pque[tqp] = P ;
 154       procrecque[tqp] = procrec ;
 155       pbuf = buffer_for_proc( P, procrec->unpack_table_nbytes, RSL_RECVBUF ) ;
 156       mtype = MTYPE_FROMTO( MSG_PERCOM,
 157                             rsl_c_comp2phys_proc (procrec->P),
 158                             rsl_myproc ) ;
 159       typeque[tqp] = mtype ;
 160       procrec->nrecvs++ ; /* diagnostic */
 161 #if 0
 162 fprintf(stderr,"debug posting async recv for %d bytes from %d\n", procrec->unpack_table_nbytes, rsl_c_comp2phys_proc (procrec->P) ) ;
 163 #endif
 164       RSL_RECVBEGIN ( pbuf, procrec->unpack_table_nbytes, mtype ) ;
 165       tqp++ ;
 166     }
 167   }
 168   nprocs = tqp ;
 169
 170   /* pack buffers and issue sends */
 171
 172   for ( procrec = per->procs[dir][d] ; procrec != NULL ; procrec = procrec->next )
 173   {
 174     pbuf=buffer_for_proc(procrec->P, procrec->pack_table_nbytes, RSL_SENDBUF) ;
 175     pr = procrec->pack_table ;
 176     for ( curs = 0, i = 0 ; i < procrec->pack_table_size ; i++, pr++ )
 177     {
 178       if ( per->has_f90_fields && procrec->pack_table_size > 0 )
 179         base = (void *) get_base_for_index ( pr->f90_table_index ) ;
 180       else
 181         base = pr->base ;
 182       for ( j = 0 ; j < pr->nelems ; j++ )
 183       {
 184 #if 0
 185 fprintf(stderr,"pck base %08x, %08x, buf %08x, curs %5d, n %5d, off %5d, j %5d, s %5d\n",
 186 (char *)(base),
 187 (char *)(base) + pr->offset + j * pr->stride,
 188 &(pbuf[curs]), curs, pr->n,
 189 pr->offset, j, pr->stride ) ;
 190 #endif
 191         bcopy((char *)(base) + pr->offset + j * pr->stride,
 192               &(pbuf[curs]),pr->n) ;
 193         curs += pr->n ;
 194       }
 195     }
 196     if ( curs > 0 )
 197     {
 198       mdest = rsl_c_comp2phys_proc (procrec->P) ;
 199       mtype = MTYPE_FROMTO( MSG_PERCOM, rsl_myproc, mdest ) ;
 200       procrec->nsends++ ;
 201       if ( curs > procrec->pack_table_nbytes )
 202       {
 203         sprintf(mess,"pack buffer overflow %d > %d\n",curs,procrec->pack_table_nbytes) ;
 204         RSL_TEST_ERR(1,mess) ;
 205       }
 206 #if 0
 207 fprintf(stderr,"debug sending %d bytes to %d\n", curs, mdest ) ;
 208 #endif
 209 #if 1
 210       RSL_SEND ( pbuf, curs, mtype, mdest ) ;
 211 #else
 212
 213 {
 214    MPI_Request waitHandle ;
 215    MPI_Isend (pbuf,
 216               curs,
 217               MPI_BYTE,
 218               mdest,
 219               mtype,
 220               rsl_mpi_communicator,
 221               &waitHandle);
 222 }
 223
 224 #endif
 225     }
 226     else if ( curs == 0 && procrec->pack_table_nbytes != 0 )
 227     {
 228       RSL_TEST_ERR(1,"internal error") ;
 229     }
 230   }
 231
 232
 233   /* wait on receives and unpack messages as they come in */
 234   ndone = 0 ;
 235   tqp = 0 ;
 236   retval = 1 ;
 237
 238   while( ndone < nprocs )
 239   {
 240     if (tqp >= nprocs ) tqp = 0 ;
 241     if (typeque[tqp] != RSL_INVALID)
 242     {
 243       mtype = typeque[tqp] ;
 244       if ( rsl_noprobe == NULL )
 245         RSL_PROBE ( mtype, &retval ) ;
 246       /* else, retval will always be 1 */
 247
 248       if ( retval )
 249       {
 250 #ifdef PGON
 251 /* on the Paragon, calling RSL_PROBE clears the message so this
 252    would bomb on an unknown message id.  Don't call unless the probe
 253    is disabled (rsl_noprobe != NULL). */
 254         if ( rsl_noprobe != NULL ) RSL_RECVEND ( mtype ) ;
 255 #else
 256         RSL_RECVEND ( mtype ) ;
 257 #endif
 258
 259         curs = 0 ;
 260         pbuf = buffer_for_proc( Pque[tqp], 0, RSL_RECVBUF ) ;
 261         procrec = procrecque[tqp] ;
 262         pr = procrec->unpack_table ;
 263         for ( curs = 0, i = 0 ; i < procrec->unpack_table_size ; i++, pr++ )
 264         {
 265           if ( per->has_f90_fields && procrec->unpack_table_size > 0 )
 266             base = (void *) get_base_for_index ( pr->f90_table_index ) ;
 267           else
 268             base = pr->base ;
 269           for ( j = 0 ; j < pr->nelems ; j++ )
 270           {
 271 #if 0
 272 fprintf(stderr,"uck base %08x, %08x, buf %08x, curs %5d, n %5d, off %5d, j %5d, s %5d\n",
 273 (char *)(base),
 274 (char *)(base) + pr->offset + j * pr->stride,
 275 &(pbuf[curs]), curs, pr->n,
 276 pr->offset, j, pr->stride ) ;
 277 #endif
 278             bcopy(&(pbuf[curs]),
 279                  (char *)(base) + pr->offset + j * pr->stride, pr->n) ;
 280             curs += pr->n ;
 281           }
 282         }
 283         if ( curs == 0 )
 284         {
 285           RSL_TEST_ERR(1,"internal error") ;
 286         }
 287         if ( curs > procrec->unpack_table_nbytes )
 288         {
 289           sprintf(mess,"unpack buffer overflow %d > %d\n",curs,procrec->pack_table_nbytes) ;
 290           RSL_TEST_ERR(1,mess) ;
 291         }
 292 #if 0
 293 fprintf(stderr,"debug got message from %d and unpacked %d bytes\n", Pque[tqp], curs ) ;
 294 #endif
 295         typeque[tqp] = RSL_INVALID ;
 296         ndone++ ;
 297       }
 298     }
 299     tqp++ ;
 300   }
 301 #ifdef UPSHOT
 302 MPE_Log_event( 16, s, "per end" ) ;
 303 #endif
 304
 305 }
 306