src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C

   1 /*---------------------------------------------------------------------------*\
   2   =========                 |
   3   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
   4    \\    /   O peration     |
   5     \\  /    A nd           | Copyright (C) 1991-2009 OpenCFD Ltd.
   6      \\/     M anipulation  |
   7 -------------------------------------------------------------------------------
   8 License
   9     This file is part of OpenFOAM.
  10
  11     OpenFOAM is free software; you can redistribute it and/or modify it
  12     under the terms of the GNU General Public License as published by the
  13     Free Software Foundation; either version 2 of the License, or (at your
  14     option) any later version.
  15
  16     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
  17     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  18     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  19     for more details.
  20
  21     You should have received a copy of the GNU General Public License
  22     along with OpenFOAM; if not, write to the Free Software Foundation,
  23     Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  24
  25 \*---------------------------------------------------------------------------*/
  26
  27 #include "Pstream.H"
  28 #include <OpenFOAM/debug.H>
  29 #include <OpenFOAM/dictionary.H>
  30
  31 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
  32
  33 defineTypeNameAndDebug(Foam::Pstream, 0);
  34
  35 template<>
  36 const char* Foam::NamedEnum<Foam::Pstream::commsTypes, 3>::names[] =
  37 {
  38     "blocking",
  39     "scheduled",
  40     "nonBlocking"
  41 };
  42
  43 const Foam::NamedEnum<Foam::Pstream::commsTypes, 3>
  44     Foam::Pstream::commsTypeNames;
  45
  46 Foam::autoPtr<Foam::PstreamImpl> Foam::Pstream::impl_;
  47
  48 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
  49
  50
  51 void Foam::Pstream::calcLinearComm(const label nProcs)
  52 {
  53     linearCommunication_.setSize(nProcs);
  54
  55     // Master
  56     labelList belowIDs(nProcs - 1);
  57     forAll(belowIDs, i)
  58     {
  59         belowIDs[i] = i + 1;
  60     }
  61
  62     linearCommunication_[0] = commsStruct
  63     (
  64         nProcs,
  65         0,
  66         -1,
  67         belowIDs,
  68         labelList(0)
  69     );
  70
  71     // Slaves. Have no below processors, only communicate up to master
  72     for (label procID = 1; procID < nProcs; procID++)
  73     {
  74         linearCommunication_[procID] = commsStruct
  75         (
  76             nProcs,
  77             procID,
  78             0,
  79             labelList(0),
  80             labelList(0)
  81         );
  82     }
  83 }
  84
  85
  86 // Append my children (and my children children etc.) to allReceives.
  87 void Foam::Pstream::collectReceives
  88 (
  89     const label procID,
  90     const List<DynamicList<label> >& receives,
  91     DynamicList<label>& allReceives
  92 )
  93 {
  94     const DynamicList<label>& myChildren = receives[procID];
  95
  96     forAll(myChildren, childI)
  97     {
  98         allReceives.append(myChildren[childI]);
  99         collectReceives(myChildren[childI], receives, allReceives);
 100     }
 101 }
 102
 103
 104 // Tree like schedule. For 8 procs:
 105 // (level 0)
 106 //      0 receives from 1
 107 //      2 receives from 3
 108 //      4 receives from 5
 109 //      6 receives from 7
 110 // (level 1)
 111 //      0 receives from 2
 112 //      4 receives from 6
 113 // (level 2)
 114 //      0 receives from 4
 115 //
 116 // The sends/receives for all levels are collected per processor (one send per
 117 // processor; multiple receives possible) creating a table:
 118 //
 119 // So per processor:
 120 // proc     receives from   sends to
 121 // ----     -------------   --------
 122 //  0       1,2,4           -
 123 //  1       -               0
 124 //  2       3               0
 125 //  3       -               2
 126 //  4       5               0
 127 //  5       -               4
 128 //  6       7               4
 129 //  7       -               6
 130 void Foam::Pstream::calcTreeComm(label nProcs)
 131 {
 132     label nLevels = 1;
 133     while ((1 << nLevels) < nProcs)
 134     {
 135         nLevels++;
 136     }
 137
 138     List<DynamicList<label> > receives(nProcs);
 139     labelList sends(nProcs, -1);
 140
 141     // Info<< "Using " << nLevels << " communication levels" << endl;
 142
 143     label offset = 2;
 144     label childOffset = offset/2;
 145
 146     for (label level = 0; level < nLevels; level++)
 147     {
 148         label receiveID = 0;
 149         while (receiveID < nProcs)
 150         {
 151             // Determine processor that sends and we receive from
 152             label sendID = receiveID + childOffset;
 153
 154             if (sendID < nProcs)
 155             {
 156                 receives[receiveID].append(sendID);
 157                 sends[sendID] = receiveID;
 158             }
 159
 160             receiveID += offset;
 161         }
 162
 163         offset <<= 1;
 164         childOffset <<= 1;
 165     }
 166
 167     // For all processors find the processors it receives data from
 168     // (and the processors they receive data from etc.)
 169     List<DynamicList<label> > allReceives(nProcs);
 170     for (label procID = 0; procID < nProcs; procID++)
 171     {
 172         collectReceives(procID, receives, allReceives[procID]);
 173     }
 174
 175
 176     treeCommunication_.setSize(nProcs);
 177
 178     for (label procID = 0; procID < nProcs; procID++)
 179     {
 180         treeCommunication_[procID] = commsStruct
 181         (
 182             nProcs,
 183             procID,
 184             sends[procID],
 185             receives[procID].shrink(),
 186             allReceives[procID].shrink()
 187         );
 188     }
 189 }
 190
 191
 192 // Callback from Pstream::init() : initialize linear and tree communication
 193 // schedules now that nProcs is known.
 194 void Foam::Pstream::initCommunicationSchedule()
 195 {
 196     calcLinearComm(nProcs());
 197     calcTreeComm(nProcs());
 198 }
 199
 200
 201 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 202
 203 // Initialise my process number to 0 (the master)
 204 int Foam::Pstream::myProcNo_(0);
 205
 206 // By default this is not a parallel run
 207 bool Foam::Pstream::parRun_(false);
 208
 209 // List of process IDs
 210 Foam::List<int> Foam::Pstream::procIDs_(1, 0);
 211
 212 // Standard transfer message type
 213 const int Foam::Pstream::msgType_(1);
 214
 215 // Linear communication schedule
 216 Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::linearCommunication_(0);
 217
 218 // Multi level communication schedule
 219 Foam::List<Foam::Pstream::commsStruct> Foam::Pstream::treeCommunication_(0);
 220
 221 // Should compact transfer be used in which floats replace doubles
 222 // reducing the bandwidth requirement at the expense of some loss
 223 // in accuracy
 224 bool Foam::Pstream::floatTransfer
 225 (
 226     debug::optimisationSwitch("floatTransfer", 0)
 227 );
 228
 229 // Number of processors at which the reduce algorithm changes from linear to
 230 // tree
 231 int Foam::Pstream::nProcsSimpleSum
 232 (
 233     debug::optimisationSwitch("nProcsSimpleSum", 16)
 234 );
 235
 236 // Default commsType
 237 Foam::Pstream::commsTypes Foam::Pstream::defaultCommsType
 238 (
 239     commsTypeNames.read(debug::optimisationSwitches().lookup("commsType"))
 240 );
 241
 242
 243 // ************************ vim: set sw=4 sts=4 et: ************************ //