Merge branch 'master' of ssh://opencfd@repo.or.cz/srv/git/OpenFOAM-1.5.x
[OpenFOAM-1.5.x.git] / applications / utilities / parallelProcessing / redistributeMeshPar / redistributeMeshPar.C
blob3c2f7f66627c3541fddc3b79450a3376d36a8ab1
1 /*---------------------------------------------------------------------------*\
2   =========                 |
3   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
4    \\    /   O peration     |
5     \\  /    A nd           | Copyright (C) 1991-2008 OpenCFD Ltd.
6      \\/     M anipulation  |
7 -------------------------------------------------------------------------------
8 License
9     This file is part of OpenFOAM.
11     OpenFOAM is free software; you can redistribute it and/or modify it
12     under the terms of the GNU General Public License as published by the
13     Free Software Foundation; either version 2 of the License, or (at your
14     option) any later version.
16     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
19     for more details.
21     You should have received a copy of the GNU General Public License
22     along with OpenFOAM; if not, write to the Free Software Foundation,
23     Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
25 Application
26     redistributeMeshPar
28 Description
29     Parallel redecomposition of mesh.
31 \*---------------------------------------------------------------------------*/
33 #include "Field.H"
34 #include "fvMesh.H"
35 #include "decompositionMethod.H"
36 #include "PstreamReduceOps.H"
37 #include "fvCFD.H"
38 #include "fvMeshDistribute.H"
39 #include "mapDistributePolyMesh.H"
40 #include "IOobjectList.H"
42 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
44 // Tolerance (as fraction of the bounding box). Needs to be fairly lax since
45 // usually meshes get written with limited precision (6 digits)
46 static const scalar defaultMergeTol = 1E-6;
49 // Read mesh if available. Otherwise create empty mesh with same non-proc
50 // patches as proc0 mesh. Requires all processors to have all patches
51 // (and in same order).
52 autoPtr<fvMesh> createMesh
54     const Time& runTime,
55     const fileName& instDir,
56     const bool haveMesh
59     Pout<< "Create mesh for time = "
60         << runTime.timeName() << nl << endl;
62     // Create dummy mesh. Only used on procs that don't have mesh.
63     // Note constructed on all processors since does parallel comms.
64     fvMesh dummyMesh
65     (
66         IOobject
67         (
68             fvMesh::defaultRegion,
69             instDir,
70             runTime,
71             IOobject::MUST_READ
72         ),
73         pointField(0),
74         faceList(0),
75         labelList(0),
76         labelList(0)
77     );
79     if (!haveMesh)
80     {
81         Pout<< "Writing dummy mesh to " << runTime.path()/instDir << endl;
82         dummyMesh.write();
83     }
85     Pout<< "Reading mesh from " << runTime.path()/instDir << endl;
86     autoPtr<fvMesh> meshPtr
87     (
88         new fvMesh
89         (
90             IOobject
91             (
92                 fvMesh::defaultRegion,
93                 instDir,
94                 runTime,
95                 IOobject::MUST_READ
96             )
97         )
98     );
99     fvMesh& mesh = meshPtr();
102     // Determine patches.
103     if (Pstream::master())
104     {
105         // Send patches
106         for
107         (
108             int slave=Pstream::firstSlave();
109             slave<=Pstream::lastSlave();
110             slave++
111         )
112         {
113             OPstream toSlave(Pstream::blocking, slave);
114             toSlave << mesh.boundaryMesh();
115         }
116     }
117     else
118     {
119         // Receive patches
120         IPstream fromMaster(Pstream::blocking, Pstream::masterNo());
121         PtrList<entry> patchEntries(fromMaster);
123         if (haveMesh)
124         {
125             // Check master names against mine
127             const polyBoundaryMesh& patches = mesh.boundaryMesh();
129             forAll(patchEntries, patchI)
130             {
131                 const entry& e = patchEntries[patchI];
132                 const word type(e.dict().lookup("type"));
133                 const word& name = e.keyword();
135                 if (type == processorPolyPatch::typeName)
136                 {
137                     break;
138                 }
140                 if (patchI >= patches.size())
141                 {
142                     FatalErrorIn
143                     (
144                         "createMesh(const Time&, const fileName&, const bool)"
145                     )   << "Non-processor patches not synchronised."
146                         << endl
147                         << "Processor " << Pstream::myProcNo()
148                         << " has only " << patches.size()
149                         << " patches, master has "
150                         << patchI
151                         << exit(FatalError);
152                 }
154                 if
155                 (
156                     type != patches[patchI].type()
157                  || name != patches[patchI].name()
158                 )
159                 {
160                     FatalErrorIn
161                     (
162                         "createMesh(const Time&, const fileName&, const bool)"
163                     )   << "Non-processor patches not synchronised."
164                         << endl
165                         << "Master patch " << patchI
166                         << " name:" << type
167                         << " type:" << type << endl
168                         << "Processor " << Pstream::myProcNo()
169                         << " patch " << patchI
170                         << " has name:" << patches[patchI].name()
171                         << " type:" << patches[patchI].type()
172                         << exit(FatalError);
173                 }
174             }
175         }
176         else
177         {
178             // Add patch
179             List<polyPatch*> patches(patchEntries.size());
180             label nPatches = 0;
182             forAll(patchEntries, patchI)
183             {
184                 const entry& e = patchEntries[patchI];
185                 const word type(e.dict().lookup("type"));
186                 const word& name = e.keyword();
188                 if (type == processorPolyPatch::typeName)
189                 {
190                     break;
191                 }
193                 Pout<< "Adding patch:" << nPatches
194                     << " name:" << name
195                     << " type:" << type << endl;
197                 dictionary patchDict(e.dict());
198                 patchDict.remove("nFaces");
199                 patchDict.add("nFaces", 0);
200                 patchDict.remove("startFace");
201                 patchDict.add("startFace", 0);
203                 patches[patchI] = polyPatch::New
204                 (
205                     name,
206                     patchDict,
207                     nPatches++,
208                     mesh.boundaryMesh()
209                 ).ptr();
210             }
211             patches.setSize(nPatches);
212             mesh.addFvPatches(patches, false);  // no parallel comms
214             //// Write empty mesh now we have correct patches
215             //meshPtr().write();
216         }
217     }
219     if (!haveMesh)
220     {
221         // We created a dummy mesh file above. Delete it.
222         Pout<< "Removing dummy mesh in " << runTime.path()/instDir << endl;
223         rmDir(runTime.path()/instDir/polyMesh::meshSubDir);
224     }
226     // Force recreation of globalMeshData.
227     mesh.clearOut();
228     mesh.globalData();
230     return meshPtr;
234 // Get merging distance when matching face centres
235 scalar getMergeDistance
237     const argList& args,
238     const Time& runTime,
239     const boundBox& bb
242     scalar mergeTol = defaultMergeTol;
243     if (args.options().found("mergeTol"))
244     {
245         mergeTol = readScalar(IStringStream(args.options()["mergeTol"])());
246     }
247     scalar writeTol = 
248         Foam::pow(scalar(10.0), -scalar(IOstream::defaultPrecision()));
250     Info<< "Merge tolerance : " << mergeTol << nl
251         << "Write tolerance : " << writeTol << endl;
253     if (runTime.writeFormat() == IOstream::ASCII && mergeTol < writeTol)
254     {
255         FatalErrorIn("getMergeDistance")
256             << "Your current settings specify ASCII writing with "
257             << IOstream::defaultPrecision() << " digits precision." << endl
258             << "Your merging tolerance (" << mergeTol << ") is finer than this."
259             << endl
260             << "Please change your writeFormat to binary"
261             << " or increase the writePrecision" << endl
262             << "or adjust the merge tolerance (-mergeTol)."
263             << exit(FatalError);
264     }
266     scalar mergeDist = mergeTol*mag(bb.max() - bb.min());
268     Info<< "Overall meshes bounding box : " << bb << nl
269         << "Relative tolerance          : " << mergeTol << nl
270         << "Absolute matching distance  : " << mergeDist << nl
271         << endl;
273     return mergeDist;
277 void printMeshData(Ostream& os, const polyMesh& mesh)
279     os  << "Number of points:           "
280         << mesh.points().size() << nl
281         << "          edges:            "
282         << mesh.edges().size() << nl
283         << "          faces:            "
284         << mesh.faces().size() << nl
285         << "          internal faces:   "
286         << mesh.faceNeighbour().size() << nl
287         << "          cells:            "
288         << mesh.cells().size() << nl
289         << "          boundary patches: "
290         << mesh.boundaryMesh().size() << nl
291         << "          point zones:      "
292         << mesh.pointZones().size() << nl
294         << "          face zones:       "
295         << mesh.faceZones().size() << nl
297         << "          cell zones:       "
298         << mesh.cellZones().size() << nl;
302 // Debugging: write volScalarField with decomposition for post processing.
303 void writeDecomposition
305     const word& name,
306     const fvMesh& mesh,
307     const labelList& decomp
310     Info<< "Writing wanted cell distribution to volScalarField " << name
311         << " for postprocessing purposes." << nl << endl;    
313     volScalarField procCells
314     (
315         IOobject
316         (
317             name,
318             mesh.time().timeName(),
319             mesh,
320             IOobject::NO_READ,
321             IOobject::AUTO_WRITE,
322             false                   // do not register
323         ),
324         mesh,
325         dimensionedScalar(name, dimless, -1),
326         zeroGradientFvPatchScalarField::typeName
327     );
329     forAll(procCells, cI)
330     {
331         procCells[cI] = decomp[cI];
332     }
333     procCells.write();
337 // Read vol or surface fields
338 //template<class T, class Mesh>
339 template<class GeoField>
340 void readFields
342     const boolList& haveMesh,
343     const fvMesh& mesh,
344     const autoPtr<fvMeshSubset>& subsetterPtr,
345     IOobjectList& allObjects,
346     PtrList<GeoField>& fields
349     //typedef GeometricField<T, fvPatchField, Mesh> fldType;
351     // Get my objects of type
352     IOobjectList objects(allObjects.lookupClass(GeoField::typeName));
353     // Check that we all have all objects
354     wordList objectNames = objects.toc();
355     // Get master names
356     wordList masterNames(objectNames);
357     Pstream::scatter(masterNames);
359     if (haveMesh[Pstream::myProcNo()] && objectNames != masterNames)
360     {
361         FatalErrorIn("readFields()")
362             << "differing fields of type " << GeoField::typeName
363             << " on processors." << endl
364             << "Master has:" << masterNames << endl
365             << Pstream::myProcNo() << " has:" << objectNames
366             << abort(FatalError);
367     }
369     fields.setSize(masterNames.size());
371     // Have master send all fields to processors that don't have a mesh
372     if (Pstream::master())
373     {
374         forAll(masterNames, i)
375         {
376             const word& name = masterNames[i];
377             IOobject& io = *objects[name];
378             io.writeOpt() = IOobject::AUTO_WRITE;
380             // Load field
381             fields.set(i, new GeoField(io, mesh));
383             // Create zero sized field and send
384             if (subsetterPtr.valid())
385             {
386                 tmp<GeoField> tsubfld = subsetterPtr().interpolate(fields[i]);
388                 // Send to all processors that don't have a mesh
389                 for (label procI = 1; procI < Pstream::nProcs(); procI++)
390                 {
391                     if (!haveMesh[procI])
392                     {
393                         OPstream toProc(Pstream::blocking, procI);
394                         toProc<< tsubfld();
395                     }
396                 }
397             }
398         }
399     }
400     else if (!haveMesh[Pstream::myProcNo()])
401     {
402         // Don't have mesh (nor fields). Receive empty field from master.
404         forAll(masterNames, i)
405         {
406             const word& name = masterNames[i];
408             // Receive field
409             IPstream fromMaster(Pstream::blocking, Pstream::masterNo());
411             fields.set
412             (
413                 i,
414                 new GeoField
415                 (
416                     IOobject
417                     (
418                         name,
419                         mesh.time().timeName(),
420                         mesh,
421                         IOobject::NO_READ,
422                         IOobject::AUTO_WRITE
423                     ),
424                     mesh,
425                     fromMaster
426                 )
427             );
429             //// Write it for next time round (since mesh gets written as well)
430             //fields[i].write();
431         }
432     }
433     else
434     {
435         // Have mesh so just try to load
436         forAll(masterNames, i)
437         {
438             const word& name = masterNames[i];
439             IOobject& io = *objects[name];
440             io.writeOpt() = IOobject::AUTO_WRITE;
442             // Load field
443             fields.set(i, new GeoField(io, mesh));
444         }
445     }
449 // Debugging: compare two fields.
450 void compareFields
452     const scalar tolDim,
453     const volVectorField& a,
454     const volVectorField& b
457     forAll(a, cellI)
458     {
459         if (mag(b[cellI] - a[cellI]) > tolDim)
460         {
461             FatalErrorIn
462             (
463                 "compareFields"
464                 "(const scalar, const volVectorField&, const volVectorField&)"
465             )   << "Did not map volVectorField correctly:" << nl
466                 << "cell:" << cellI
467                 << " transfer b:" << b[cellI]
468                 << " real cc:" << a[cellI]
469                 << abort(FatalError);
470         }
471     }
472     forAll(a.boundaryField(), patchI)
473     {
474         // We have real mesh cellcentre and
475         // mapped original cell centre.
477         const fvPatchVectorField& aBoundary =
478             a.boundaryField()[patchI];
480         const fvPatchVectorField& bBoundary =
481             b.boundaryField()[patchI];
483         if (!bBoundary.coupled())
484         {
485             forAll(aBoundary, i)
486             {
487                 if (mag(aBoundary[i] - bBoundary[i]) > tolDim)
488                 {
489                     FatalErrorIn
490                     (
491                         "compareFields"
492                         "(const scalar, const volVectorField&"
493                         ", const volVectorField&)"
494                     )   << "Did not map volVectorField correctly:"
495                         << endl
496                         << "patch:" << patchI << " patchFace:" << i
497                         << " cc:" << endl
498                         << "    real    :" << aBoundary[i] << endl
499                         << "    mapped  :" << bBoundary[i] << endl
500                         << abort(FatalError);
501                 }
502             }
503         }
504     }
508 // Main program:
510 int main(int argc, char *argv[])
512     argList::validOptions.insert("mergeTol", "relative merge distance");
513 #   include "setRootCase.H"
515     // Create processor directory if non-existing
516     if (!Pstream::master() && !dir(args.path()))
517     {
518         Pout<< "Creating case directory " << args.path() << endl;
519         mkDir(args.path());
520     }
522 #   include "createTime.H"
524     // Get time instance directory. Since not all processors have meshes
525     // just use the master one everywhere.
527     fileName masterInstDir;
528     if (Pstream::master())
529     {
530         masterInstDir = runTime.findInstance(polyMesh::meshSubDir, "points");
531     }
532     Pstream::scatter(masterInstDir);
534     // Check who has a mesh
535     const fileName meshDir = runTime.path()/masterInstDir/polyMesh::meshSubDir;
537     boolList haveMesh(Pstream::nProcs(), false);
538     haveMesh[Pstream::myProcNo()] = dir(meshDir);
539     Pstream::gatherList(haveMesh);
540     Pstream::scatterList(haveMesh);
541     Info<< "Per processor mesh availability : " << haveMesh << endl;
542     const bool allHaveMesh = (findIndex(haveMesh, false) == -1);
544     // Create mesh
545     autoPtr<fvMesh> meshPtr = createMesh
546     (
547         runTime,
548         masterInstDir,
549         haveMesh[Pstream::myProcNo()]
550     );
551     fvMesh& mesh = meshPtr();
553     Pout<< "Read mesh:" << endl;
554     printMeshData(Pout, mesh);
555     Pout<< endl;
557     IOdictionary decompositionDict
558     (
559         IOobject
560         (
561             "decomposeParDict",
562             runTime.system(),
563             mesh,
564             IOobject::MUST_READ,
565             IOobject::NO_WRITE
566         )
567     );
569     labelList finalDecomp;
571     // Create decompositionMethod and new decomposition
572     {
573         autoPtr<decompositionMethod> decomposer
574         (
575             decompositionMethod::New
576             (
577                 decompositionDict,
578                 mesh
579             )
580         );
582         if (!decomposer().parallelAware())
583         {
584             WarningIn(args.executable())
585                 << "You have selected decomposition method "
586                 << decomposer().typeName
587                 << " which does" << endl
588                 << "not synchronise the decomposition across"
589                 << " processor patches." << endl
590                 << "    You might want to select a decomposition method which"
591                 << " is aware of this. Continuing."
592                 << endl;
593         }
595         finalDecomp = decomposer().decompose(mesh.cellCentres());
596     }
598     // Dump decomposition to volScalarField
599     writeDecomposition("decomposition", mesh, finalDecomp);
602     // Create 0 sized mesh to do all the generation of zero sized
603     // fields on processors that have zero sized meshes. Note that this is
604     // only nessecary on master but since polyMesh construction with
605     // Pstream::parRun does parallel comms we have to do it on all
606     // processors
607     autoPtr<fvMeshSubset> subsetterPtr;
609     if (!allHaveMesh)
610     {
611         // Find last non-processor patch.
612         const polyBoundaryMesh& patches = mesh.boundaryMesh();
614         label nonProcI = -1;
616         forAll(patches, patchI)
617         {
618             if (isA<processorPolyPatch>(patches[patchI]))
619             {
620                 break;
621             }
622             nonProcI++;
623         }
625         if (nonProcI == -1)
626         {
627             FatalErrorIn(args.executable())
628                 << "Cannot find non-processor patch on processor "
629                 << Pstream::myProcNo() << endl
630                 << " Current patches:" << patches.names() << abort(FatalError);
631         }
633         // Subset 0 cells, no parallel comms. This is used to create zero-sized
634         // fields.
635         subsetterPtr.reset(new fvMeshSubset(mesh));
636         subsetterPtr().setLargeCellSubset(labelHashSet(0), nonProcI, false);
637     }
640     // Get original objects (before incrementing time!)
641     IOobjectList objects(mesh, runTime.timeName());
642     // We don't want to map the decomposition (mapping already tested when
643     // mapping the cell centre field)
644     IOobjectList::iterator iter = objects.find("decomposition");
645     if (iter != objects.end())
646     {
647         objects.erase(iter);
648     }
650     PtrList<volScalarField> volScalarFields;
651     readFields
652     (
653         haveMesh,
654         mesh,
655         subsetterPtr,
656         objects,
657         volScalarFields
658     );
660     PtrList<volVectorField> volVectorFields;
661     readFields
662     (
663         haveMesh,
664         mesh,
665         subsetterPtr,
666         objects,
667         volVectorFields
668     );
670     PtrList<volSphericalTensorField> volSphereTensorFields;
671     readFields
672     (
673         haveMesh,
674         mesh,
675         subsetterPtr,
676         objects,
677         volSphereTensorFields
678     );
680     PtrList<volSymmTensorField> volSymmTensorFields;
681     readFields
682     (
683         haveMesh,
684         mesh,
685         subsetterPtr,
686         objects,
687         volSymmTensorFields
688     );
690     PtrList<volTensorField> volTensorFields;
691     readFields
692     (
693         haveMesh,
694         mesh,
695         subsetterPtr,
696         objects,
697         volTensorFields
698     );
700     PtrList<surfaceScalarField> surfScalarFields;
701     readFields
702     (
703         haveMesh,
704         mesh,
705         subsetterPtr,
706         objects,
707         surfScalarFields
708     );
710     PtrList<surfaceVectorField> surfVectorFields;
711     readFields
712     (
713         haveMesh,
714         mesh,
715         subsetterPtr,
716         objects,
717         surfVectorFields
718     );
720     PtrList<surfaceSphericalTensorField> surfSphereTensorFields;
721     readFields
722     (
723         haveMesh,
724         mesh,
725         subsetterPtr,
726         objects,
727         surfSphereTensorFields
728     );
730     PtrList<surfaceSymmTensorField> surfSymmTensorFields;
731     readFields
732     (
733         haveMesh,
734         mesh,
735         subsetterPtr,
736         objects,
737         surfSymmTensorFields
738     );
740     PtrList<surfaceTensorField> surfTensorFields;
741     readFields
742     (
743         haveMesh,
744         mesh,
745         subsetterPtr,
746         objects,
747         surfTensorFields
748     );
751     // Debugging: Create additional volField that will be mapped.
752     // Used to test correctness of mapping
753     volVectorField mapCc("mapCc", 1*mesh.C());
755     // Global matching tolerance
756     const scalar tolDim = getMergeDistance
757     (
758         args,
759         runTime,
760         mesh.globalData().bb()
761     );
763     // Mesh distribution engine
764     fvMeshDistribute distributor(mesh, tolDim);
766     Pout<< "Wanted distribution:"
767         << distributor.countCells(finalDecomp) << nl << endl;
769     // Do actual sending/receiving of mesh
770     autoPtr<mapDistributePolyMesh> map = distributor.distribute(finalDecomp);
772     //// Distribute any non-registered data accordingly
773     //map().distributeFaceData(faceCc);
776     // Print a bit
777     Pout<< "After distribution mesh:" << endl;
778     printMeshData(Pout, mesh);
779     Pout<< endl;
781     runTime++;
782     Pout<< "Writing redistributed mesh to " << runTime.timeName() << nl << endl;
783     mesh.write();
786     // Debugging: test mapped cellcentre field.
787     compareFields(tolDim, mesh.C(), mapCc);
789     // Print nice message
790     // ~~~~~~~~~~~~~~~~~~
792     // Determine which processors remain so we can print nice final message.
793     labelList nFaces(Pstream::nProcs());
794     nFaces[Pstream::myProcNo()] = mesh.nFaces();
795     Pstream::gatherList(nFaces);
796     Pstream::scatterList(nFaces);
798     Info<< nl
799         << "You can pick up the redecomposed mesh from the polyMesh directory"
800         << " in " << runTime.timeName() << "." << nl
801         << "If you redecomposed the mesh to less processors you can delete"
802         << nl
803         << "the processor directories with 0 sized meshes in them." << nl
804         << "Below is a sample set of commands to do this."
805         << " Take care when issueing these" << nl
806         << "commands." << nl << endl;
808     forAll(nFaces, procI)
809     {
810         fileName procDir = "processor" + name(procI);
812         if (nFaces[procI] == 0)
813         {
814             Info<< "    rm -r " << procDir.c_str() << nl;
815         }
816         else
817         {
818             fileName timeDir = procDir/runTime.timeName()/polyMesh::meshSubDir;
819             fileName constDir = procDir/runTime.constant()/polyMesh::meshSubDir;
821             Info<< "    rm -r " << constDir.c_str() << nl
822                 << "    mv " << timeDir.c_str()
823                 << ' ' << constDir.c_str() << nl;
824         }
825     }
826     Info<< endl;
829     Pout<< "End\n" << endl;
831     return 0;
835 // ************************************************************************* //