1 /*---------------------------------------------------------------------------*\
3 \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
5 \\ / A nd | Copyright (C) 1991-2008 OpenCFD Ltd.
7 -------------------------------------------------------------------------------
9 This file is part of OpenFOAM.
11 OpenFOAM is free software; you can redistribute it and/or modify it
12 under the terms of the GNU General Public License as published by the
13 Free Software Foundation; either version 2 of the License, or (at your
14 option) any later version.
16 OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with OpenFOAM; if not, write to the Free Software Foundation,
23 Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
29 Parallel redecomposition of mesh.
31 \*---------------------------------------------------------------------------*/
35 #include "decompositionMethod.H"
36 #include "PstreamReduceOps.H"
38 #include "fvMeshDistribute.H"
39 #include "mapDistributePolyMesh.H"
40 #include "IOobjectList.H"
42 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
44 // Tolerance (as fraction of the bounding box). Needs to be fairly lax since
45 // usually meshes get written with limited precision (6 digits)
46 static const scalar defaultMergeTol = 1E-6;
49 // Read mesh if available. Otherwise create empty mesh with same non-proc
50 // patches as proc0 mesh. Requires all processors to have all patches
51 // (and in same order).
52 autoPtr<fvMesh> createMesh
55 const fileName& instDir,
59 Pout<< "Create mesh for time = "
60 << runTime.timeName() << nl << endl;
62 // Create dummy mesh. Only used on procs that don't have mesh.
63 // Note constructed on all processors since does parallel comms.
68 fvMesh::defaultRegion,
81 Pout<< "Writing dummy mesh to " << runTime.path()/instDir << endl;
85 Pout<< "Reading mesh from " << runTime.path()/instDir << endl;
86 autoPtr<fvMesh> meshPtr
92 fvMesh::defaultRegion,
99 fvMesh& mesh = meshPtr();
102 // Determine patches.
103 if (Pstream::master())
108 int slave=Pstream::firstSlave();
109 slave<=Pstream::lastSlave();
113 OPstream toSlave(Pstream::blocking, slave);
114 toSlave << mesh.boundaryMesh();
120 IPstream fromMaster(Pstream::blocking, Pstream::masterNo());
121 PtrList<entry> patchEntries(fromMaster);
125 // Check master names against mine
127 const polyBoundaryMesh& patches = mesh.boundaryMesh();
129 forAll(patchEntries, patchI)
131 const entry& e = patchEntries[patchI];
132 const word type(e.dict().lookup("type"));
133 const word& name = e.keyword();
135 if (type == processorPolyPatch::typeName)
140 if (patchI >= patches.size())
144 "createMesh(const Time&, const fileName&, const bool)"
145 ) << "Non-processor patches not synchronised."
147 << "Processor " << Pstream::myProcNo()
148 << " has only " << patches.size()
149 << " patches, master has "
156 type != patches[patchI].type()
157 || name != patches[patchI].name()
162 "createMesh(const Time&, const fileName&, const bool)"
163 ) << "Non-processor patches not synchronised."
165 << "Master patch " << patchI
167 << " type:" << type << endl
168 << "Processor " << Pstream::myProcNo()
169 << " patch " << patchI
170 << " has name:" << patches[patchI].name()
171 << " type:" << patches[patchI].type()
179 List<polyPatch*> patches(patchEntries.size());
182 forAll(patchEntries, patchI)
184 const entry& e = patchEntries[patchI];
185 const word type(e.dict().lookup("type"));
186 const word& name = e.keyword();
188 if (type == processorPolyPatch::typeName)
193 Pout<< "Adding patch:" << nPatches
195 << " type:" << type << endl;
197 dictionary patchDict(e.dict());
198 patchDict.remove("nFaces");
199 patchDict.add("nFaces", 0);
200 patchDict.remove("startFace");
201 patchDict.add("startFace", 0);
203 patches[patchI] = polyPatch::New
211 patches.setSize(nPatches);
212 mesh.addFvPatches(patches, false); // no parallel comms
214 //// Write empty mesh now we have correct patches
221 // We created a dummy mesh file above. Delete it.
222 Pout<< "Removing dummy mesh in " << runTime.path()/instDir << endl;
223 rmDir(runTime.path()/instDir/polyMesh::meshSubDir);
226 // Force recreation of globalMeshData.
234 // Get merging distance when matching face centres
235 scalar getMergeDistance
242 scalar mergeTol = defaultMergeTol;
243 if (args.options().found("mergeTol"))
245 mergeTol = readScalar(IStringStream(args.options()["mergeTol"])());
248 Foam::pow(scalar(10.0), -scalar(IOstream::defaultPrecision()));
250 Info<< "Merge tolerance : " << mergeTol << nl
251 << "Write tolerance : " << writeTol << endl;
253 if (runTime.writeFormat() == IOstream::ASCII && mergeTol < writeTol)
255 FatalErrorIn("getMergeDistance")
256 << "Your current settings specify ASCII writing with "
257 << IOstream::defaultPrecision() << " digits precision." << endl
258 << "Your merging tolerance (" << mergeTol << ") is finer than this."
260 << "Please change your writeFormat to binary"
261 << " or increase the writePrecision" << endl
262 << "or adjust the merge tolerance (-mergeTol)."
266 scalar mergeDist = mergeTol*mag(bb.max() - bb.min());
268 Info<< "Overall meshes bounding box : " << bb << nl
269 << "Relative tolerance : " << mergeTol << nl
270 << "Absolute matching distance : " << mergeDist << nl
277 void printMeshData(Ostream& os, const polyMesh& mesh)
279 os << "Number of points: "
280 << mesh.points().size() << nl
282 << mesh.edges().size() << nl
284 << mesh.faces().size() << nl
285 << " internal faces: "
286 << mesh.faceNeighbour().size() << nl
288 << mesh.cells().size() << nl
289 << " boundary patches: "
290 << mesh.boundaryMesh().size() << nl
292 << mesh.pointZones().size() << nl
295 << mesh.faceZones().size() << nl
298 << mesh.cellZones().size() << nl;
302 // Debugging: write volScalarField with decomposition for post processing.
303 void writeDecomposition
307 const labelList& decomp
310 Info<< "Writing wanted cell distribution to volScalarField " << name
311 << " for postprocessing purposes." << nl << endl;
313 volScalarField procCells
318 mesh.time().timeName(),
321 IOobject::AUTO_WRITE,
322 false // do not register
325 dimensionedScalar(name, dimless, -1),
326 zeroGradientFvPatchScalarField::typeName
329 forAll(procCells, cI)
331 procCells[cI] = decomp[cI];
337 // Read vol or surface fields
338 //template<class T, class Mesh>
339 template<class GeoField>
342 const boolList& haveMesh,
344 const autoPtr<fvMeshSubset>& subsetterPtr,
345 IOobjectList& allObjects,
346 PtrList<GeoField>& fields
349 //typedef GeometricField<T, fvPatchField, Mesh> fldType;
351 // Get my objects of type
352 IOobjectList objects(allObjects.lookupClass(GeoField::typeName));
353 // Check that we all have all objects
354 wordList objectNames = objects.toc();
356 wordList masterNames(objectNames);
357 Pstream::scatter(masterNames);
359 if (haveMesh[Pstream::myProcNo()] && objectNames != masterNames)
361 FatalErrorIn("readFields()")
362 << "differing fields of type " << GeoField::typeName
363 << " on processors." << endl
364 << "Master has:" << masterNames << endl
365 << Pstream::myProcNo() << " has:" << objectNames
366 << abort(FatalError);
369 fields.setSize(masterNames.size());
371 // Have master send all fields to processors that don't have a mesh
372 if (Pstream::master())
374 forAll(masterNames, i)
376 const word& name = masterNames[i];
377 IOobject& io = *objects[name];
378 io.writeOpt() = IOobject::AUTO_WRITE;
381 fields.set(i, new GeoField(io, mesh));
383 // Create zero sized field and send
384 if (subsetterPtr.valid())
386 tmp<GeoField> tsubfld = subsetterPtr().interpolate(fields[i]);
388 // Send to all processors that don't have a mesh
389 for (label procI = 1; procI < Pstream::nProcs(); procI++)
391 if (!haveMesh[procI])
393 OPstream toProc(Pstream::blocking, procI);
400 else if (!haveMesh[Pstream::myProcNo()])
402 // Don't have mesh (nor fields). Receive empty field from master.
404 forAll(masterNames, i)
406 const word& name = masterNames[i];
409 IPstream fromMaster(Pstream::blocking, Pstream::masterNo());
419 mesh.time().timeName(),
429 //// Write it for next time round (since mesh gets written as well)
435 // Have mesh so just try to load
436 forAll(masterNames, i)
438 const word& name = masterNames[i];
439 IOobject& io = *objects[name];
440 io.writeOpt() = IOobject::AUTO_WRITE;
443 fields.set(i, new GeoField(io, mesh));
449 // Debugging: compare two fields.
453 const volVectorField& a,
454 const volVectorField& b
459 if (mag(b[cellI] - a[cellI]) > tolDim)
464 "(const scalar, const volVectorField&, const volVectorField&)"
465 ) << "Did not map volVectorField correctly:" << nl
467 << " transfer b:" << b[cellI]
468 << " real cc:" << a[cellI]
469 << abort(FatalError);
472 forAll(a.boundaryField(), patchI)
474 // We have real mesh cellcentre and
475 // mapped original cell centre.
477 const fvPatchVectorField& aBoundary =
478 a.boundaryField()[patchI];
480 const fvPatchVectorField& bBoundary =
481 b.boundaryField()[patchI];
483 if (!bBoundary.coupled())
487 if (mag(aBoundary[i] - bBoundary[i]) > tolDim)
492 "(const scalar, const volVectorField&"
493 ", const volVectorField&)"
494 ) << "Did not map volVectorField correctly:"
496 << "patch:" << patchI << " patchFace:" << i
498 << " real :" << aBoundary[i] << endl
499 << " mapped :" << bBoundary[i] << endl
500 << abort(FatalError);
510 int main(int argc, char *argv[])
512 argList::validOptions.insert("mergeTol", "relative merge distance");
513 # include "setRootCase.H"
515 // Create processor directory if non-existing
516 if (!Pstream::master() && !dir(args.path()))
518 Pout<< "Creating case directory " << args.path() << endl;
522 # include "createTime.H"
524 // Get time instance directory. Since not all processors have meshes
525 // just use the master one everywhere.
527 fileName masterInstDir;
528 if (Pstream::master())
530 masterInstDir = runTime.findInstance(polyMesh::meshSubDir, "points");
532 Pstream::scatter(masterInstDir);
534 // Check who has a mesh
535 const fileName meshDir = runTime.path()/masterInstDir/polyMesh::meshSubDir;
537 boolList haveMesh(Pstream::nProcs(), false);
538 haveMesh[Pstream::myProcNo()] = dir(meshDir);
539 Pstream::gatherList(haveMesh);
540 Pstream::scatterList(haveMesh);
541 Info<< "Per processor mesh availability : " << haveMesh << endl;
542 const bool allHaveMesh = (findIndex(haveMesh, false) == -1);
545 autoPtr<fvMesh> meshPtr = createMesh
549 haveMesh[Pstream::myProcNo()]
551 fvMesh& mesh = meshPtr();
553 Pout<< "Read mesh:" << endl;
554 printMeshData(Pout, mesh);
557 IOdictionary decompositionDict
569 labelList finalDecomp;
571 // Create decompositionMethod and new decomposition
573 autoPtr<decompositionMethod> decomposer
575 decompositionMethod::New
582 if (!decomposer().parallelAware())
584 WarningIn(args.executable())
585 << "You have selected decomposition method "
586 << decomposer().typeName
587 << " which does" << endl
588 << "not synchronise the decomposition across"
589 << " processor patches." << endl
590 << " You might want to select a decomposition method which"
591 << " is aware of this. Continuing."
595 finalDecomp = decomposer().decompose(mesh.cellCentres());
598 // Dump decomposition to volScalarField
599 writeDecomposition("decomposition", mesh, finalDecomp);
602 // Create 0 sized mesh to do all the generation of zero sized
603 // fields on processors that have zero sized meshes. Note that this is
604 // only nessecary on master but since polyMesh construction with
605 // Pstream::parRun does parallel comms we have to do it on all
607 autoPtr<fvMeshSubset> subsetterPtr;
611 // Find last non-processor patch.
612 const polyBoundaryMesh& patches = mesh.boundaryMesh();
616 forAll(patches, patchI)
618 if (isA<processorPolyPatch>(patches[patchI]))
627 FatalErrorIn(args.executable())
628 << "Cannot find non-processor patch on processor "
629 << Pstream::myProcNo() << endl
630 << " Current patches:" << patches.names() << abort(FatalError);
633 // Subset 0 cells, no parallel comms. This is used to create zero-sized
635 subsetterPtr.reset(new fvMeshSubset(mesh));
636 subsetterPtr().setLargeCellSubset(labelHashSet(0), nonProcI, false);
640 // Get original objects (before incrementing time!)
641 IOobjectList objects(mesh, runTime.timeName());
642 // We don't want to map the decomposition (mapping already tested when
643 // mapping the cell centre field)
644 IOobjectList::iterator iter = objects.find("decomposition");
645 if (iter != objects.end())
650 PtrList<volScalarField> volScalarFields;
660 PtrList<volVectorField> volVectorFields;
670 PtrList<volSphericalTensorField> volSphereTensorFields;
677 volSphereTensorFields
680 PtrList<volSymmTensorField> volSymmTensorFields;
690 PtrList<volTensorField> volTensorFields;
700 PtrList<surfaceScalarField> surfScalarFields;
710 PtrList<surfaceVectorField> surfVectorFields;
720 PtrList<surfaceSphericalTensorField> surfSphereTensorFields;
727 surfSphereTensorFields
730 PtrList<surfaceSymmTensorField> surfSymmTensorFields;
740 PtrList<surfaceTensorField> surfTensorFields;
751 // Debugging: Create additional volField that will be mapped.
752 // Used to test correctness of mapping
753 volVectorField mapCc("mapCc", 1*mesh.C());
755 // Global matching tolerance
756 const scalar tolDim = getMergeDistance
760 mesh.globalData().bb()
763 // Mesh distribution engine
764 fvMeshDistribute distributor(mesh, tolDim);
766 Pout<< "Wanted distribution:"
767 << distributor.countCells(finalDecomp) << nl << endl;
769 // Do actual sending/receiving of mesh
770 autoPtr<mapDistributePolyMesh> map = distributor.distribute(finalDecomp);
772 //// Distribute any non-registered data accordingly
773 //map().distributeFaceData(faceCc);
777 Pout<< "After distribution mesh:" << endl;
778 printMeshData(Pout, mesh);
782 Pout<< "Writing redistributed mesh to " << runTime.timeName() << nl << endl;
786 // Debugging: test mapped cellcentre field.
787 compareFields(tolDim, mesh.C(), mapCc);
789 // Print nice message
790 // ~~~~~~~~~~~~~~~~~~
792 // Determine which processors remain so we can print nice final message.
793 labelList nFaces(Pstream::nProcs());
794 nFaces[Pstream::myProcNo()] = mesh.nFaces();
795 Pstream::gatherList(nFaces);
796 Pstream::scatterList(nFaces);
799 << "You can pick up the redecomposed mesh from the polyMesh directory"
800 << " in " << runTime.timeName() << "." << nl
801 << "If you redecomposed the mesh to less processors you can delete"
803 << "the processor directories with 0 sized meshes in them." << nl
804 << "Below is a sample set of commands to do this."
805 << " Take care when issueing these" << nl
806 << "commands." << nl << endl;
808 forAll(nFaces, procI)
810 fileName procDir = "processor" + name(procI);
812 if (nFaces[procI] == 0)
814 Info<< " rm -r " << procDir.c_str() << nl;
818 fileName timeDir = procDir/runTime.timeName()/polyMesh::meshSubDir;
819 fileName constDir = procDir/runTime.constant()/polyMesh::meshSubDir;
821 Info<< " rm -r " << constDir.c_str() << nl
822 << " mv " << timeDir.c_str()
823 << ' ' << constDir.c_str() << nl;
829 Pout<< "End\n" << endl;
835 // ************************************************************************* //