1 /*---------------------------------------------------------------------------*\
3 \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
5 \\ / A nd | Copyright (C) 1991-2009 OpenCFD Ltd.
7 -------------------------------------------------------------------------------
9 This file is part of OpenFOAM.
11 OpenFOAM is free software; you can redistribute it and/or modify it
12 under the terms of the GNU General Public License as published by the
13 Free Software Foundation; either version 2 of the License, or (at your
14 option) any later version.
16 OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with OpenFOAM; if not, write to the Free Software Foundation,
23 Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
29 Private member of domainDecomposition.
30 Decomposes the mesh into bits
32 \*---------------------------------------------------------------------------*/
34 #include "domainDecomposition.H"
35 #include "IOstreams.H"
36 #include "SLPtrList.H"
38 #include "primitiveMesh.H"
39 #include "cyclicPolyPatch.H"
41 // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
43 void domainDecomposition::decomposeMesh(const bool filterEmptyPatches)
45 // Decide which cell goes to which processor
48 // Distribute the cells according to the given processor label
50 // calculate the addressing information for the original mesh
51 Info<< "\nCalculating original mesh data" << endl;
53 // set references to the original mesh
54 const polyBoundaryMesh& patches = boundaryMesh();
55 const faceList& fcs = faces();
56 const labelList& owner = faceOwner();
57 const labelList& neighbour = faceNeighbour();
59 // loop through the list of processor labels for the cell and add the
60 // cell shape to the list of cells for the appropriate processor
62 Info<< "\nDistributing cells to processors" << endl;
66 List<SLList<label> > procCellList(nProcs_);
68 forAll (cellToProc_, celli)
70 if (cellToProc_[celli] >= nProcs_)
72 FatalErrorIn("domainDecomposition::decomposeMesh()")
73 << "Impossible processor label " << cellToProc_[celli]
74 << "for cell " << celli
79 procCellList[cellToProc_[celli]].append(celli);
83 // Convert linked lists into normal lists
84 forAll (procCellList, procI)
86 procCellAddressing_[procI] = procCellList[procI];
90 Info << "\nDistributing faces to processors" << endl;
92 // Loop through all internal faces and decide which processor they belong to
93 // First visit all internal faces. If cells at both sides belong to the
94 // same processor, the face is an internal face. If they are different,
95 // it belongs to both processors.
99 List<SLList<label> > procFaceList(nProcs_);
101 forAll (neighbour, facei)
103 if (cellToProc_[owner[facei]] == cellToProc_[neighbour[facei]])
105 // Face internal to processor
106 procFaceList[cellToProc_[owner[facei]]].append(facei);
110 // Detect inter-processor boundaries
112 // Neighbour processor for each subdomain
113 List<SLList<label> > interProcBoundaries(nProcs_);
115 // Face labels belonging to each inter-processor boundary
116 List<SLList<SLList<label> > > interProcBFaces(nProcs_);
118 List<SLList<label> > procPatchIndex(nProcs_);
120 forAll (neighbour, facei)
122 if (cellToProc_[owner[facei]] != cellToProc_[neighbour[facei]])
124 // inter - processor patch face found. Go through the list of
125 // inside boundaries for the owner processor and try to find
126 // this inter-processor patch.
128 label ownerProc = cellToProc_[owner[facei]];
129 label neighbourProc = cellToProc_[neighbour[facei]];
131 SLList<label>::iterator curInterProcBdrsOwnIter =
132 interProcBoundaries[ownerProc].begin();
134 SLList<SLList<label> >::iterator curInterProcBFacesOwnIter =
135 interProcBFaces[ownerProc].begin();
137 bool interProcBouFound = false;
139 // WARNING: Synchronous SLList iterators
144 curInterProcBdrsOwnIter
145 != interProcBoundaries[ownerProc].end()
146 && curInterProcBFacesOwnIter
147 != interProcBFaces[ownerProc].end();
148 ++curInterProcBdrsOwnIter, ++curInterProcBFacesOwnIter
151 if (curInterProcBdrsOwnIter() == neighbourProc)
153 // the inter - processor boundary exists. Add the face
154 interProcBouFound = true;
156 curInterProcBFacesOwnIter().append(facei);
158 SLList<label>::iterator curInterProcBdrsNeiIter =
159 interProcBoundaries[neighbourProc].begin();
161 SLList<SLList<label> >::iterator
162 curInterProcBFacesNeiIter =
163 interProcBFaces[neighbourProc].begin();
165 bool neighbourFound = false;
167 // WARNING: Synchronous SLList iterators
172 curInterProcBdrsNeiIter !=
173 interProcBoundaries[neighbourProc].end()
174 && curInterProcBFacesNeiIter !=
175 interProcBFaces[neighbourProc].end();
176 ++curInterProcBdrsNeiIter,
177 ++curInterProcBFacesNeiIter
180 if (curInterProcBdrsNeiIter() == ownerProc)
182 // boundary found. Add the face
183 neighbourFound = true;
185 curInterProcBFacesNeiIter().append(facei);
188 if (neighbourFound) break;
191 if (interProcBouFound && !neighbourFound)
193 FatalErrorIn("domainDecomposition::decomposeMesh()")
194 << "Inconsistency in inter - "
195 << "processor boundary lists for processors "
196 << ownerProc << " and " << neighbourProc
197 << abort(FatalError);
201 if (interProcBouFound) break;
204 if (!interProcBouFound)
206 // inter - processor boundaries do not exist and need to
209 // set the new addressing information
212 interProcBoundaries[ownerProc].append(neighbourProc);
213 interProcBFaces[ownerProc].append(SLList<label>(facei));
216 interProcBoundaries[neighbourProc].append(ownerProc);
217 interProcBFaces[neighbourProc].append(SLList<label>(facei));
222 // Loop through patches. For cyclic boundaries detect inter-processor
223 // faces; for all other, add faces to the face list and remember start
224 // and size of all patches.
226 // for all processors, set the size of start index and patch size
227 // lists to the number of patches in the mesh
228 forAll (procPatchSize_, procI)
230 procPatchSize_[procI].setSize(patches.size());
231 procPatchStartIndex_[procI].setSize(patches.size());
234 forAll (patches, patchi)
236 // Reset size and start index for all processors
237 forAll (procPatchSize_, procI)
239 procPatchSize_[procI][patchi] = 0;
240 procPatchStartIndex_[procI][patchi] =
241 procFaceList[procI].size();
244 const label patchStart = patches[patchi].start();
246 if (typeid(patches[patchi]) != typeid(cyclicPolyPatch))
248 // Normal patch. Add faces to processor where the cell
249 // next to the face lives
251 const unallocLabelList& patchFaceCells =
252 patches[patchi].faceCells();
254 forAll (patchFaceCells, facei)
256 const label curProc = cellToProc_[patchFaceCells[facei]];
259 procFaceList[curProc].append(patchStart + facei);
261 // increment the number of faces for this patch
262 procPatchSize_[curProc][patchi]++;
267 // Cyclic patch special treatment
269 const polyPatch& cPatch = patches[patchi];
271 const label cycOffset = cPatch.size()/2;
273 // Set reference to faceCells for both patches
274 const labelList::subList firstFaceCells
280 const labelList::subList secondFaceCells
287 forAll (firstFaceCells, facei)
291 cellToProc_[firstFaceCells[facei]]
292 != cellToProc_[secondFaceCells[facei]]
295 // This face becomes an inter-processor boundary face
296 // inter - processor patch face found. Go through
297 // the list of inside boundaries for the owner
298 // processor and try to find this inter-processor
301 cyclicParallel_ = true;
303 label ownerProc = cellToProc_[firstFaceCells[facei]];
304 label neighbourProc =
305 cellToProc_[secondFaceCells[facei]];
307 SLList<label>::iterator curInterProcBdrsOwnIter =
308 interProcBoundaries[ownerProc].begin();
310 SLList<SLList<label> >::iterator
311 curInterProcBFacesOwnIter =
312 interProcBFaces[ownerProc].begin();
314 bool interProcBouFound = false;
316 // WARNING: Synchronous SLList iterators
321 curInterProcBdrsOwnIter !=
322 interProcBoundaries[ownerProc].end()
323 && curInterProcBFacesOwnIter !=
324 interProcBFaces[ownerProc].end();
325 ++curInterProcBdrsOwnIter,
326 ++curInterProcBFacesOwnIter
329 if (curInterProcBdrsOwnIter() == neighbourProc)
331 // the inter - processor boundary exists.
333 interProcBouFound = true;
335 curInterProcBFacesOwnIter().append
336 (patchStart + facei);
338 SLList<label>::iterator curInterProcBdrsNeiIter
339 = interProcBoundaries[neighbourProc].begin();
341 SLList<SLList<label> >::iterator
342 curInterProcBFacesNeiIter =
343 interProcBFaces[neighbourProc].begin();
345 bool neighbourFound = false;
347 // WARNING: Synchronous SLList iterators
352 curInterProcBdrsNeiIter
353 != interProcBoundaries[neighbourProc].end()
354 && curInterProcBFacesNeiIter
355 != interProcBFaces[neighbourProc].end();
356 ++curInterProcBdrsNeiIter,
357 ++curInterProcBFacesNeiIter
360 if (curInterProcBdrsNeiIter() == ownerProc)
362 // boundary found. Add the face
363 neighbourFound = true;
365 curInterProcBFacesNeiIter()
374 if (neighbourFound) break;
377 if (interProcBouFound && !neighbourFound)
381 "domainDecomposition::decomposeMesh()"
382 ) << "Inconsistency in inter-processor "
383 << "boundary lists for processors "
384 << ownerProc << " and " << neighbourProc
385 << " in cyclic boundary matching"
386 << abort(FatalError);
390 if (interProcBouFound) break;
393 if (!interProcBouFound)
395 // inter - processor boundaries do not exist
396 // and need to be created
398 // set the new addressing information
401 interProcBoundaries[ownerProc]
402 .append(neighbourProc);
403 interProcBFaces[ownerProc]
404 .append(SLList<label>(patchStart + facei));
407 interProcBoundaries[neighbourProc]
409 interProcBFaces[neighbourProc]
423 // This cyclic face remains on the processor
424 label ownerProc = cellToProc_[firstFaceCells[facei]];
427 procFaceList[ownerProc].append(patchStart + facei);
429 // increment the number of faces for this patch
430 procPatchSize_[ownerProc][patchi]++;
432 // Note: I cannot add the other side of the cyclic
433 // boundary here because this would violate the order.
434 // They will be added in a separate loop below
439 // Ordering in cyclic boundaries is important.
440 // Add the other half of cyclic faces for cyclic boundaries
441 // that remain on the processor
442 forAll (secondFaceCells, facei)
446 cellToProc_[firstFaceCells[facei]]
447 == cellToProc_[secondFaceCells[facei]]
450 // This cyclic face remains on the processor
451 label ownerProc = cellToProc_[firstFaceCells[facei]];
453 // add the second face
454 procFaceList[ownerProc].append
455 (patchStart + cycOffset + facei);
457 // increment the number of faces for this patch
458 procPatchSize_[ownerProc][patchi]++;
464 // Convert linked lists into normal lists
465 // Add inter-processor boundaries and remember start indices
466 forAll (procFaceList, procI)
468 // Get internal and regular boundary processor faces
469 SLList<label>& curProcFaces = procFaceList[procI];
471 // Get reference to processor face addressing
472 labelList& curProcFaceAddressing = procFaceAddressing_[procI];
474 labelList& curProcNeighbourProcessors =
475 procNeighbourProcessors_[procI];
477 labelList& curProcProcessorPatchSize =
478 procProcessorPatchSize_[procI];
480 labelList& curProcProcessorPatchStartIndex =
481 procProcessorPatchStartIndex_[procI];
483 // calculate the size
484 label nFacesOnProcessor = curProcFaces.size();
488 SLList<SLList<label> >::iterator curInterProcBFacesIter =
489 interProcBFaces[procI].begin();
490 curInterProcBFacesIter != interProcBFaces[procI].end();
491 ++curInterProcBFacesIter
494 nFacesOnProcessor += curInterProcBFacesIter().size();
497 curProcFaceAddressing.setSize(nFacesOnProcessor);
499 // Fill in the list. Calculate turning index.
500 // Turning index will be -1 only for some faces on processor
501 // boundaries, i.e. the ones where the current processor ID
502 // is in the cell which is a face neighbour.
503 // Turning index is stored as the sign of the face addressing list
507 // Add internal and boundary faces
508 // Remember to increment the index by one such that the
509 // turning index works properly.
512 SLList<label>::iterator curProcFacesIter = curProcFaces.begin();
513 curProcFacesIter != curProcFaces.end();
517 curProcFaceAddressing[nFaces] = curProcFacesIter() + 1;
521 // Add inter-processor boundary faces. At the beginning of each
522 // patch, grab the patch start index and size
524 curProcNeighbourProcessors.setSize
526 interProcBoundaries[procI].size()
529 curProcProcessorPatchSize.setSize
531 interProcBoundaries[procI].size()
534 curProcProcessorPatchStartIndex.setSize
536 interProcBoundaries[procI].size()
539 label nProcPatches = 0;
541 SLList<label>::iterator curInterProcBdrsIter =
542 interProcBoundaries[procI].begin();
544 SLList<SLList<label> >::iterator curInterProcBFacesIter =
545 interProcBFaces[procI].begin();
550 curInterProcBdrsIter != interProcBoundaries[procI].end()
551 && curInterProcBFacesIter != interProcBFaces[procI].end();
552 ++curInterProcBdrsIter, ++curInterProcBFacesIter
555 curProcNeighbourProcessors[nProcPatches] =
556 curInterProcBdrsIter();
558 // Get start index for processor patch
559 curProcProcessorPatchStartIndex[nProcPatches] = nFaces;
562 curProcProcessorPatchSize[nProcPatches];
566 // add faces for this processor boundary
570 SLList<label>::iterator curFacesIter =
571 curInterProcBFacesIter().begin();
572 curFacesIter != curInterProcBFacesIter().end();
578 // Remember to increment the index by one such that the
579 // turning index works properly.
580 if (cellToProc_[owner[curFacesIter()]] == procI)
582 curProcFaceAddressing[nFaces] = curFacesIter() + 1;
587 curProcFaceAddressing[nFaces] = -(curFacesIter() + 1);
590 // increment the size
601 Info << "\nCalculating processor boundary addressing" << endl;
602 // For every patch of processor boundary, find the index of the original
603 // patch. Mis-alignment is caused by the fact that patches with zero size
604 // are omitted. For processor patches, set index to -1.
605 // At the same time, filter the procPatchSize_ and procPatchStartIndex_
606 // lists to exclude zero-size patches
607 forAll (procPatchSize_, procI)
609 // Make a local copy of old lists
610 const labelList oldPatchSizes = procPatchSize_[procI];
612 const labelList oldPatchStarts = procPatchStartIndex_[procI];
614 labelList& curPatchSizes = procPatchSize_[procI];
616 labelList& curPatchStarts = procPatchStartIndex_[procI];
618 const labelList& curProcessorPatchSizes =
619 procProcessorPatchSize_[procI];
621 labelList& curBoundaryAddressing = procBoundaryAddressing_[procI];
623 curBoundaryAddressing.setSize
626 + curProcessorPatchSizes.size()
631 forAll (oldPatchSizes, patchi)
633 if (!filterEmptyPatches || oldPatchSizes[patchi] > 0)
635 curBoundaryAddressing[nPatches] = patchi;
637 curPatchSizes[nPatches] = oldPatchSizes[patchi];
639 curPatchStarts[nPatches] = oldPatchStarts[patchi];
645 // reset to the size of live patches
646 curPatchSizes.setSize(nPatches);
647 curPatchStarts.setSize(nPatches);
649 forAll (curProcessorPatchSizes, procPatchI)
651 curBoundaryAddressing[nPatches] = -1;
656 curBoundaryAddressing.setSize(nPatches);
659 Info << "\nDistributing points to processors" << endl;
660 // For every processor, loop through the list of faces for the processor.
661 // For every face, loop through the list of points and mark the point as
662 // used for the processor. Collect the list of used points for the
665 forAll (procPointAddressing_, procI)
667 boolList pointLabels(nPoints(), false);
669 // Get reference to list of used faces
670 const labelList& procFaceLabels = procFaceAddressing_[procI];
672 forAll (procFaceLabels, facei)
674 // Because of the turning index, some labels may be negative
675 const labelList& facePoints = fcs[mag(procFaceLabels[facei]) - 1];
677 forAll (facePoints, pointi)
679 // Mark the point as used
680 pointLabels[facePoints[pointi]] = true;
684 // Collect the used points
685 labelList& procPointLabels = procPointAddressing_[procI];
687 procPointLabels.setSize(pointLabels.size());
689 label nUsedPoints = 0;
691 forAll (pointLabels, pointi)
693 if (pointLabels[pointi])
695 procPointLabels[nUsedPoints] = pointi;
701 // Reset the size of used points
702 procPointLabels.setSize(nUsedPoints);
705 // Gather data about globally shared points
709 labelList pointsUsage(nPoints(), 0);
711 // Globally shared points are the ones used by more than 2 processors
712 // Size the list approximately and gather the points
713 labelHashSet gSharedPoints
715 min(100, nPoints()/1000)
718 // Loop through all the processors and mark up points used by
719 // processor boundaries. When a point is used twice, it is a
720 // globally shared point
722 for (label procI = 0; procI < nProcs_; procI++)
724 // Get list of face labels
725 const labelList& curFaceLabels = procFaceAddressing_[procI];
727 // Get start of processor faces
728 const labelList& curProcessorPatchStarts =
729 procProcessorPatchStartIndex_[procI];
731 const labelList& curProcessorPatchSizes =
732 procProcessorPatchSize_[procI];
734 // Reset the lookup list
737 forAll (curProcessorPatchStarts, patchi)
739 const label curStart = curProcessorPatchStarts[patchi];
740 const label curEnd = curStart + curProcessorPatchSizes[patchi];
744 label facei = curStart;
749 // Mark the original face as used
750 // Remember to decrement the index by one (turning index)
752 const label curF = mag(curFaceLabels[facei]) - 1;
754 const face& f = fcs[curF];
758 if (pointsUsage[f[pointi]] == 0)
760 // Point not previously used
761 pointsUsage[f[pointi]] = patchi + 1;
763 else if (pointsUsage[f[pointi]] != patchi + 1)
765 // Point used by some other patch = global point!
766 gSharedPoints.insert(f[pointi]);
773 // Grab the result from the hash list
774 globallySharedPoints_ = gSharedPoints.toc();
775 sort(globallySharedPoints_);