initial commit for version 1.6.x patch release
[OpenFOAM-1.6.x.git] / applications / utilities / parallelProcessing / decomposePar / decomposeMesh.C
bloba3df89e3d7af5d1eb0e5aa90004682b411ab7888
1 /*---------------------------------------------------------------------------*\
2   =========                 |
3   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
4    \\    /   O peration     |
5     \\  /    A nd           | Copyright (C) 1991-2009 OpenCFD Ltd.
6      \\/     M anipulation  |
7 -------------------------------------------------------------------------------
8 License
9     This file is part of OpenFOAM.
11     OpenFOAM is free software; you can redistribute it and/or modify it
12     under the terms of the GNU General Public License as published by the
13     Free Software Foundation; either version 2 of the License, or (at your
14     option) any later version.
16     OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17     ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18     FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
19     for more details.
21     You should have received a copy of the GNU General Public License
22     along with OpenFOAM; if not, write to the Free Software Foundation,
23     Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
25 InClass
26     domainDecomposition
28 Description
29     Private member of domainDecomposition.
30     Decomposes the mesh into bits
32 \*---------------------------------------------------------------------------*/
34 #include "domainDecomposition.H"
35 #include "IOstreams.H"
36 #include "SLPtrList.H"
37 #include "boolList.H"
38 #include "primitiveMesh.H"
39 #include "cyclicPolyPatch.H"
41 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
43 void domainDecomposition::decomposeMesh(const bool filterEmptyPatches)
45     // Decide which cell goes to which processor
46     distributeCells();
48     // Distribute the cells according to the given processor label
50     // calculate the addressing information for the original mesh
51     Info<< "\nCalculating original mesh data" << endl;
53     // set references to the original mesh
54     const polyBoundaryMesh& patches = boundaryMesh();
55     const faceList& fcs = faces();
56     const labelList& owner = faceOwner();
57     const labelList& neighbour = faceNeighbour();
59     // loop through the list of processor labels for the cell and add the
60     // cell shape to the list of cells for the appropriate processor
62     Info<< "\nDistributing cells to processors" << endl;
64     // Memory management
65     {
66         List<SLList<label> > procCellList(nProcs_);
68         forAll (cellToProc_, celli)
69         {
70             if (cellToProc_[celli] >= nProcs_)
71             {
72                 FatalErrorIn("domainDecomposition::decomposeMesh()")
73                     << "Impossible processor label " << cellToProc_[celli]
74                     << "for cell " << celli
75                     << abort(FatalError);
76             }
77             else
78             {
79                 procCellList[cellToProc_[celli]].append(celli);
80             }
81         }
83         // Convert linked lists into normal lists
84         forAll (procCellList, procI)
85         {
86             procCellAddressing_[procI] = procCellList[procI];
87         }
88     }
90     Info << "\nDistributing faces to processors" << endl;
92     // Loop through all internal faces and decide which processor they belong to
93     // First visit all internal faces. If cells at both sides belong to the
94     // same processor, the face is an internal face. If they are different,
95     // it belongs to both processors.
97     // Memory management
98     {
99         List<SLList<label> > procFaceList(nProcs_);
101         forAll (neighbour, facei)
102         {
103             if (cellToProc_[owner[facei]] == cellToProc_[neighbour[facei]])
104             {
105                 // Face internal to processor
106                 procFaceList[cellToProc_[owner[facei]]].append(facei);
107             }
108         }
110         // Detect inter-processor boundaries
112         // Neighbour processor for each subdomain
113         List<SLList<label> > interProcBoundaries(nProcs_);
115         // Face labels belonging to each inter-processor boundary
116         List<SLList<SLList<label> > > interProcBFaces(nProcs_);
118         List<SLList<label> > procPatchIndex(nProcs_);
120         forAll (neighbour, facei)
121         {
122             if (cellToProc_[owner[facei]] != cellToProc_[neighbour[facei]])
123             {
124                 // inter - processor patch face found. Go through the list of
125                 // inside boundaries for the owner processor and try to find
126                 // this inter-processor patch.
128                 label ownerProc = cellToProc_[owner[facei]];
129                 label neighbourProc = cellToProc_[neighbour[facei]];
131                 SLList<label>::iterator curInterProcBdrsOwnIter =
132                     interProcBoundaries[ownerProc].begin();
134                 SLList<SLList<label> >::iterator curInterProcBFacesOwnIter =
135                     interProcBFaces[ownerProc].begin();
137                 bool interProcBouFound = false;
139                 // WARNING: Synchronous SLList iterators
141                 for
142                 (
143                     ;
144                     curInterProcBdrsOwnIter
145                  != interProcBoundaries[ownerProc].end()
146                  && curInterProcBFacesOwnIter
147                  != interProcBFaces[ownerProc].end();
148                     ++curInterProcBdrsOwnIter, ++curInterProcBFacesOwnIter
149                 )
150                 {
151                     if (curInterProcBdrsOwnIter() == neighbourProc)
152                     {
153                         // the inter - processor boundary exists. Add the face
154                         interProcBouFound = true;
156                         curInterProcBFacesOwnIter().append(facei);
158                         SLList<label>::iterator curInterProcBdrsNeiIter =
159                             interProcBoundaries[neighbourProc].begin();
161                         SLList<SLList<label> >::iterator 
162                             curInterProcBFacesNeiIter =
163                             interProcBFaces[neighbourProc].begin();
165                         bool neighbourFound = false;
167                         // WARNING: Synchronous SLList iterators
169                         for
170                         (
171                             ;
172                             curInterProcBdrsNeiIter !=
173                             interProcBoundaries[neighbourProc].end()
174                          && curInterProcBFacesNeiIter !=
175                             interProcBFaces[neighbourProc].end();
176                             ++curInterProcBdrsNeiIter,
177                             ++curInterProcBFacesNeiIter
178                         )
179                         {
180                             if (curInterProcBdrsNeiIter() == ownerProc)
181                             {
182                                 // boundary found. Add the face
183                                 neighbourFound = true;
185                                 curInterProcBFacesNeiIter().append(facei);
186                             }
188                             if (neighbourFound) break;
189                         }
191                         if (interProcBouFound && !neighbourFound)
192                         {
193                             FatalErrorIn("domainDecomposition::decomposeMesh()")
194                                 << "Inconsistency in inter - "
195                                 << "processor boundary lists for processors "
196                                 << ownerProc << " and " << neighbourProc
197                                 << abort(FatalError);
198                         }
199                     }
201                     if (interProcBouFound) break;
202                 }
204                 if (!interProcBouFound)
205                 {
206                     // inter - processor boundaries do not exist and need to
207                     // be created
209                     // set the new addressing information
211                     // owner
212                     interProcBoundaries[ownerProc].append(neighbourProc);
213                     interProcBFaces[ownerProc].append(SLList<label>(facei));
215                     // neighbour
216                     interProcBoundaries[neighbourProc].append(ownerProc);
217                     interProcBFaces[neighbourProc].append(SLList<label>(facei));
218                 }
219             }
220         }
222         // Loop through patches. For cyclic boundaries detect inter-processor
223         // faces; for all other, add faces to the face list and remember start
224         // and size of all patches.
226         // for all processors, set the size of start index and patch size
227         // lists to the number of patches in the mesh
228         forAll (procPatchSize_, procI)
229         {
230             procPatchSize_[procI].setSize(patches.size());
231             procPatchStartIndex_[procI].setSize(patches.size());
232         }
234         forAll (patches, patchi)
235         {
236             // Reset size and start index for all processors
237             forAll (procPatchSize_, procI)
238             {
239                 procPatchSize_[procI][patchi] = 0;
240                 procPatchStartIndex_[procI][patchi] =
241                     procFaceList[procI].size();
242             }
244             const label patchStart = patches[patchi].start();
246             if (typeid(patches[patchi]) != typeid(cyclicPolyPatch))
247             {
248                 // Normal patch. Add faces to processor where the cell
249                 // next to the face lives
251                 const unallocLabelList& patchFaceCells =
252                     patches[patchi].faceCells();
254                 forAll (patchFaceCells, facei)
255                 {
256                     const label curProc = cellToProc_[patchFaceCells[facei]];
258                     // add the face
259                     procFaceList[curProc].append(patchStart + facei);
261                     // increment the number of faces for this patch
262                     procPatchSize_[curProc][patchi]++;
263                 }
264             }
265             else
266             {
267                 // Cyclic patch special treatment
269                 const polyPatch& cPatch = patches[patchi];
271                 const label cycOffset = cPatch.size()/2;
273                 // Set reference to faceCells for both patches
274                 const labelList::subList firstFaceCells
275                 (
276                     cPatch.faceCells(),
277                     cycOffset
278                 );
280                 const labelList::subList secondFaceCells
281                 (
282                     cPatch.faceCells(),
283                     cycOffset,
284                     cycOffset
285                 );
287                 forAll (firstFaceCells, facei)
288                 {
289                     if
290                     (
291                         cellToProc_[firstFaceCells[facei]]
292                      != cellToProc_[secondFaceCells[facei]]
293                     )
294                     {
295                         // This face becomes an inter-processor boundary face
296                         // inter - processor patch face found. Go through
297                         // the list of inside boundaries for the owner
298                         // processor and try to find this inter-processor
299                         // patch.
301                         cyclicParallel_ = true;
303                         label ownerProc = cellToProc_[firstFaceCells[facei]];
304                         label neighbourProc =
305                             cellToProc_[secondFaceCells[facei]];
307                         SLList<label>::iterator curInterProcBdrsOwnIter =
308                             interProcBoundaries[ownerProc].begin();
310                         SLList<SLList<label> >::iterator 
311                             curInterProcBFacesOwnIter =
312                             interProcBFaces[ownerProc].begin();
314                         bool interProcBouFound = false;
316                         // WARNING: Synchronous SLList iterators
318                         for
319                         (
320                             ;
321                             curInterProcBdrsOwnIter !=
322                             interProcBoundaries[ownerProc].end()
323                          && curInterProcBFacesOwnIter !=
324                             interProcBFaces[ownerProc].end();
325                             ++curInterProcBdrsOwnIter,
326                             ++curInterProcBFacesOwnIter
327                         )
328                         {
329                             if (curInterProcBdrsOwnIter() == neighbourProc)
330                             {
331                                 // the inter - processor boundary exists.
332                                 // Add the face
333                                 interProcBouFound = true;
335                                 curInterProcBFacesOwnIter().append
336                                     (patchStart + facei);
338                                 SLList<label>::iterator curInterProcBdrsNeiIter
339                                    = interProcBoundaries[neighbourProc].begin();
341                                 SLList<SLList<label> >::iterator
342                                     curInterProcBFacesNeiIter =
343                                     interProcBFaces[neighbourProc].begin();
345                                 bool neighbourFound = false;
347                                 // WARNING: Synchronous SLList iterators
349                                 for
350                                 (
351                                     ;
352                                     curInterProcBdrsNeiIter
353                                    != interProcBoundaries[neighbourProc].end()
354                                  && curInterProcBFacesNeiIter
355                                    != interProcBFaces[neighbourProc].end();
356                                     ++curInterProcBdrsNeiIter,
357                                     ++curInterProcBFacesNeiIter
358                                 )
359                                 {
360                                     if (curInterProcBdrsNeiIter() == ownerProc)
361                                     {
362                                         // boundary found. Add the face
363                                         neighbourFound = true;
365                                         curInterProcBFacesNeiIter()
366                                             .append
367                                             (
368                                                 patchStart
369                                               + cycOffset
370                                               + facei
371                                             );
372                                     }
374                                     if (neighbourFound) break;
375                                 }
377                                 if (interProcBouFound && !neighbourFound)
378                                 {
379                                     FatalErrorIn
380                                     (
381                                         "domainDecomposition::decomposeMesh()"
382                                     )   << "Inconsistency in inter-processor "
383                                         << "boundary lists for processors "
384                                         << ownerProc << " and " << neighbourProc
385                                         << " in cyclic boundary matching"
386                                         << abort(FatalError);
387                                 }
388                             }
390                             if (interProcBouFound) break;
391                         }
393                         if (!interProcBouFound)
394                         {
395                             // inter - processor boundaries do not exist
396                             // and need to be created
398                             // set the new addressing information
400                             // owner
401                             interProcBoundaries[ownerProc]
402                                 .append(neighbourProc);
403                             interProcBFaces[ownerProc]
404                                 .append(SLList<label>(patchStart + facei));
406                             // neighbour
407                             interProcBoundaries[neighbourProc]
408                                 .append(ownerProc);
409                             interProcBFaces[neighbourProc]
410                                 .append
411                                 (
412                                     SLList<label>
413                                     (
414                                         patchStart
415                                       + cycOffset
416                                       + facei
417                                     )
418                                 );
419                         }
420                     }
421                     else
422                     {
423                         // This cyclic face remains on the processor
424                         label ownerProc = cellToProc_[firstFaceCells[facei]];
426                         // add the face
427                         procFaceList[ownerProc].append(patchStart + facei);
429                         // increment the number of faces for this patch
430                         procPatchSize_[ownerProc][patchi]++;
432                         // Note: I cannot add the other side of the cyclic
433                         // boundary here because this would violate the order.
434                         // They will be added in a separate loop below
435                         // 
436                     }
437                 }
439                 // Ordering in cyclic boundaries is important.
440                 // Add the other half of cyclic faces for cyclic boundaries
441                 // that remain on the processor
442                 forAll (secondFaceCells, facei)
443                 {
444                     if
445                     (
446                         cellToProc_[firstFaceCells[facei]]
447                      == cellToProc_[secondFaceCells[facei]]
448                     )
449                     {
450                         // This cyclic face remains on the processor
451                         label ownerProc = cellToProc_[firstFaceCells[facei]];
453                         // add the second face
454                         procFaceList[ownerProc].append
455                             (patchStart + cycOffset + facei);
457                         // increment the number of faces for this patch
458                         procPatchSize_[ownerProc][patchi]++;
459                     }
460                 }
461             }
462         }
464         // Convert linked lists into normal lists
465         // Add inter-processor boundaries and remember start indices
466         forAll (procFaceList, procI)
467         {
468             // Get internal and regular boundary processor faces
469             SLList<label>& curProcFaces = procFaceList[procI];
471             // Get reference to processor face addressing
472             labelList& curProcFaceAddressing = procFaceAddressing_[procI];
474             labelList& curProcNeighbourProcessors =
475                 procNeighbourProcessors_[procI];
477             labelList& curProcProcessorPatchSize =
478                 procProcessorPatchSize_[procI];
480             labelList& curProcProcessorPatchStartIndex =
481                 procProcessorPatchStartIndex_[procI];
483             // calculate the size
484             label nFacesOnProcessor = curProcFaces.size();
486             for 
487             (
488                 SLList<SLList<label> >::iterator curInterProcBFacesIter =
489                     interProcBFaces[procI].begin();
490                 curInterProcBFacesIter != interProcBFaces[procI].end();
491                 ++curInterProcBFacesIter
492             )
493             {
494                 nFacesOnProcessor += curInterProcBFacesIter().size();
495             }
497             curProcFaceAddressing.setSize(nFacesOnProcessor);
499             // Fill in the list. Calculate turning index.
500             // Turning index will be -1 only for some faces on processor
501             // boundaries, i.e. the ones where the current processor ID
502             // is in the cell which is a face neighbour.
503             // Turning index is stored as the sign of the face addressing list
505             label nFaces = 0;
507             // Add internal and boundary faces
508             // Remember to increment the index by one such that the
509             // turning index works properly.  
510             for
511             (
512                 SLList<label>::iterator curProcFacesIter = curProcFaces.begin();
513                 curProcFacesIter != curProcFaces.end();
514                 ++curProcFacesIter
515             )
516             {
517                 curProcFaceAddressing[nFaces] = curProcFacesIter() + 1;
518                 nFaces++;
519             }
521             // Add inter-processor boundary faces. At the beginning of each
522             // patch, grab the patch start index and size
524             curProcNeighbourProcessors.setSize
525             (
526                 interProcBoundaries[procI].size()
527             );
529             curProcProcessorPatchSize.setSize
530             (
531                 interProcBoundaries[procI].size()
532             );
534             curProcProcessorPatchStartIndex.setSize
535             (
536                 interProcBoundaries[procI].size()
537             );
539             label nProcPatches = 0;
541             SLList<label>::iterator curInterProcBdrsIter =
542                 interProcBoundaries[procI].begin();
544             SLList<SLList<label> >::iterator curInterProcBFacesIter =
545                 interProcBFaces[procI].begin();
547             for
548             (
549                 ;
550                 curInterProcBdrsIter != interProcBoundaries[procI].end()
551              && curInterProcBFacesIter != interProcBFaces[procI].end();
552                 ++curInterProcBdrsIter, ++curInterProcBFacesIter
553             )
554             {
555                 curProcNeighbourProcessors[nProcPatches] =
556                     curInterProcBdrsIter();
558                 // Get start index for processor patch
559                 curProcProcessorPatchStartIndex[nProcPatches] = nFaces;
561                 label& curSize =
562                     curProcProcessorPatchSize[nProcPatches];
564                 curSize = 0;
566                 // add faces for this processor boundary
568                 for
569                 (
570                     SLList<label>::iterator curFacesIter =
571                         curInterProcBFacesIter().begin();
572                     curFacesIter != curInterProcBFacesIter().end();
573                     ++curFacesIter
574                 )
575                 {
576                     // add the face
578                     // Remember to increment the index by one such that the
579                     // turning index works properly.  
580                     if (cellToProc_[owner[curFacesIter()]] == procI)
581                     {
582                         curProcFaceAddressing[nFaces] = curFacesIter() + 1;
583                     }
584                     else
585                     {
586                         // turning face
587                         curProcFaceAddressing[nFaces] = -(curFacesIter() + 1);
588                     }
590                     // increment the size
591                     curSize++;
593                     nFaces++;
594                 }
596                 nProcPatches++;
597             }
598         }
599     }
601     Info << "\nCalculating processor boundary addressing" << endl;
602     // For every patch of processor boundary, find the index of the original
603     // patch. Mis-alignment is caused by the fact that patches with zero size
604     // are omitted. For processor patches, set index to -1.
605     // At the same time, filter the procPatchSize_ and procPatchStartIndex_
606     // lists to exclude zero-size patches
607     forAll (procPatchSize_, procI)
608     {
609         // Make a local copy of old lists
610         const labelList oldPatchSizes = procPatchSize_[procI];
612         const labelList oldPatchStarts = procPatchStartIndex_[procI];
614         labelList& curPatchSizes = procPatchSize_[procI];
616         labelList& curPatchStarts = procPatchStartIndex_[procI];
618         const labelList& curProcessorPatchSizes =
619             procProcessorPatchSize_[procI];
621         labelList& curBoundaryAddressing = procBoundaryAddressing_[procI];
623         curBoundaryAddressing.setSize
624         (
625             oldPatchSizes.size()
626           + curProcessorPatchSizes.size()
627         );
629         label nPatches = 0;
631         forAll (oldPatchSizes, patchi)
632         {
633             if (!filterEmptyPatches || oldPatchSizes[patchi] > 0)
634             {
635                 curBoundaryAddressing[nPatches] = patchi;
637                 curPatchSizes[nPatches] = oldPatchSizes[patchi];
639                 curPatchStarts[nPatches] = oldPatchStarts[patchi];
641                 nPatches++;
642             }
643         }
645         // reset to the size of live patches
646         curPatchSizes.setSize(nPatches);
647         curPatchStarts.setSize(nPatches);
649         forAll (curProcessorPatchSizes, procPatchI)
650         {
651             curBoundaryAddressing[nPatches] = -1;
653             nPatches++;
654         }
656         curBoundaryAddressing.setSize(nPatches);
657     }
659     Info << "\nDistributing points to processors" << endl;
660     // For every processor, loop through the list of faces for the processor.
661     // For every face, loop through the list of points and mark the point as
662     // used for the processor. Collect the list of used points for the
663     // processor.
665     forAll (procPointAddressing_, procI)
666     {
667         boolList pointLabels(nPoints(), false);
669         // Get reference to list of used faces
670         const labelList& procFaceLabels = procFaceAddressing_[procI];
672         forAll (procFaceLabels, facei)
673         {
674             // Because of the turning index, some labels may be negative
675             const labelList& facePoints = fcs[mag(procFaceLabels[facei]) - 1];
677             forAll (facePoints, pointi)
678             {
679                 // Mark the point as used
680                 pointLabels[facePoints[pointi]] = true;
681             }
682         }
684         // Collect the used points
685         labelList& procPointLabels = procPointAddressing_[procI];
687         procPointLabels.setSize(pointLabels.size());
689         label nUsedPoints = 0;
691         forAll (pointLabels, pointi)
692         {
693             if (pointLabels[pointi])
694             {
695                 procPointLabels[nUsedPoints] = pointi;
697                 nUsedPoints++;
698             }
699         }
701         // Reset the size of used points
702         procPointLabels.setSize(nUsedPoints);
703     }
705     // Gather data about globally shared points
707     // Memory management
708     {
709         labelList pointsUsage(nPoints(), 0);
711         // Globally shared points are the ones used by more than 2 processors
712         // Size the list approximately and gather the points
713         labelHashSet gSharedPoints
714         (
715             min(100, nPoints()/1000)
716         );
718         // Loop through all the processors and mark up points used by
719         // processor boundaries.  When a point is used twice, it is a
720         // globally shared point
722         for (label procI = 0; procI < nProcs_; procI++)
723         {
724             // Get list of face labels
725             const labelList& curFaceLabels = procFaceAddressing_[procI];
727             // Get start of processor faces
728             const labelList& curProcessorPatchStarts =
729                 procProcessorPatchStartIndex_[procI];
731             const labelList& curProcessorPatchSizes =
732                 procProcessorPatchSize_[procI];
734             // Reset the lookup list
735             pointsUsage = 0;
737             forAll (curProcessorPatchStarts, patchi)
738             {
739                 const label curStart = curProcessorPatchStarts[patchi];
740                 const label curEnd = curStart + curProcessorPatchSizes[patchi];
742                 for
743                 (
744                     label facei = curStart;
745                     facei < curEnd;
746                     facei++
747                 )
748                 {
749                     // Mark the original face as used
750                     // Remember to decrement the index by one (turning index)
751                     // 
752                     const label curF = mag(curFaceLabels[facei]) - 1;
754                     const face& f = fcs[curF];
756                     forAll (f, pointi)
757                     {
758                         if (pointsUsage[f[pointi]] == 0)
759                         {
760                             // Point not previously used
761                             pointsUsage[f[pointi]] = patchi + 1;
762                         }
763                         else if (pointsUsage[f[pointi]] != patchi + 1)
764                         {
765                             // Point used by some other patch = global point!
766                             gSharedPoints.insert(f[pointi]);
767                         }
768                     }
769                 }
770             }
771         }
773         // Grab the result from the hash list
774         globallySharedPoints_ = gSharedPoints.toc();
775         sort(globallySharedPoints_);
776     }