BUG: decompositionMethod: constraints across coupled faces

src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C

@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2013 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2014 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -33,6 +33,8 @@ InClass
 #include "faceSet.H"
 #include "regionSplit.H"
 #include "localPointRegion.H"
+#include "minData.H"
+#include "FaceCellWave.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -956,6 +958,72 @@ Foam::labelList Foam::decompositionMethod::decompose
         }
 
 
+        // blockedFaces corresponding to processor faces need to be handled
+        // separately since not handled by local regionSplit. We need to
+        // walk now across coupled faces and make sure to move a whole
+        // global region across
+        if (Pstream::parRun())
+        {
+            // Re-do regionSplit
+
+            // Field on cells and faces.
+            List<minData> cellData(mesh.nCells());
+            List<minData> faceData(mesh.nFaces());
+
+            // Take over blockedFaces by seeding a negative number
+            // (so is always less than the decomposition)
+            label nUnblocked = 0;
+            forAll(blockedFace, faceI)
+            {
+                if (blockedFace[faceI])
+                {
+                    faceData[faceI] = minData(-123);
+                }
+                else
+                {
+                    nUnblocked++;
+                }
+            }
+
+            // Seed unblocked faces with destination processor
+            labelList seedFaces(nUnblocked);
+            List<minData> seedData(nUnblocked);
+            nUnblocked = 0;
+
+            forAll(blockedFace, faceI)
+            {
+                if (!blockedFace[faceI])
+                {
+                    label own = mesh.faceOwner()[faceI];
+                    seedFaces[nUnblocked] = faceI;
+                    seedData[nUnblocked] = minData(finalDecomp[own]);
+                    nUnblocked++;
+                }
+            }
+
+
+            // Propagate information inwards
+            FaceCellWave<minData> deltaCalc
+            (
+                mesh,
+                seedFaces,
+                seedData,
+                faceData,
+                cellData,
+                mesh.globalData().nTotalCells()+1
+            );
+
+            // And extract
+            forAll(finalDecomp, cellI)
+            {
+                if (cellData[cellI].valid(deltaCalc.data()))
+                {
+                    finalDecomp[cellI] = cellData[cellI].data();
+                }
+            }
+        }
+
+
         // For specifiedProcessorFaces rework the cellToProc to enforce
         // all on one processor since we can't guarantee that the input
         // to regionSplit was a single region.
@@ -1000,6 +1068,44 @@ Foam::labelList Foam::decompositionMethod::decompose
                 }
             }
         }
+
+
+        if (debug && Pstream::parRun())
+        {
+            labelList nbrDecomp;
+            syncTools::swapBoundaryCellList(mesh, finalDecomp, nbrDecomp);
+
+            const polyBoundaryMesh& patches = mesh.boundaryMesh();
+            forAll(patches, patchI)
+            {
+                const polyPatch& pp = patches[patchI];
+                if (pp.coupled())
+                {
+                    forAll(pp, i)
+                    {
+                        label faceI = pp.start()+i;
+                        label own = mesh.faceOwner()[faceI];
+                        label bFaceI = faceI-mesh.nInternalFaces();
+
+                        if (!blockedFace[faceI])
+                        {
+                            label ownProc = finalDecomp[own];
+                            label nbrProc = nbrDecomp[bFaceI];
+                            if (ownProc != nbrProc)
+                            {
+                                FatalErrorIn("decompositionMethod::decompose()")
+                                    << "patch:" << pp.name()
+                                    << " face:" << faceI
+                                    << " at:" << mesh.faceCentres()[faceI]
+                                    << " ownProc:" << ownProc
+                                    << " nbrProc:" << nbrProc
+                                    << exit(FatalError);
+                            }
+                        }
+                    }
+                }
+            }
+        }
     }
 
     return finalDecomp;

src/parallel/decompose/decompositionMethods/decompositionMethod/minData.H

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2014 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::minData
+
+Description
+    For use with FaceCellWave. Transports minimum passive data
+
+SourceFiles
+    minDataI.H
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef minData_H
+#define minData_H
+
+#include "point.H"
+#include "tensor.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+
+class polyPatch;
+class polyMesh;
+
+/*---------------------------------------------------------------------------*\
+                           Class minData Declaration
+\*---------------------------------------------------------------------------*/
+
+class minData
+{
+    // Private data
+
+        //- Starting data
+        label data_;
+
+
+public:
+
+    // Constructors
+
+        //- Construct null
+        inline minData();
+
+        //- Construct from count
+        inline minData(const label data);
+
+
+    // Member Functions
+
+        // Access
+
+            inline label data() const
+            {
+                return data_;
+            }
+
+
+        // Needed by FaceCellWave
+
+            //- Check whether origin has been changed at all or
+            //  still contains original (invalid) value.
+            template<class TrackingData>
+            inline bool valid(TrackingData& td) const;
+
+            //- Check for identical geometrical data. Used for cyclics checking.
+            template<class TrackingData>
+            inline bool sameGeometry
+            (
+                const polyMesh&,
+                const minData&,
+                const scalar,
+                TrackingData& td
+            ) const;
+
+            //- Convert any absolute coordinates into relative to (patch)face
+            //  centre
+            template<class TrackingData>
+            inline void leaveDomain
+            (
+                const polyMesh&,
+                const polyPatch&,
+                const label patchFaceI,
+                const point& faceCentre,
+                TrackingData& td
+            );
+
+            //- Reverse of leaveDomain
+            template<class TrackingData>
+            inline void enterDomain
+            (
+                const polyMesh&,
+                const polyPatch&,
+                const label patchFaceI,
+                const point& faceCentre,
+                TrackingData& td
+            );
+
+            //- Apply rotation matrix to any coordinates
+            template<class TrackingData>
+            inline void transform
+            (
+                const polyMesh&,
+                const tensor&,
+                TrackingData& td
+            );
+
+            //- Influence of neighbouring face.
+            template<class TrackingData>
+            inline bool updateCell
+            (
+                const polyMesh&,
+                const label thisCellI,
+                const label neighbourFaceI,
+                const minData& neighbourInfo,
+                const scalar tol,
+                TrackingData& td
+            );
+
+            //- Influence of neighbouring cell.
+            template<class TrackingData>
+            inline bool updateFace
+            (
+                const polyMesh&,
+                const label thisFaceI,
+                const label neighbourCellI,
+                const minData& neighbourInfo,
+                const scalar tol,
+                TrackingData& td
+            );
+
+            //- Influence of different value on same face.
+            template<class TrackingData>
+            inline bool updateFace
+            (
+                const polyMesh&,
+                const label thisFaceI,
+                const minData& neighbourInfo,
+                const scalar tol,
+                TrackingData& td
+            );
+
+            //- Same (like operator==)
+            template<class TrackingData>
+            inline bool equal(const minData&, TrackingData& td) const;
+
+    // Member Operators
+
+        // Needed for List IO
+        inline bool operator==(const minData&) const;
+
+        inline bool operator!=(const minData&) const;
+
+
+    // IOstream Operators
+
+        friend Ostream& operator<<(Ostream&, const minData&);
+        friend Istream& operator>>(Istream&, minData&);
+};
+
+
+//- Data associated with minData type are contiguous
+template<>
+inline bool contiguous<minData>()
+{
+    return true;
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#include "minDataI.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //

src/parallel/decompose/decompositionMethods/decompositionMethod/minDataI.H

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2014 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "polyMesh.H"
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+inline Foam::minData::minData()
+:
+    data_(labelMax)
+{}
+
+
+inline Foam::minData::minData(const label data)
+:
+    data_(data)
+{}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+template<class TrackingData>
+inline bool Foam::minData::valid(TrackingData& td) const
+{
+    return data_ != labelMax;
+}
+
+
+template<class TrackingData>
+inline bool Foam::minData::sameGeometry
+(
+    const polyMesh&,
+    const minData&,
+    const scalar,
+    TrackingData&
+) const
+{
+    return true;
+}
+
+
+template<class TrackingData>
+inline void Foam::minData::leaveDomain
+(
+    const polyMesh&,
+    const polyPatch& patch,
+    const label patchFaceI,
+    const point& faceCentre,
+    TrackingData&
+)
+{}
+
+
+template<class TrackingData>
+inline void Foam::minData::transform
+(
+    const polyMesh&,
+    const tensor& rotTensor,
+    TrackingData&
+)
+{}
+
+
+template<class TrackingData>
+inline void Foam::minData::enterDomain
+(
+    const polyMesh&,
+    const polyPatch& patch,
+    const label patchFaceI,
+    const point& faceCentre,
+    TrackingData&
+)
+{}
+
+
+template<class TrackingData>
+inline bool Foam::minData::updateCell
+(
+    const polyMesh&,
+    const label thisCellI,
+    const label neighbourFaceI,
+    const minData& neighbourInfo,
+    const scalar tol,
+    TrackingData&
+)
+{
+    if (neighbourInfo.data_ < data_)
+    {
+        operator=(neighbourInfo);
+        return true;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+
+template<class TrackingData>
+inline bool Foam::minData::updateFace
+(
+    const polyMesh& mesh,
+    const label thisFaceI,
+    const label neighbourCellI,
+    const minData& neighbourInfo,
+    const scalar tol,
+    TrackingData&
+)
+{
+    // From cell to its faces.
+
+    if (neighbourInfo.data_ < data_)
+    {
+        operator=(neighbourInfo);
+        return true;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+
+template<class TrackingData>
+inline bool Foam::minData::updateFace
+(
+    const polyMesh&,
+    const label thisFaceI,
+    const minData& neighbourInfo,
+    const scalar tol,
+    TrackingData&
+)
+{
+    // From face to face (e.g. coupled faces)
+    if (neighbourInfo.data_ < data_)
+    {
+        operator=(neighbourInfo);
+        return true;
+    }
+    else
+    {
+        return false;
+    }
+}
+
+
+template<class TrackingData>
+inline bool Foam::minData::equal
+(
+    const minData& rhs,
+    TrackingData& td
+) const
+{
+    return operator==(rhs);
+}
+
+
+// * * * * * * * * * * * * * * * Member Operators  * * * * * * * * * * * * * //
+
+inline bool Foam::minData::operator==
+(
+    const Foam::minData& rhs
+) const
+{
+    return data() == rhs.data();
+}
+
+
+inline bool Foam::minData::operator!=
+(
+    const Foam::minData& rhs
+) const
+{
+    return !(*this == rhs);
+}
+
+
+// * * * * * * * * * * * * * * * Friend Operators  * * * * * * * * * * * * * //
+
+Foam::Ostream& Foam::operator<<
+(
+    Foam::Ostream& os,
+    const Foam::minData& wDist
+)
+{
+    return os << wDist.data_;
+}
+
+
+Foam::Istream& Foam::operator>>
+(
+    Foam::Istream& is,
+    Foam::minData& wDist
+)
+{
+    return is >> wDist.data_;
+}
+
+
+// ************************************************************************* //