ENH: have ptscotch

ptscotch - compiles into ptscotchDecomp. All thirdparty decompositionMethods
now moved out of decompositionMethods so add them explicitly to link line
for programs that need them (decomposePar, snappyHexMesh etc.)

applications/utilities/mesh/generation/snappyHexMesh/Make/options

@@ -11,6 +11,7 @@ EXE_INC = \
 EXE_LIBS = \
     -lfiniteVolume \
     -ldecompositionMethods \
+    -L$(FOAM_MPI_LIBBIN) -lptscotchDecomp -lparMetisDecomp -lmetisDecomp \
     -lmeshTools \
     -ldynamicMesh \
     -lautoMesh

applications/utilities/mesh/generation/snappyHexMesh/snappyHexMesh.C

@@ -347,7 +347,7 @@ int main(int argc, char *argv[])
             << "You have selected decomposition method "
             << decomposer.typeName
             << " which is not parallel aware." << endl
-            << "Please select one that is (hierarchical, parMetis)"
+            << "Please select one that is (hierarchical, ptscotch, parMetis)"
             << exit(FatalError);
     }
 

applications/utilities/parallelProcessing/decomposePar/Make/options

@@ -7,6 +7,6 @@ EXE_INC = \
 EXE_LIBS = \
     -lfiniteVolume \
     -lgenericPatchFields \
-    -ldecompositionMethods \
+    -ldecompositionMethods -lmetisDecomp -lscotchDecomp \
     -llagrangian \
     -lmeshTools

applications/utilities/parallelProcessing/redistributeMeshPar/Make/options

@@ -7,5 +7,6 @@ EXE_INC = \
 EXE_LIBS = \
     -lfiniteVolume \
     -ldecompositionMethods \
+    -L$(FOAM_MPI_LIBBIN) -lptscotchDecomp -lparMetisDecomp -lmetisDecomp \
     -lmeshTools \
     -ldynamicMesh

src/dummyThirdParty/Allwmake

@@ -3,6 +3,7 @@ cd ${0%/*} || exit 1    # run from this directory
 set -x
 
 wmake libso scotchDecomp
+wmake libso ptscotchDecomp
 wmake libso metisDecomp
 wmake libso parMetisDecomp
 wmake libso MGridGen

src/dummyThirdParty/parMetisDecomp/dummyParMetisDecomp.C

@@ -163,23 +163,4 @@ Foam::labelList Foam::parMetisDecomp::decompose
 }
 
 
-void Foam::parMetisDecomp::calcMetisDistributedCSR
-(
-    const polyMesh& mesh,
-    List<int>& adjncy,
-    List<int>& xadj
-)
-{
-    FatalErrorIn
-    (
-        "void parMetisDecomp::calcMetisDistributedCSR"
-        "("
-            "const polyMesh&, "
-            "List<int>&, "
-            "List<int>&"
-        ")"
-    )   << notImplementedMessage << exit(FatalError);
-}
-
-
 // ************************************************************************* //

src/dummyThirdParty/ptscotchDecomp/Make/files

+dummyPtscotchDecomp.C
+
+LIB = $(FOAM_LIBBIN)/dummy/libptscotchDecomp

src/dummyThirdParty/ptscotchDecomp/Make/options

+EXE_INC = \
+    -I$(FOAM_SRC)/parallel/decompose/decompositionMethods/lnInclude \
+    -I$(FOAM_SRC)/parallel/decompose/ptscotchDecomp/lnInclude
+
+LIB_LIBS =

src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C

+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2010-2010 OpenCFD Ltd.
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software; you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by the
+    Free Software Foundation; either version 2 of the License, or (at your
+    option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM; if not, write to the Free Software Foundation,
+    Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+
+\*---------------------------------------------------------------------------*/
+
+#include "ptscotchDecomp.H"
+#include "addToRunTimeSelectionTable.H"
+#include "Time.H"
+
+static const char* notImplementedMessage =
+"You are trying to use ptscotch but do not have the "
+"ptscotchDecomp library loaded."
+"\nThis message is from the dummy ptscotchDecomp stub library instead.\n"
+"\n"
+"Please install ptscotch and make sure that libptscotch.so is in your "
+"LD_LIBRARY_PATH.\n"
+"The ptscotchDecomp library can then be built in "
+"$FOAM_SRC/parallel/decompose/ptscotchDecomp\n";
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+    defineTypeNameAndDebug(ptscotchDecomp, 0);
+
+    addToRunTimeSelectionTable
+    (
+        decompositionMethod,
+        ptscotchDecomp,
+        dictionaryMesh
+    );
+}
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+void Foam::ptscotchDecomp::check(const int retVal, const char* str)
+{}
+
+
+Foam::label Foam::ptscotchDecomp::decompose
+(
+    List<int>& adjncy,
+    List<int>& xadj,
+    const scalarField& cWeights,
+    List<int>& finalDecomp
+)
+{
+    FatalErrorIn
+    (
+        "label ptscotchDecomp::decompose"
+        "("
+            "const List<int>&, "
+            "const List<int>&, "
+            "const scalarField&, "
+            "List<int>&"
+        ")"
+    )   << notImplementedMessage << exit(FatalError);
+
+    return -1;
+}
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::ptscotchDecomp::ptscotchDecomp
+(
+    const dictionary& decompositionDict,
+    const polyMesh& mesh
+)
+:
+    decompositionMethod(decompositionDict),
+    mesh_(mesh)
+{}
+
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+Foam::labelList Foam::ptscotchDecomp::decompose
+(
+    const pointField& points,
+    const scalarField& pointWeights
+)
+{
+    FatalErrorIn
+    (
+        "labelList ptscotchDecomp::decompose"
+        "("
+            "const pointField&, "
+            "const scalarField&"
+        ")"
+    )   << notImplementedMessage << exit(FatalError);
+
+    return labelList::null();
+}
+
+
+Foam::labelList Foam::ptscotchDecomp::decompose
+(
+    const labelList& agglom,
+    const pointField& agglomPoints,
+    const scalarField& pointWeights
+)
+{
+    FatalErrorIn
+    (
+        "labelList ptscotchDecomp::decompose"
+        "("
+            "const labelList&, "
+            "const pointField&, "
+            "const scalarField&"
+        ")"
+    )   << notImplementedMessage << exit(FatalError);
+
+    return labelList::null();
+}
+
+
+Foam::labelList Foam::ptscotchDecomp::decompose
+(
+    const labelListList& globalCellCells,
+    const pointField& cellCentres,
+    const scalarField& cWeights
+)
+{
+    FatalErrorIn
+    (
+        "labelList ptscotchDecomp::decompose"
+        "("
+            "const labelListList&, "
+            "const pointField&, "
+            "const scalarField&"
+        ")"
+    )   << notImplementedMessage << exit(FatalError);
+
+    return labelList::null();
+}
+
+
+// ************************************************************************* //

src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C

@@ -22,87 +22,6 @@ License
     along with OpenFOAM; if not, write to the Free Software Foundation,
     Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
 
-    From scotch forum:
-
-    By: Francois PELLEGRINI RE: Graph mapping 'strategy' string [ reply ]
-    2008-08-22 10:09 Strategy handling in Scotch is a bit tricky. In order
-    not to be confused, you must have a clear view of how they are built.
-    Here are some rules:
-
-    1- Strategies are made up of "methods" which are combined by means of
-    "operators".
-
-    2- A method is of the form "m{param=value,param=value,...}", where "m"
-    is a single character (this is your first error: "f" is a method name,
-    not a parameter name).
-
-    3- There exist different sort of strategies : bipartitioning strategies,
-    mapping strategies, ordering strategies, which cannot be mixed. For
-    instance, you cannot build a bipartitioning strategy and feed it to a
-    mapping method (this is your second error).
-
-    To use the "mapCompute" routine, you must create a mapping strategy, not
-    a bipartitioning one, and so use stratGraphMap() and not
-    stratGraphBipart(). Your mapping strategy should however be based on the
-    "recursive bipartitioning" method ("b"). For instance, a simple (and
-    hence not very efficient) mapping strategy can be :
-
-    "b{sep=f}"
-
-    which computes mappings with the recursive bipartitioning method "b",
-    this latter using the Fiduccia-Mattheyses method "f" to compute its
-    separators.
-
-    If you want an exact partition (see your previous post), try
-    "b{sep=fx}".
-
-    However, these strategies are not the most efficient, as they do not
-    make use of the multi-level framework.
-
-    To use the multi-level framework, try for instance:
-
-    "b{sep=m{vert=100,low=h,asc=f}x}"
-
-    The current default mapping strategy in Scotch can be seen by using the
-    "-vs" option of program gmap. It is, to date:
-
-    b
-    {
-        job=t,
-        map=t,
-        poli=S,
-        sep=
-        (
-            m
-            {
-                asc=b
-                {
-                    bnd=d{pass=40,dif=1,rem=1}f{move=80,pass=-1,bal=0.005},
-                    org=f{move=80,pass=-1,bal=0.005},
-                    width=3
-                },
-                low=h{pass=10}f{move=80,pass=-1,bal=0.0005},
-                type=h,
-                vert=80,
-                rat=0.8
-            }
-          | m
-            {
-                asc=b
-                {
-                    bnd=d{pass=40,dif=1,rem=1}f{move=80,pass=-1,bal=0.005},
-                    org=f{move=80,pass=-1,bal=0.005},
-                    width=3
-                },
-                low=h{pass=10}f{move=80,pass=-1,bal=0.0005},
-                type=h,
-                vert=80,
-                rat=0.8
-            }
-        )
-    }
-
-
 \*---------------------------------------------------------------------------*/
 
 #include "scotchDecomp.H"
@@ -239,42 +158,4 @@ Foam::labelList Foam::scotchDecomp::decompose
 }
 
 
-void Foam::scotchDecomp::calcCSR
-(
-    const polyMesh& mesh,
-    List<int>& adjncy,
-    List<int>& xadj
-)
-{
-    FatalErrorIn
-    (
-        "labelList scotchDecomp::decompose"
-        "("
-            "const polyMesh&, "
-            "const List<int>&, "
-            "const List<int>&"
-        ")"
-    )   << notImplementedMessage << exit(FatalError);
-}
-
-
-void Foam::scotchDecomp::calcCSR
-(
-    const labelListList& cellCells,
-    List<int>& adjncy,
-    List<int>& xadj
-)
-{
-    FatalErrorIn
-    (
-        "labelList scotchDecomp::decompose"
-        "("
-            "const labelListList&, "
-            "const List<int>&, "
-            "const List<int>&"
-        ")"
-    )   << notImplementedMessage << exit(FatalError);
-}
-
-
 // ************************************************************************* //

src/parallel/decompose/Allwmake

@@ -9,7 +9,7 @@ wmake libso metisDecomp
 
 if [ -d "$FOAM_MPI_LIBBIN" ]
 then
-    ( WM_OPTIONS=${WM_OPTIONS}$WM_MPLIB; wmake libso parMetisDecomp )
+    ( WM_OPTIONS=${WM_OPTIONS}$WM_MPLIB; wmake libso ptscotchDecomp && wmake libso parMetisDecomp )
 fi
 
 wmake libso decompositionMethods

src/parallel/decompose/AllwmakeLnInclude

@@ -6,5 +6,6 @@ wmakeLnInclude decompositionMethods
 wmakeLnInclude metisDecomp
 wmakeLnInclude parMetisDecomp
 wmakeLnInclude scotchDecomp
+wmakeLnInclude ptscotchDecomp
 
 # ----------------------------------------------------------------- end-of-file

src/parallel/decompose/decompositionMethods/Make/options

 EXE_INC =
 
 LIB_LIBS = \
-    -L$(FOAM_LIBBIN)/dummy \
-    -L$(FOAM_MPI_LIBBIN) \
-    -lscotchDecomp \
-    -lmetisDecomp \
-    -lparMetisDecomp
+    /* -L$(FOAM_LIBBIN)/dummy */ \
+    /* -L$(FOAM_MPI_LIBBIN) */ \
+    /* -lscotchDecomp */ \
+    /* -lmetisDecomp */ \
+    /* -lparMetisDecomp */
 

src/parallel/decompose/decompositionMethods/decompositionMethod/decompositionMethod.C

@@ -28,6 +28,9 @@ InClass
 \*---------------------------------------------------------------------------*/
 
 #include "decompositionMethod.H"
+#include "globalIndex.H"
+#include "cyclicPolyPatch.H"
+#include "syncTools.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -156,6 +159,18 @@ Foam::labelList Foam::decompositionMethod::decompose
 }
 
 
+Foam::labelList Foam::decompositionMethod::decompose
+(
+    const labelListList& globalCellCells,
+    const pointField& cc
+)
+{
+    scalarField cWeights(0);
+
+    return decompose(globalCellCells, cc, cWeights);
+}
+
+
 void Foam::decompositionMethod::calcCellCells
 (
     const polyMesh& mesh,
@@ -201,15 +216,284 @@ void Foam::decompositionMethod::calcCellCells
 }
 
 
-Foam::labelList Foam::decompositionMethod::decompose
+void Foam::decompositionMethod::calcCSR
 (
-    const labelListList& globalCellCells,
-    const pointField& cc
+    const polyMesh& mesh,
+    List<int>& adjncy,
+    List<int>& xadj
 )
 {
-    scalarField cWeights(0);
+    // Make Metis CSR (Compressed Storage Format) storage
+    //   adjncy      : contains neighbours (= edges in graph)
+    //   xadj(celli) : start of information in adjncy for celli
 
-    return decompose(globalCellCells, cc, cWeights);
+    xadj.setSize(mesh.nCells()+1);
+
+    // Initialise the number of internal faces of the cells to twice the
+    // number of internal faces
+    label nInternalFaces = 2*mesh.nInternalFaces();
+
+    // Check the boundary for coupled patches and add to the number of
+    // internal faces
+    const polyBoundaryMesh& pbm = mesh.boundaryMesh();
+
+    forAll(pbm, patchi)
+    {
+        if (isA<cyclicPolyPatch>(pbm[patchi]))
+        {
+            nInternalFaces += pbm[patchi].size();
+        }
+    }
+
+    // Create the adjncy array the size of the total number of internal and
+    // coupled faces
+    adjncy.setSize(nInternalFaces);
+
+    // Fill in xadj
+    // ~~~~~~~~~~~~
+    label freeAdj = 0;
+
+    for (label cellI = 0; cellI < mesh.nCells(); cellI++)
+    {
+        xadj[cellI] = freeAdj;
+
+        const labelList& cFaces = mesh.cells()[cellI];
+
+        forAll(cFaces, i)
+        {
+            label faceI = cFaces[i];
+
+            if
+            (
+                mesh.isInternalFace(faceI)
+             || isA<cyclicPolyPatch>(pbm[pbm.whichPatch(faceI)])
+            )
+            {
+                freeAdj++;
+            }
+        }
+    }
+    xadj[mesh.nCells()] = freeAdj;
+
+
+    // Fill in adjncy
+    // ~~~~~~~~~~~~~~
+
+    labelList nFacesPerCell(mesh.nCells(), 0);
+
+    // Internal faces
+    for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++)
+    {
+        label own = mesh.faceOwner()[faceI];
+        label nei = mesh.faceNeighbour()[faceI];
+
+        adjncy[xadj[own] + nFacesPerCell[own]++] = nei;
+        adjncy[xadj[nei] + nFacesPerCell[nei]++] = own;
+    }
+
+    // Coupled faces. Only cyclics done.
+    forAll(pbm, patchi)
+    {
+        if (isA<cyclicPolyPatch>(pbm[patchi]))
+        {
+            const unallocLabelList& faceCells = pbm[patchi].faceCells();
+
+            label sizeby2 = faceCells.size()/2;
+
+            for (label facei=0; facei<sizeby2; facei++)
+            {
+                label own = faceCells[facei];
+                label nei = faceCells[facei + sizeby2];
+
+                adjncy[xadj[own] + nFacesPerCell[own]++] = nei;
+                adjncy[xadj[nei] + nFacesPerCell[nei]++] = own;
+            }
+        }
+    }
+}
+
+
+// From cell-cell connections to Metis format (like CompactListList)
+void Foam::decompositionMethod::calcCSR
+(
+    const labelListList& cellCells,
+    List<int>& adjncy,
+    List<int>& xadj
+)
+{
+    // Count number of internal faces
+    label nConnections = 0;
+
+    forAll(cellCells, coarseI)
+    {
+        nConnections += cellCells[coarseI].size();
+    }
+
+    // Create the adjncy array as twice the size of the total number of
+    // internal faces
+    adjncy.setSize(nConnections);
+
+    xadj.setSize(cellCells.size()+1);
+
+
+    // Fill in xadj
+    // ~~~~~~~~~~~~
+    label freeAdj = 0;
+
+    forAll(cellCells, coarseI)
+    {
+        xadj[coarseI] = freeAdj;
+
+        const labelList& cCells = cellCells[coarseI];
+
+        forAll(cCells, i)
+        {
+            adjncy[freeAdj++] = cCells[i];
+        }
+    }
+    xadj[cellCells.size()] = freeAdj;
+}
+
+
+void Foam::decompositionMethod::calcDistributedCSR
+(
+    const polyMesh& mesh,
+    List<int>& adjncy,
+    List<int>& xadj
+)
+{
+    // Create global cell numbers
+    // ~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    globalIndex globalCells(mesh.nCells());
+
+
+    //
+    // Make Metis Distributed CSR (Compressed Storage Format) storage
+    //   adjncy      : contains cellCells (= edges in graph)
+    //   xadj(celli) : start of information in adjncy for celli
+    //
+
+
+    const labelList& faceOwner = mesh.faceOwner();
+    const labelList& faceNeighbour = mesh.faceNeighbour();