Commit 9f5c39af authored by mattijs's avatar mattijs
Browse files

ENH: have ptscotch

ptscotch - compiles into ptscotchDecomp. All thirdparty decompositionMethods
now moved out of decompositionMethods so add them explicitly to link line
for programs that need them (decomposePar, snappyHexMesh etc.)
parent 536087b5
......@@ -11,6 +11,7 @@ EXE_INC = \
EXE_LIBS = \
-lfiniteVolume \
-ldecompositionMethods \
-L$(FOAM_MPI_LIBBIN) -lptscotchDecomp -lparMetisDecomp -lmetisDecomp \
-lmeshTools \
-ldynamicMesh \
-lautoMesh
......@@ -347,7 +347,7 @@ int main(int argc, char *argv[])
<< "You have selected decomposition method "
<< decomposer.typeName
<< " which is not parallel aware." << endl
<< "Please select one that is (hierarchical, parMetis)"
<< "Please select one that is (hierarchical, ptscotch, parMetis)"
<< exit(FatalError);
}
......
......@@ -7,6 +7,6 @@ EXE_INC = \
EXE_LIBS = \
-lfiniteVolume \
-lgenericPatchFields \
-ldecompositionMethods \
-ldecompositionMethods -lmetisDecomp -lscotchDecomp \
-llagrangian \
-lmeshTools
......@@ -7,5 +7,6 @@ EXE_INC = \
EXE_LIBS = \
-lfiniteVolume \
-ldecompositionMethods \
-L$(FOAM_MPI_LIBBIN) -lptscotchDecomp -lparMetisDecomp -lmetisDecomp \
-lmeshTools \
-ldynamicMesh
......@@ -3,6 +3,7 @@ cd ${0%/*} || exit 1 # run from this directory
set -x
wmake libso scotchDecomp
wmake libso ptscotchDecomp
wmake libso metisDecomp
wmake libso parMetisDecomp
wmake libso MGridGen
......
......@@ -163,23 +163,4 @@ Foam::labelList Foam::parMetisDecomp::decompose
}
void Foam::parMetisDecomp::calcMetisDistributedCSR
(
const polyMesh& mesh,
List<int>& adjncy,
List<int>& xadj
)
{
FatalErrorIn
(
"void parMetisDecomp::calcMetisDistributedCSR"
"("
"const polyMesh&, "
"List<int>&, "
"List<int>&"
")"
) << notImplementedMessage << exit(FatalError);
}
// ************************************************************************* //
dummyPtscotchDecomp.C
LIB = $(FOAM_LIBBIN)/dummy/libptscotchDecomp
EXE_INC = \
-I$(FOAM_SRC)/parallel/decompose/decompositionMethods/lnInclude \
-I$(FOAM_SRC)/parallel/decompose/ptscotchDecomp/lnInclude
LIB_LIBS =
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2010-2010 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by the
Free Software Foundation; either version 2 of the License, or (at your
option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM; if not, write to the Free Software Foundation,
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
\*---------------------------------------------------------------------------*/
#include "ptscotchDecomp.H"
#include "addToRunTimeSelectionTable.H"
#include "Time.H"
static const char* notImplementedMessage =
"You are trying to use ptscotch but do not have the "
"ptscotchDecomp library loaded."
"\nThis message is from the dummy ptscotchDecomp stub library instead.\n"
"\n"
"Please install ptscotch and make sure that libptscotch.so is in your "
"LD_LIBRARY_PATH.\n"
"The ptscotchDecomp library can then be built in "
"$FOAM_SRC/parallel/decompose/ptscotchDecomp\n";
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{
defineTypeNameAndDebug(ptscotchDecomp, 0);
addToRunTimeSelectionTable
(
decompositionMethod,
ptscotchDecomp,
dictionaryMesh
);
}
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
void Foam::ptscotchDecomp::check(const int retVal, const char* str)
{}
Foam::label Foam::ptscotchDecomp::decompose
(
List<int>& adjncy,
List<int>& xadj,
const scalarField& cWeights,
List<int>& finalDecomp
)
{
FatalErrorIn
(
"label ptscotchDecomp::decompose"
"("
"const List<int>&, "
"const List<int>&, "
"const scalarField&, "
"List<int>&"
")"
) << notImplementedMessage << exit(FatalError);
return -1;
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::ptscotchDecomp::ptscotchDecomp
(
const dictionary& decompositionDict,
const polyMesh& mesh
)
:
decompositionMethod(decompositionDict),
mesh_(mesh)
{}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::labelList Foam::ptscotchDecomp::decompose
(
const pointField& points,
const scalarField& pointWeights
)
{
FatalErrorIn
(
"labelList ptscotchDecomp::decompose"
"("
"const pointField&, "
"const scalarField&"
")"
) << notImplementedMessage << exit(FatalError);
return labelList::null();
}
Foam::labelList Foam::ptscotchDecomp::decompose
(
const labelList& agglom,
const pointField& agglomPoints,
const scalarField& pointWeights
)
{
FatalErrorIn
(
"labelList ptscotchDecomp::decompose"
"("
"const labelList&, "
"const pointField&, "
"const scalarField&"
")"
) << notImplementedMessage << exit(FatalError);
return labelList::null();
}
Foam::labelList Foam::ptscotchDecomp::decompose
(
const labelListList& globalCellCells,
const pointField& cellCentres,
const scalarField& cWeights
)
{
FatalErrorIn
(
"labelList ptscotchDecomp::decompose"
"("
"const labelListList&, "
"const pointField&, "
"const scalarField&"
")"
) << notImplementedMessage << exit(FatalError);
return labelList::null();
}
// ************************************************************************* //
......@@ -22,87 +22,6 @@ License
along with OpenFOAM; if not, write to the Free Software Foundation,
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
From scotch forum:
By: Francois PELLEGRINI RE: Graph mapping 'strategy' string [ reply ]
2008-08-22 10:09 Strategy handling in Scotch is a bit tricky. In order
not to be confused, you must have a clear view of how they are built.
Here are some rules:
1- Strategies are made up of "methods" which are combined by means of
"operators".
2- A method is of the form "m{param=value,param=value,...}", where "m"
is a single character (this is your first error: "f" is a method name,
not a parameter name).
3- There exist different sort of strategies : bipartitioning strategies,
mapping strategies, ordering strategies, which cannot be mixed. For
instance, you cannot build a bipartitioning strategy and feed it to a
mapping method (this is your second error).
To use the "mapCompute" routine, you must create a mapping strategy, not
a bipartitioning one, and so use stratGraphMap() and not
stratGraphBipart(). Your mapping strategy should however be based on the
"recursive bipartitioning" method ("b"). For instance, a simple (and
hence not very efficient) mapping strategy can be :
"b{sep=f}"
which computes mappings with the recursive bipartitioning method "b",
this latter using the Fiduccia-Mattheyses method "f" to compute its
separators.
If you want an exact partition (see your previous post), try
"b{sep=fx}".
However, these strategies are not the most efficient, as they do not
make use of the multi-level framework.
To use the multi-level framework, try for instance:
"b{sep=m{vert=100,low=h,asc=f}x}"
The current default mapping strategy in Scotch can be seen by using the
"-vs" option of program gmap. It is, to date:
b
{
job=t,
map=t,
poli=S,
sep=
(
m
{
asc=b
{
bnd=d{pass=40,dif=1,rem=1}f{move=80,pass=-1,bal=0.005},
org=f{move=80,pass=-1,bal=0.005},
width=3
},
low=h{pass=10}f{move=80,pass=-1,bal=0.0005},
type=h,
vert=80,
rat=0.8
}
| m
{
asc=b
{
bnd=d{pass=40,dif=1,rem=1}f{move=80,pass=-1,bal=0.005},
org=f{move=80,pass=-1,bal=0.005},
width=3
},
low=h{pass=10}f{move=80,pass=-1,bal=0.0005},
type=h,
vert=80,
rat=0.8
}
)
}
\*---------------------------------------------------------------------------*/
#include "scotchDecomp.H"
......@@ -239,42 +158,4 @@ Foam::labelList Foam::scotchDecomp::decompose
}
void Foam::scotchDecomp::calcCSR
(
const polyMesh& mesh,
List<int>& adjncy,
List<int>& xadj
)
{
FatalErrorIn
(
"labelList scotchDecomp::decompose"
"("
"const polyMesh&, "
"const List<int>&, "
"const List<int>&"
")"
) << notImplementedMessage << exit(FatalError);
}
void Foam::scotchDecomp::calcCSR
(
const labelListList& cellCells,
List<int>& adjncy,
List<int>& xadj
)
{
FatalErrorIn
(
"labelList scotchDecomp::decompose"
"("
"const labelListList&, "
"const List<int>&, "
"const List<int>&"
")"
) << notImplementedMessage << exit(FatalError);
}
// ************************************************************************* //
......@@ -9,7 +9,7 @@ wmake libso metisDecomp
if [ -d "$FOAM_MPI_LIBBIN" ]
then
( WM_OPTIONS=${WM_OPTIONS}$WM_MPLIB; wmake libso parMetisDecomp )
( WM_OPTIONS=${WM_OPTIONS}$WM_MPLIB; wmake libso ptscotchDecomp && wmake libso parMetisDecomp )
fi
wmake libso decompositionMethods
......
......@@ -6,5 +6,6 @@ wmakeLnInclude decompositionMethods
wmakeLnInclude metisDecomp
wmakeLnInclude parMetisDecomp
wmakeLnInclude scotchDecomp
wmakeLnInclude ptscotchDecomp
# ----------------------------------------------------------------- end-of-file
EXE_INC =
LIB_LIBS = \
-L$(FOAM_LIBBIN)/dummy \
-L$(FOAM_MPI_LIBBIN) \
-lscotchDecomp \
-lmetisDecomp \
-lparMetisDecomp
/* -L$(FOAM_LIBBIN)/dummy */ \
/* -L$(FOAM_MPI_LIBBIN) */ \
/* -lscotchDecomp */ \
/* -lmetisDecomp */ \
/* -lparMetisDecomp */
......@@ -28,6 +28,9 @@ InClass
\*---------------------------------------------------------------------------*/
#include "decompositionMethod.H"
#include "globalIndex.H"
#include "cyclicPolyPatch.H"
#include "syncTools.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
......@@ -156,6 +159,18 @@ Foam::labelList Foam::decompositionMethod::decompose
}
Foam::labelList Foam::decompositionMethod::decompose
(
const labelListList& globalCellCells,
const pointField& cc
)
{
scalarField cWeights(0);
return decompose(globalCellCells, cc, cWeights);
}
void Foam::decompositionMethod::calcCellCells
(
const polyMesh& mesh,
......@@ -201,15 +216,284 @@ void Foam::decompositionMethod::calcCellCells
}
Foam::labelList Foam::decompositionMethod::decompose
void Foam::decompositionMethod::calcCSR
(
const labelListList& globalCellCells,
const pointField& cc
const polyMesh& mesh,
List<int>& adjncy,
List<int>& xadj
)
{
scalarField cWeights(0);
// Make Metis CSR (Compressed Storage Format) storage
// adjncy : contains neighbours (= edges in graph)
// xadj(celli) : start of information in adjncy for celli
return decompose(globalCellCells, cc, cWeights);
xadj.setSize(mesh.nCells()+1);
// Initialise the number of internal faces of the cells to twice the
// number of internal faces
label nInternalFaces = 2*mesh.nInternalFaces();
// Check the boundary for coupled patches and add to the number of
// internal faces
const polyBoundaryMesh& pbm = mesh.boundaryMesh();
forAll(pbm, patchi)
{
if (isA<cyclicPolyPatch>(pbm[patchi]))
{
nInternalFaces += pbm[patchi].size();
}
}
// Create the adjncy array the size of the total number of internal and
// coupled faces
adjncy.setSize(nInternalFaces);
// Fill in xadj
// ~~~~~~~~~~~~
label freeAdj = 0;
for (label cellI = 0; cellI < mesh.nCells(); cellI++)
{
xadj[cellI] = freeAdj;
const labelList& cFaces = mesh.cells()[cellI];
forAll(cFaces, i)
{
label faceI = cFaces[i];
if
(
mesh.isInternalFace(faceI)
|| isA<cyclicPolyPatch>(pbm[pbm.whichPatch(faceI)])
)
{
freeAdj++;
}
}
}
xadj[mesh.nCells()] = freeAdj;
// Fill in adjncy
// ~~~~~~~~~~~~~~
labelList nFacesPerCell(mesh.nCells(), 0);
// Internal faces
for (label faceI = 0; faceI < mesh.nInternalFaces(); faceI++)
{
label own = mesh.faceOwner()[faceI];
label nei = mesh.faceNeighbour()[faceI];
adjncy[xadj[own] + nFacesPerCell[own]++] = nei;
adjncy[xadj[nei] + nFacesPerCell[nei]++] = own;
}
// Coupled faces. Only cyclics done.
forAll(pbm, patchi)
{
if (isA<cyclicPolyPatch>(pbm[patchi]))
{
const unallocLabelList& faceCells = pbm[patchi].faceCells();
label sizeby2 = faceCells.size()/2;
for (label facei=0; facei<sizeby2; facei++)
{
label own = faceCells[facei];
label nei = faceCells[facei + sizeby2];
adjncy[xadj[own] + nFacesPerCell[own]++] = nei;
adjncy[xadj[nei] + nFacesPerCell[nei]++] = own;
}
}
}
}
// From cell-cell connections to Metis format (like CompactListList)
void Foam::decompositionMethod::calcCSR
(
const labelListList& cellCells,
List<int>& adjncy,
List<int>& xadj
)
{
// Count number of internal faces
label nConnections = 0;
forAll(cellCells, coarseI)
{
nConnections += cellCells[coarseI].size();
}
// Create the adjncy array as twice the size of the total number of
// internal faces
adjncy.setSize(nConnections);
xadj.setSize(cellCells.size()+1);
// Fill in xadj
// ~~~~~~~~~~~~
label freeAdj = 0;
forAll(cellCells, coarseI)
{
xadj[coarseI] = freeAdj;
const labelList& cCells = cellCells[coarseI];
forAll(cCells, i)
{
adjncy[freeAdj++] = cCells[i];
}
}
xadj[cellCells.size()] = freeAdj;
}
void Foam::decompositionMethod::calcDistributedCSR
(
const polyMesh& mesh,
List<int>& adjncy,
List<int>& xadj
)
{
// Create global cell numbers
// ~~~~~~~~~~~~~~~~~~~~~~~~~~
globalIndex globalCells(mesh.nCells());
//
// Make Metis Distributed CSR (Compressed Storage Format) storage
// adjncy : contains cellCells (= edges in graph)
// xadj(celli) : start of information in adjncy for celli
//
const labelList& faceOwner = mesh.faceOwner();
const labelList& faceNeighbour = mesh.faceNeighbour();