Skip to content

GitLab

Commit 33d6ea3e authored by mattijs's avatar mattijs
Browse files

ENH: comunicators: have solver with user-defined communicator 

(originating from polyMesh::comm() and processorPolyPatch::comm())
parent ea8d2901
Hide whitespace changes
Inline Side-by-side
Showing with 328 additions and 89 deletions
src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -48,6 +48,12 @@ void reduce
const label comm
)
{
if (UPstream::warnComm != -1 && comm != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << comm
<< endl;
error::printStack(Pout);
}
Pstream::gather(comms, Value, bop, tag, comm);
Pstream::scatter(comms, Value, tag, comm);
}
......
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -270,6 +270,19 @@ Foam::label Foam::UPstream::allocateCommunicator
forAll(procIDs_[index], i)
{
procIDs_[index][i] = subRanks[i];
// Enforce incremental order (so index is rank in next communicator)
if (i >= 1 && subRanks[i] <= subRanks[i-1])
{
FatalErrorIn
(
"UPstream::allocateCommunicator"
"(const label, const labelList&, const bool)"
) << "subranks not sorted : " << subRanks
<< " when allocating subcommunicator from parent "
<< parentIndex
<< Foam::abort(FatalError);
}
}
parentCommunicator_[index] = parentIndex;
......@@ -277,7 +290,7 @@ Foam::label Foam::UPstream::allocateCommunicator
treeCommunication_[index] = calcTreeComm(procIDs_[index].size());
if (doPstream)
if (doPstream && parRun())
{
allocatePstreamCommunicator(parentIndex, index);
}
......@@ -297,7 +310,7 @@ void Foam::UPstream::freeCommunicator
<< " myProcNo : " << myProcNo_[communicator] << endl
<< endl;
if (doPstream)
if (doPstream && parRun())
{
freePstreamCommunicator(communicator);
}
......@@ -437,6 +450,10 @@ addcommsTypeToOpt addcommsTypeToOpt_("commsType");
Foam::label Foam::UPstream::worldComm(0);
// Warn for use of any communicator
Foam::label Foam::UPstream::warnComm(-1);
// Number of polling cycles in processor updates
int Foam::UPstream::nPollProcInterfaces
(
......
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2012-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -269,6 +269,10 @@ public:
//- Default communicator (all processors)
static label worldComm;
//- Debugging: warn for use of any communicator differing from warnComm
static label warnComm;
// Constructors
//- Construct given optional buffer size
......@@ -319,17 +323,17 @@ public:
comm_(allocateCommunicator(parent, subRanks, doPstream))
{}
communicator(const label parent)
:
comm_
(
allocateCommunicator
(
parent,
identity(UPstream::nProcs(parent))
)
)
{}
// communicator(const label parent)
// :
// comm_
// (
// allocateCommunicator
// (
// parent,
// identity(UPstream::nProcs(parent))
// )
// )
// {}
~communicator()
{
......@@ -407,17 +411,24 @@ public:
return myProcNo_[communicator];
}
static label parent(const label communicator)
{
return parentCommunicator_(communicator);
}
// //- Process IDs
// static const List<int>& procIDs()
// {
// return procIDs_;
// }
//
// //- Process ID of given process index
// static int procID(int procNo)
// {
// return procIDs_[procNo];
// }
//- Process ID of given process index
static List<int>& procID(int communicator)
{
return procIDs_[communicator];
}
//- Process index of first slave
static int firstSlave()
......
src/OpenFOAM/db/error/messageStream.C
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -166,6 +166,74 @@ Foam::OSstream& Foam::messageStream::operator()
}
Foam::OSstream& Foam::messageStream::operator()(const label communicator)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** messageStream with comm:" << communicator
<< endl;
error::printStack(Pout);
}
if (communicator == UPstream::worldComm)
{
return operator()();
}
else
{
bool master = UPstream::master(communicator);
if (level)
{
bool collect = (severity_ == INFO || severity_ == WARNING);
// Report the error
if (!master && collect)
{
return Snull;
}
else
{
if (title().size())
{
if (Pstream::parRun() && !collect)
{
Pout<< title().c_str();
}
else
{
Sout<< title().c_str();
}
}
if (maxErrors_)
{
errorCount_++;
if (errorCount_ >= maxErrors_)
{
FatalErrorIn("messageStream::operator OSstream&()")
<< "Too many errors"
<< abort(FatalError);
}
}
if (Pstream::parRun() && !collect)
{
return Pout;
}
else
{
return Sout;
}
}
}
}
return Snull;
}
Foam::messageStream::operator Foam::OSstream&()
{
if (level)
......
src/OpenFOAM/db/error/messageStream.H
View file @ 33d6ea3e
......@@ -184,6 +184,11 @@ public:
const dictionary&
);
//- Convert to OSstream
// Use Info for default communicator, use Pout
// on master for non-default one.
OSstream& operator()(const label communicator);
//- Convert to OSstream for << operations
operator OSstream&();
......
src/OpenFOAM/fields/Fields/Field/FieldFunctions.C
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -477,10 +477,10 @@ TMP_UNARY_FUNCTION(Type, average)
#define G_UNARY_FUNCTION(ReturnType, gFunc, Func, rFunc) \
\
template<class Type> \
ReturnType gFunc(const UList<Type>& f) \
ReturnType gFunc(const UList<Type>& f, const int comm) \
{ \
ReturnType res = Func(f); \
reduce(res, rFunc##Op<Type>()); \
reduce(res, rFunc##Op<Type>(), Pstream::msgType(), comm); \
return res; \
} \
TMP_UNARY_FUNCTION(ReturnType, gFunc)
......@@ -495,27 +495,41 @@ G_UNARY_FUNCTION(Type, gSumCmptMag, sumCmptMag, sum)
#undef G_UNARY_FUNCTION
template<class Type>
scalar gSumProd(const UList<Type>& f1, const UList<Type>& f2)
scalar gSumProd
(
const UList<Type>& f1,
const UList<Type>& f2,
const int comm
)
{
scalar SumProd = sumProd(f1, f2);
reduce(SumProd, sumOp<scalar>());
reduce(SumProd, sumOp<scalar>(), Pstream::msgType(), comm);
return SumProd;
}
template<class Type>
Type gSumCmptProd(const UList<Type>& f1, const UList<Type>& f2)
Type gSumCmptProd
(
const UList<Type>& f1,
const UList<Type>& f2,
const int comm
)
{
Type SumProd = sumCmptProd(f1, f2);
reduce(SumProd, sumOp<Type>());
reduce(SumProd, sumOp<Type>(), Pstream::msgType(), comm);
return SumProd;
}
template<class Type>
Type gAverage(const UList<Type>& f)
Type gAverage
(
const UList<Type>& f,
const int comm
)
{
label n = f.size();
Type s = sum(f);
sumReduce(s, n);
sumReduce(s, n, Pstream::msgType(), comm);
if (n > 0)
{
......
src/OpenFOAM/fields/Fields/Field/FieldFunctions.H
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -25,6 +25,7 @@ License
#define TEMPLATE template<class Type>
#include "FieldFunctionsM.H"
#include "UPstream.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
......@@ -201,7 +202,7 @@ TMP_UNARY_FUNCTION(Type, average)
#define G_UNARY_FUNCTION(ReturnType, gFunc, Func, rFunc) \
\
template<class Type> \
ReturnType gFunc(const UList<Type>& f); \
ReturnType gFunc(const UList<Type>& f, const int comm = UPstream::worldComm); \
TMP_UNARY_FUNCTION(ReturnType, gFunc)
G_UNARY_FUNCTION(Type, gMax, max, max)
......@@ -214,13 +215,27 @@ G_UNARY_FUNCTION(Type, gSumCmptMag, sumCmptMag, sum)
#undef G_UNARY_FUNCTION
template<class Type>
scalar gSumProd(const UList<Type>& f1, const UList<Type>& f2);
scalar gSumProd
(
const UList<Type>& f1,
const UList<Type>& f2,
const int comm = UPstream::worldComm
);
template<class Type>
Type gSumCmptProd(const UList<Type>& f1, const UList<Type>& f2);
Type gSumCmptProd
(
const UList<Type>& f1,
const UList<Type>& f2,
const int comm = UPstream::worldComm
);
template<class Type>
Type gAverage(const UList<Type>& f);
Type gAverage
(
const UList<Type>& f,
const int comm = UPstream::worldComm
);
TMP_UNARY_FUNCTION(Type, gAverage)
......
src/OpenFOAM/matrices/LduMatrix/LduMatrix/SolverPerformance.C
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -24,6 +24,7 @@ License
\*---------------------------------------------------------------------------*/
#include "SolverPerformance.H"
#include "IOstreams.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
......@@ -64,7 +65,8 @@ bool Foam::SolverPerformance<Type>::checkConvergence
{
if (debug >= 2)
{
Info<< solverName_
//Info<< solverName_
Pout<< solverName_
<< ": Iteration " << noIterations_
<< " residual = " << finalResidual_
<< endl;
......
src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/processorLduInterface.H
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -84,7 +84,7 @@ public:
// Access
//- Return communicator used for sending
//- Return communicator used for parallel communication
virtual int comm() const = 0;
//- Return processor number (rank in communicator)
......
src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixSolver.C
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -186,10 +186,15 @@ Foam::scalar Foam::lduMatrix::solver::normFactor
{
// --- Calculate A dot reference value of psi
matrix_.sumA(tmpField, interfaceBouCoeffs_, interfaces_);
tmpField *= gAverage(psi);
tmpField *= gAverage(psi, matrix_.lduMesh_.comm());
return
gSum(mag(Apsi - tmpField) + mag(source - tmpField))
gSum
(
(mag(Apsi - tmpField) + mag(source - tmpField))(),
matrix_.lduMesh_.comm()
)
+ solverPerformance::small_;
// At convergence this simpler method is equivalent to the above
......
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerateLduAddressing.C
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -280,6 +280,7 @@ void Foam::GAMGAgglomeration::agglomerateLduAddressing
coarseInterfaceAddr,
coarseInterfaces,
fineMeshAddr.patchSchedule(),
fineMesh.comm(),
true
)
);
......
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.C
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -57,7 +57,7 @@ bool Foam::GAMGAgglomeration::continueAgglomerating
{
// Check the need for further agglomeration on all processors
bool contAgg = nCoarseCells >= nCellsInCoarsestLevel_;
reduce(contAgg, andOp<bool>());
mesh().reduce(contAgg, andOp<bool>());
return contAgg;
}
......
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomeration.H
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -51,6 +51,7 @@ namespace Foam
class lduMesh;
class lduMatrix;
class mapDistribute;
/*---------------------------------------------------------------------------*\
Class GAMGAgglomeration Declaration
......@@ -77,6 +78,9 @@ protected:
// Maps from the finer to the coarser level.
PtrList<labelField> restrictAddressing_;
//- For cross-processor operation
autoPtr<mapDistribute> restrictDistributeMapPtr_;
//- Face restriction addressing array.
// Maps from the finer to the coarser level.
// Positive indices map the finer faces which form part of the boundary
......@@ -85,6 +89,9 @@ protected:
// coarser cells to minus the corresponding coarser cell index minus 1.
PtrList<labelList> faceRestrictAddressing_;
//- For cross-processor operation
autoPtr<mapDistribute> faceRestrictDistributeMapPtr_;
//- Hierarchy of mesh addressing
PtrList<lduPrimitiveMesh> meshLevels_;
......
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGAgglomerations/GAMGAgglomeration/GAMGAgglomerationTemplates.C
View file @ 33d6ea3e
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -24,6 +24,7 @@ License
\*---------------------------------------------------------------------------*/
#include "GAMGAgglomeration.H"
#include "mapDistribute.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
......@@ -37,24 +38,57 @@ void Foam::GAMGAgglomeration::restrictField
{
const labelList& fineToCoarse = restrictAddressing_[fineLevelIndex];
if (ff.size() != fineToCoarse.size())
if (restrictDistributeMapPtr_.valid())
{
FatalErrorIn
(
"void GAMGAgglomeration::restrictField"
"(Field<Type>& cf, const Field<Type>& ff, "
"const label fineLevelIndex) const"
) << "field does not correspond to level " << fineLevelIndex
<< " sizes: field = " << ff.size()
<< " level = " << fineToCoarse.size()
<< abort(FatalError);
}
const mapDistribute& map = restrictDistributeMapPtr_();
Field<Type> combinedFf(map.constructSize());
forAll(ff, i)
{
combinedFf[i] = ff[i];
}
map.distribute(combinedFf);
if (combinedFf.size() != fineToCoarse.size())
{
FatalErrorIn
(
"void GAMGAgglomeration::restrictField"
"(Field<Type>& cf, const Field<Type>& ff, "
"const label fineLevelIndex) const"
) << "field does not correspond to level " << fineLevelIndex
<< " sizes: field = " << combinedFf.size()
<< " level = " << fineToCoarse.size()
<< abort(FatalError);
}
cf = pTraits<Type>::zero;
cf = pTraits<Type>::zero;
forAll(ff, i)
forAll(combinedFf, i)
{
cf[fineToCoarse[i]] += combinedFf[i];
}
}
else
{
cf[fineToCoarse[i]] += ff[i];
if (ff.size() != fineToCoarse.size())
{
FatalErrorIn
(
"void GAMGAgglomeration::restrictField"
"(Field<Type>& cf, const Field<Type>& ff, "
"const label fineLevelIndex) const"
) << "field does not correspond to level " << fineLevelIndex
<< " sizes: field = " << ff.size()
<< " level = " << fineToCoarse.size()
<< abort