Commit 4e6c2583 authored by mattijs's avatar mattijs
Browse files

ENH: communicators: more communicators through

parent bd1230eb
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -330,7 +330,7 @@ Foam::Istream& Foam::UIPstream::rewind()
void Foam::UIPstream::print(Ostream& os) const
{
os << "Reading from processor " << fromProcNo_
<< " to processor " << myProcNo << " using communicator " << comm_
<< " using communicator " << comm_
<< " and tag " << tag_
<< Foam::endl;
}
......
......@@ -339,6 +339,39 @@ void Foam::UPstream::freeCommunicators(const bool doPstream)
}
int Foam::UPstream::baseProcNo(const label myComm, const int myProcID)
{
int procID = myProcID;
label comm = myComm;
while (parent(comm) != -1)
{
const List<int>& parentRanks = UPstream::procID(comm);
procID = parentRanks[procID];
comm = UPstream::parent(comm);
}
return procID;
}
Foam::label Foam::UPstream::myProcNo(const label myComm, const int baseProcID)
{
const List<int>& parentRanks = procID(myComm);
label parentComm = parent(myComm);
if (parentComm == -1)
{
return findIndex(parentRanks, baseProcID);
}
else
{
label parentRank = myProcNo(parentComm, baseProcID);
return findIndex(parentRanks, parentRank);
}
}
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
// By default this is not a parallel run
......
......@@ -323,18 +323,6 @@ public:
comm_(allocateCommunicator(parent, subRanks, doPstream))
{}
// communicator(const label parent)
// :
// comm_
// (
// allocateCommunicator
// (
// parent,
// identity(UPstream::nProcs(parent))
// )
// )
// {}
~communicator()
{
freeCommunicator(comm_);
......@@ -346,6 +334,14 @@ public:
}
};
//- Return physical processor number (i.e. processor number in
// worldComm) given communicator and procssor
static int baseProcNo(const label myComm, const int myProcID);
//- Return processor number in communicator (given physical processor
// number) (= reverse of baseProcNo)
static label myProcNo(const label myComm, const int baseProcID);
//- Add the valid option this type of communications library
// adds/requires on the command line
......@@ -380,6 +376,12 @@ public:
static void freeTag(const word&, const int tag);
// //- Return (index of) request to use in calls to nonblocking ops
// static label allocateRequest(const word&);
//
// //- Free (index of) request
// static void freeRequest(const word&, const label request);
//- Is this a parallel run?
static bool& parRun()
......@@ -416,7 +418,6 @@ public:
return parentCommunicator_(communicator);
}
// //- Process IDs
// static const List<int>& procIDs()
// {
......
......@@ -196,14 +196,7 @@ Foam::OSstream& Foam::messageStream::operator()(const label communicator)
{
if (title().size())
{
if (Pstream::parRun() && !collect)
{
Pout<< title().c_str();
}
else
{
Sout<< title().c_str();
}
Pout<< title().c_str();
}
if (maxErrors_)
......@@ -218,14 +211,7 @@ Foam::OSstream& Foam::messageStream::operator()(const label communicator)
}
}
if (Pstream::parRun() && !collect)
{
return Pout;
}
else
{
return Sout;
}
return Pout;
}
}
}
......
......@@ -33,8 +33,8 @@ License
Foam::LUscalarMatrix::LUscalarMatrix(const scalarSquareMatrix& matrix)
:
comm_(Pstream::worldComm),
scalarSquareMatrix(matrix),
comm_(Pstream::worldComm),
pivotIndices_(n())
{
LUDecompose(*this, pivotIndices_);
......@@ -67,11 +67,6 @@ Foam::LUscalarMatrix::LUscalarMatrix
)
);
Pout<< "LUscalarMatrix :"
<< " comm:" << comm_
<< " master:" << Pstream::master(comm_) << endl;
if (Pstream::master(comm_))
{
for
......@@ -81,8 +76,6 @@ Pout<< "LUscalarMatrix :"
slave++
)
{
Pout<< "Receiving from " << slave
<< " using comm:" << comm_ << endl;
lduMatrices.set
(
lduMatrixi++,
......
......@@ -108,7 +108,7 @@ public:
Pout<< "lduPrimitiveMesh :"
<< " nCells:" << nCells
<< " l:" << lowerAddr_.size()
<< " u::" << upperAddr_.size()
<< " u:" << upperAddr_.size()
<< " pa:" << patchAddr_.size()
<< " interfaces:" << interfaces_.size()
<< " comm:" << comm_
......@@ -124,9 +124,11 @@ public:
const processorLduInterface
>(interfaces_[i]);
Pout<< " comm:" << pi.comm()
Pout<< " patch:" << i
<< " size:" << patchAddr_[i].size()
<< " myProcNo:" << pi.myProcNo()
<< " neighbProcNo:" << pi.neighbProcNo()
<< " comm:" << pi.comm()
<< endl;
}
}
......@@ -158,7 +160,7 @@ public:
Pout<< "lduPrimitiveMesh :"
<< " nCells:" << nCells
<< " l:" << lowerAddr_.size()
<< " u::" << upperAddr_.size()
<< " u:" << upperAddr_.size()
<< " pa:" << patchAddr_.size()
<< " interfaces:" << interfaces_.size()
<< " comm:" << comm_
......@@ -174,9 +176,11 @@ public:
const processorLduInterface
>(interfaces_[i]);
Pout<< " comm:" << pi.comm()
Pout<< " patch:" << i
<< " size:" << patchAddr_[i].size()
<< " myProcNo:" << pi.myProcNo()
<< " neighbProcNo:" << pi.neighbProcNo()
<< " comm:" << pi.comm()
<< endl;
}
}
......
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -31,19 +31,52 @@ Foam::globalIndex::globalIndex
(
const label localSize,
const int tag,
const label comm,
const bool parallel
)
:
offsets_(Pstream::nProcs()+1)
offsets_(Pstream::nProcs(comm)+1)
{
labelList localSizes(Pstream::nProcs(), 0);
localSizes[Pstream::myProcNo()] = localSize;
labelList localSizes(Pstream::nProcs(comm), 0);
localSizes[Pstream::myProcNo(comm)] = localSize;
if (parallel)
{
Pstream::gatherList(localSizes, tag);
Pstream::scatterList(localSizes, tag);
Pstream::gatherList(localSizes, tag, comm);
Pstream::scatterList(localSizes, tag, comm);
}
label offset = 0;
offsets_[0] = 0;
for (label procI = 0; procI < Pstream::nProcs(comm); procI++)
{
label oldOffset = offset;
offset += localSizes[procI];
if (offset < oldOffset)
{
FatalErrorIn
(
"globalIndex::globalIndex"
"(const label, const int, const label, const bool)"
) << "Overflow : sum of sizes " << localSizes
<< " exceeds capability of label (" << labelMax
<< "). Please recompile with larger datatype for label."
<< exit(FatalError);
}
offsets_[procI+1] = offset;
}
}
Foam::globalIndex::globalIndex(const label localSize)
:
offsets_(Pstream::nProcs()+1)
{
labelList localSizes(Pstream::nProcs(), 0);
localSizes[Pstream::myProcNo()] = localSize;
Pstream::gatherList(localSizes, Pstream::msgType());
Pstream::scatterList(localSizes, Pstream::msgType());
label offset = 0;
offsets_[0] = 0;
for (label procI = 0; procI < Pstream::nProcs(); procI++)
......@@ -64,6 +97,12 @@ Foam::globalIndex::globalIndex
}
Foam::globalIndex::globalIndex(const labelList& offsets)
:
offsets_(offsets)
{}
Foam::globalIndex::globalIndex(Istream& is)
{
is >> offsets_;
......
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -71,21 +71,29 @@ public:
// Constructors
//- Construct from local max size
globalIndex(const label localSize);
//- Construct from local max size
globalIndex
(
const label localSize,
const int tag = Pstream::msgType(),
const bool parallel = true // use parallel comms
const int tag,
const label comm,
const bool parallel // use parallel comms
);
//- Construct from components
globalIndex(const labelList& offsets);
//- Construct from Istream
globalIndex(Istream& is);
// Member Functions
// Queries relating to my processor
// Queries relating to my processor (using world communicator)
//- my local size
inline label localSize() const;
......
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -47,7 +47,7 @@ inline Foam::label Foam::globalIndex::localSize() const
inline Foam::label Foam::globalIndex::size() const
{
return offsets_[Pstream::nProcs()];
return offsets_.last();
}
......@@ -104,7 +104,7 @@ inline Foam::label Foam::globalIndex::toLocal(const label i) const
inline Foam::label Foam::globalIndex::whichProcID(const label i) const
{
if (i < 0 || i >= offsets_[Pstream::nProcs()])
if (i < 0 || i >= offsets_.size()-1)
{
FatalErrorIn("globalIndex::whichProcID(const label)")
<< "Global " << i << " does not belong on any processor."
......
......@@ -2,7 +2,7 @@
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2012 OpenFOAM Foundation
\\ / A nd | Copyright (C) 2011-2013 OpenFOAM Foundation
\\/ M anipulation |
-------------------------------------------------------------------------------
License
......@@ -61,7 +61,7 @@ Foam::processorPolyPatch::processorPolyPatch
)
:
coupledPolyPatch(name, size, start, index, bm, typeName, transform),
comm_(comm),
// comm_(comm),
myProcNo_(myProcNo),
neighbProcNo_(neighbProcNo),
neighbFaceCentres_(),
......@@ -80,10 +80,10 @@ Foam::processorPolyPatch::processorPolyPatch
)
:
coupledPolyPatch(name, dict, index, bm, patchType),
comm_
(
dict.lookupOrDefault("communicator", UPstream::worldComm)
),
// comm_
// (
// dict.lookupOrDefault("communicator", UPstream::worldComm)
// ),
myProcNo_(readLabel(dict.lookup("myProcNo"))),
neighbProcNo_(readLabel(dict.lookup("neighbProcNo"))),
neighbFaceCentres_(),
......@@ -99,7 +99,7 @@ Foam::processorPolyPatch::processorPolyPatch
)
:
coupledPolyPatch(pp, bm),
comm_(pp.comm_),
// comm_(pp.comm_),
myProcNo_(pp.myProcNo_),
neighbProcNo_(pp.neighbProcNo_),
neighbFaceCentres_(),
......@@ -118,7 +118,7 @@ Foam::processorPolyPatch::processorPolyPatch
)
:
coupledPolyPatch(pp, bm, index, newSize, newStart),
comm_(pp.comm_),
// comm_(pp.comm_),
myProcNo_(pp.myProcNo_),
neighbProcNo_(pp.neighbProcNo_),
neighbFaceCentres_(),
......@@ -137,7 +137,7 @@ Foam::processorPolyPatch::processorPolyPatch
)
:
coupledPolyPatch(pp, bm, index, mapAddressing, newStart),
comm_(pp.comm_),
// comm_(pp.comm_),
myProcNo_(pp.myProcNo_),
neighbProcNo_(pp.neighbProcNo_),
neighbFaceCentres_(),
......@@ -1091,11 +1091,11 @@ bool Foam::processorPolyPatch::order
void Foam::processorPolyPatch::write(Ostream& os) const
{
coupledPolyPatch::write(os);
if (comm_ != UPstream::worldComm)
{
os.writeKeyword("communicator") << comm_
<< token::END_STATEMENT << nl;
}
// if (comm_ != UPstream::worldComm)
// {
// os.writeKeyword("communicator") << comm_
// << token::END_STATEMENT << nl;
// }
os.writeKeyword("myProcNo") << myProcNo_
<< token::END_STATEMENT << nl;
os.writeKeyword("neighbProcNo") << neighbProcNo_
......
......@@ -42,6 +42,7 @@ SourceFiles
#include "coupledPolyPatch.H"
#include "polyBoundaryMesh.H"
#include "faceListFwd.H"
#include "polyMesh.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
......@@ -58,9 +59,6 @@ class processorPolyPatch
{
// Private data
//- Communicator to use
label comm_;
int myProcNo_;
int neighbProcNo_;
......@@ -252,7 +250,8 @@ public:
//- Return communicator used for communication
label comm() const
{
return comm_;
//return comm_;
return boundaryMesh().mesh().comm();
}
//- Return processor number
......@@ -270,11 +269,11 @@ public:
// For testing
//- Return communicator used for communication
label& comm()
{
return comm_;
}
// //- Return communicator used for communication
// label& comm()
// {
// return comm_;
// }
//- Return processor number
int& myProcNo()
......
......@@ -37,6 +37,27 @@ namespace Foam
DynamicList<MPI_Request> PstreamGlobals::outstandingRequests_;
//! \endcond
//// Max outstanding non-blocking operations.
////! \cond fileScope
//int PstreamGlobals::nRequests_ = 0;
////! \endcond
// Free'd non-blocking operations.
//! \cond fileScope
//DynamicList<label> PstreamGlobals::freedRequests_;
//! \endcond
// Max outstanding message tag operations.
//! \cond fileScope
int PstreamGlobals::nTags_ = 0;
//! \endcond
// Free'd message tags
//! \cond fileScope
DynamicList<int> PstreamGlobals::freedTags_;
//! \endcond
// Allocated communicators.
//! \cond fileScope
DynamicList<MPI_Comm> PstreamGlobals::MPICommunicators_;
......
......@@ -54,6 +54,13 @@ namespace PstreamGlobals
extern DynamicList<MPI_Request> outstandingRequests_;
//extern int nRequests_;
//extern DynamicList<label> freedRequests_;
extern int nTags_;
extern DynamicList<int> freedTags_;
// Current communicators. First element will be MPI_COMM_WORLD
extern DynamicList<MPI_Comm> MPICommunicators_;
......
......@@ -70,7 +70,7 @@ Foam::UIPstream::UIPstream
label wantedSize = externalBuf_.capacity();
if (debug&1)
if (debug)
{
Pout<< "UIPstream::UIPstream : read from:" << fromProcNo
<< " tag:" << tag << " wanted size:" << wantedSize
......@@ -94,7 +94,7 @@ Foam::UIPstream::UIPstream
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
if (debug&1)
if (debug)
{
Pout<< "UIPstream::UIPstream : probed size:" << wantedSize
<< Foam::endl;
......@@ -151,7 +151,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
// Message is already received into externalBuf
messageSize_ = buffers.recvBuf_[fromProcNo].size();
if (debug&1)
if (debug)
{
Pout<< "UIPstream::UIPstream PstreamBuffers :"
<< " fromProcNo:" << fromProcNo
......@@ -165,7 +165,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
label wantedSize = externalBuf_.capacity();
if (debug&1)
if (debug)
{
Pout<< "UIPstream::UIPstream PstreamBuffers :"
<< " read from:" << fromProcNo
......@@ -189,7 +189,7 @@ Foam::UIPstream::UIPstream(const int fromProcNo, PstreamBuffers& buffers)
externalBuf_.setCapacity(messageSize_);
wantedSize = messageSize_;
if (debug&1)
if (debug)
{
Pout<< "UIPstream::UIPstream PstreamBuffers : probed size:"
<< wantedSize << Foam::endl;
......@@ -229,7 +229,7 @@ Foam::label Foam::UIPstream::read
const label communicator
)
{
if (debug&1)
if (debug)
{
Pout<< "UIPstream::read : starting read from:" << fromProcNo
<< " tag:" << tag << " comm:" << communicator
......@@ -282,7 +282,7 @@ Foam::label Foam::UIPstream::read
int messageSize;
MPI_Get_count(&status, MPI_BYTE, &messageSize);
if (debug&1)
if (debug)
{
Pout<< "UIPstream::read : finished read from:" << fromProcNo
<< " tag:" << tag << " read size:" << label(bufSize)
......@@ -332,7 +332,7 @@ Foam::label Foam::UIPstream::read
return 0;
}
if (debug&1)
if (debug)
{
Pout<< "UIPstream::read : started read from:" << fromProcNo
<< " tag:" << tag << " read size:" << label(bufSize)
......
......@@ -43,7 +43,7 @@ bool Foam::UOPstream::write
const label communicator
)
{
if (debug&1)
if (debug)
{
Pout<< "UOPstream::write : starting write to:" << toProcNo
<< " tag:" << tag
......@@ -80,7 +80,7 @@ bool Foam::UOPstream::write
PstreamGlobals::MPICommunicators_[communicator] //MPI_COMM_WORLD
);