Skip to content

GitLab

Commit 89f2cda3 authored by mattijs's avatar mattijs Committed by Andrew Heather
Browse files

ENH: mpi: use per-application communicator.

parent 627d79db
Expand all Hide whitespace changes
Inline Side-by-side
applications/utilities/parallelProcessing/redistributePar/redistributePar.C
View file @ 89f2cda3
......@@ -511,9 +511,9 @@ void writeProcAddressing
patchDistMap.constructMap(),
patchDistMap.constructHasFlip(),
patchMap,
label(-1),
eqOp<label>(),
flipOp(),
label(-1),
UPstream::msgType()
);
}
......
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
View file @ 89f2cda3
......@@ -90,8 +90,8 @@ void Foam::UPstream::setParRun(const label nProcs, const bool haveThreads)
<< Foam::exit(FatalError);
}
Pout.prefix() = '[' + name(myProcNo(Pstream::worldComm)) + "] ";
Perr.prefix() = '[' + name(myProcNo(Pstream::worldComm)) + "] ";
Pout.prefix() = '[' + name(myProcNo(comm)) + "] ";
Perr.prefix() = '[' + name(myProcNo(comm)) + "] ";
}
}
......@@ -373,6 +373,8 @@ Foam::DynamicList<Foam::List<int>> Foam::UPstream::procIDs_(10);
Foam::DynamicList<Foam::label> Foam::UPstream::parentCommunicator_(10);
Foam::wordList Foam::UPstream::allWorlds_(1, "");
Foam::labelList Foam::UPstream::worldIDs_(1, 0);
Foam::DynamicList<Foam::List<Foam::UPstream::commsStruct>>
Foam::UPstream::linearCommunication_(10);
......
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View file @ 89f2cda3
......@@ -194,6 +194,13 @@ private:
//- Standard transfer message type
static int msgType_;
//- Names of all worlds
static wordList allWorlds_;
//- Per processor the name of the world
static labelList worldIDs_;
// Communicator specific data
//- Free communicators
......@@ -437,7 +444,7 @@ public:
}
//- Number of processes in parallel run, and 1 for serial run
static label nProcs(const label communicator = 0)
static label nProcs(const label communicator = worldComm)
{
return procIDs_[communicator].size();
}
......@@ -449,13 +456,13 @@ public:
}
//- Am I the master process
static bool master(const label communicator = 0)
static bool master(const label communicator = worldComm)
{
return myProcNo_[communicator] == masterNo();
}
//- Number of this process (starting from masterNo() = 0)
static int myProcNo(const label communicator = 0)
static int myProcNo(const label communicator = worldComm)
{
return myProcNo_[communicator];
}
......@@ -471,15 +478,43 @@ public:
return procIDs_[communicator];
}
// Worlds
//- All worlds
static const wordList& allWorlds()
{
return allWorlds_;
}
//- worldID (index in allWorlds) of all processes
static const labelList& worldIDs()
{
return worldIDs_;
}
//- My worldID
static label myWorldID()
{
return worldIDs_[myProcNo(0)];
}
//- My world
static const word& myWorld()
{
return allWorlds()[myWorldID()];
}
//- Range of process indices for all processes
static rangeType allProcs(const label communicator = 0)
static rangeType allProcs(const label communicator = worldComm)
{
// Proc 0 -> nProcs (int value)
return rangeType(static_cast<int>(nProcs(communicator)));
}
//- Range of process indices for sub-processes
static rangeType subProcs(const label communicator = 0)
static rangeType subProcs(const label communicator = worldComm)
{
// Proc 1 -> nProcs (int value)
return rangeType(1, static_cast<int>(nProcs(communicator)-1));
......@@ -488,7 +523,7 @@ public:
//- Communication schedule for linear all-to-master (proc 0)
static const List<commsStruct>& linearCommunication
(
const label communicator = 0
const label communicator = worldComm
)
{
return linearCommunication_[communicator];
......@@ -497,7 +532,7 @@ public:
//- Communication schedule for tree all-to-master (proc 0)
static const List<commsStruct>& treeCommunication
(
const label communicator = 0
const label communicator = worldComm
)
{
return treeCommunication_[communicator];
......@@ -542,7 +577,7 @@ public:
(
const labelUList& sendData,
labelUList& recvData,
const label communicator = 0
const label communicator = worldComm
);
//- Exchange data with all processors (in the communicator)
......@@ -559,7 +594,7 @@ public:
const UList<int>& recvSizes,
const UList<int>& recvOffsets,
const label communicator = 0
const label communicator = worldComm
);
//- Receive data from all processors on the master
......@@ -571,7 +606,7 @@ public:
char* recvData,
const UList<int>& recvSizes,
const UList<int>& recvOffsets,
const label communicator = 0
const label communicator = worldComm
);
//- Send data to all processors from the root of the communicator
......@@ -583,7 +618,7 @@ public:
char* recvData,
int recvSize,
const label communicator = 0
const label communicator = worldComm
);
......@@ -598,7 +633,7 @@ public:
//- Process index of last sub-process
// \deprecated(2020-09) use subProcs() method instead
static int lastSlave(const label communicator = 0)
static int lastSlave(const label communicator = worldComm)
{
return nProcs(communicator) - 1;
}
......
src/OpenFOAM/global/argList/argList.C
View file @ 89f2cda3
......@@ -166,6 +166,20 @@ Foam::argList::initValidTables::initValidTables()
true // advanced option
);
argList::addOption
(
"world",
"Name",
"Name of local world",
true
);
validParOptions.set
(
"world",
"Name of local world"
);
// Some standard option aliases (with or without version warnings)
// argList::addOptionCompat
// (
......@@ -493,6 +507,7 @@ void Foam::argList::noParallel()
removeOption("roots");
removeOption("decomposeParDict");
removeOption("hostRoots");
removeOption("world");
validParOptions.clear();
}
......@@ -711,6 +726,7 @@ void Foam::argList::setCasePaths()
}
else
{
caseDir.expand();
caseDir.toAbsolute();
}
}
......@@ -1241,7 +1257,7 @@ void Foam::argList::parse
dictNProcs = roots.size()+1;
}
}
else if (checkProcessorDirectories_)
else if (checkProcessorDirectories_ && Pstream::nProcs() > 1)
{
// Use values from decomposeParDict, the location was already
// established above.
......@@ -1268,6 +1284,13 @@ void Foam::argList::parse
Pstream::parRun(oldParRun); // Restore parallel state
decompDict.readEntry("numberOfSubdomains", dictNProcs);
if (Pstream::nProcs() == 1)
{
WarningInFunction
<< "Running parallel on single processor. This only"
<< " makes sense for multi-world simulation" << endl;
dictNProcs = 1;
}
if (decompDict.getOrDefault("distributed", false))
{
......@@ -1300,7 +1323,12 @@ void Foam::argList::parse
// - normal running : nProcs = dictNProcs = nProcDirs
// - decomposition to more processors : nProcs = dictNProcs
// - decomposition to fewer processors : nProcs = nProcDirs
if (checkProcessorDirectories_ && dictNProcs > Pstream::nProcs())
if
(
checkProcessorDirectories_
&& Pstream::nProcs() > 1
&& dictNProcs > Pstream::nProcs()
)
{
FatalError
<< source
......@@ -1353,6 +1381,7 @@ void Foam::argList::parse
if
(
checkProcessorDirectories_
&& Pstream::nProcs() > 1
&& dictNProcs < Pstream::nProcs()
)
{
......@@ -1405,7 +1434,14 @@ void Foam::argList::parse
}
nProcs = Pstream::nProcs();
case_ = globalCase_/("processor" + Foam::name(Pstream::myProcNo()));
if (Pstream::nProcs() > 1)
{
case_ = globalCase_/("processor" + Foam::name(Pstream::myProcNo()));
}
else
{
case_ = globalCase_;
}
}
else
{
......@@ -1468,6 +1504,13 @@ void Foam::argList::parse
<< Pstream::commsTypeNames[Pstream::defaultCommsType] << nl
<< " polling iterations : " << Pstream::nPollProcInterfaces
<< endl;
if (UPstream::allWorlds().size() > 1)
{
Info<< " worlds : "
<< flatOutput(UPstream::allWorlds()) << nl
<< " world : " << UPstream::myWorld()
<< endl;
}
}
}
......
src/OpenFOAM/meshes/polyMesh/globalMeshData/globalIndexI.H
View file @ 89f2cda3
......@@ -93,7 +93,7 @@ inline Foam::label Foam::globalIndex::size() const
inline void Foam::globalIndex::reset(const label localSize)
{
reset(localSize, Pstream::msgType(), 0, true);
reset(localSize, Pstream::msgType(), UPstream::worldComm, true);
}
......
src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C
View file @ 89f2cda3
......@@ -2745,13 +2745,16 @@ void Foam::globalMeshData::updateMesh()
// *** Temporary hack to avoid problems with overlapping communication
// *** between these reductions and the calculation of deltaCoeffs
//label comm = UPstream::worldComm + 1;
label comm = UPstream::allocateCommunicator
//const label comm = UPstream::worldComm + 1;
const label comm = UPstream::allocateCommunicator
(
UPstream::worldComm,
identity(UPstream::nProcs(UPstream::worldComm)),
true
);
const label oldWarnComm = UPstream::warnComm;
UPstream::warnComm = comm;
// Total number of faces.
nTotalFaces_ = returnReduce
......@@ -2789,6 +2792,7 @@ void Foam::globalMeshData::updateMesh()
);
UPstream::freeCommunicator(comm);
UPstream::warnComm = oldWarnComm;
if (debug)
{
......
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
View file @ 89f2cda3
......@@ -148,9 +148,9 @@ void Foam::mapDistribute::printLayout(Ostream& os) const
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::mapDistribute::mapDistribute()
Foam::mapDistribute::mapDistribute(const label comm)
:
mapDistributeBase()
mapDistributeBase(comm)
{}
......@@ -176,7 +176,8 @@ Foam::mapDistribute::mapDistribute
labelListList&& subMap,
labelListList&& constructMap,
const bool subHasFlip,
const bool constructHasFlip
const bool constructHasFlip,
const label comm
)
:
mapDistributeBase
......@@ -185,7 +186,8 @@ Foam::mapDistribute::mapDistribute
std::move(subMap),
std::move(constructMap),
subHasFlip,
constructHasFlip
constructHasFlip,
comm
)
{}
......@@ -198,7 +200,8 @@ Foam::mapDistribute::mapDistribute
labelListList&& transformElements,
labelList&& transformStart,
const bool subHasFlip,
const bool constructHasFlip
const bool constructHasFlip,
const label comm
)
:
mapDistributeBase
......@@ -207,7 +210,8 @@ Foam::mapDistribute::mapDistribute
std::move(subMap),
std::move(constructMap),
subHasFlip,
constructHasFlip
constructHasFlip,
comm
),
transformElements_(std::move(transformElements)),
transformStart_(std::move(transformStart))
......@@ -217,10 +221,11 @@ Foam::mapDistribute::mapDistribute
Foam::mapDistribute::mapDistribute
(
const labelUList& sendProcs,
const labelUList& recvProcs
const labelUList& recvProcs,
const label comm
)
:
mapDistributeBase(sendProcs, recvProcs)
mapDistributeBase(sendProcs, recvProcs, comm)
{}
......@@ -229,7 +234,8 @@ Foam::mapDistribute::mapDistribute
const globalIndex& globalNumbering,
labelList& elements,
List<Map<label>>& compactMap,
const int tag
const int tag,
const label comm
)
:
mapDistributeBase
......@@ -237,7 +243,8 @@ Foam::mapDistribute::mapDistribute
globalNumbering,
elements,
compactMap,
tag
tag,
comm
)
{}
......@@ -247,7 +254,8 @@ Foam::mapDistribute::mapDistribute
const globalIndex& globalNumbering,
labelListList& cellCells,
List<Map<label>>& compactMap,
const int tag
const int tag,
const label comm
)
:
mapDistributeBase
......@@ -255,7 +263,8 @@ Foam::mapDistribute::mapDistribute
globalNumbering,
cellCells,
compactMap,
tag
tag,
comm
)
{}
......@@ -268,11 +277,14 @@ Foam::mapDistribute::mapDistribute
const labelPairList& transformedElements,
labelList& transformedIndices,
List<Map<label>>& compactMap,
const int tag
const int tag,
const label comm
)
:
mapDistributeBase()
mapDistributeBase(comm)
{
const label myRank = Pstream::myProcNo(comm);
// Construct per processor compact addressing of the global elements
// needed. The ones from the local processor are not included since
// these are always all needed.
......@@ -288,7 +300,7 @@ Foam::mapDistribute::mapDistribute
{
labelPair elem = transformedElements[i];
label proci = globalTransforms.processor(elem);
if (proci != Pstream::myProcNo())
if (proci != myRank)
{
label index = globalTransforms.index(elem);
label nCompact = compactMap[proci].size();
......@@ -345,7 +357,7 @@ Foam::mapDistribute::mapDistribute
// Get compact index for untransformed element
label rawElemI =
(
proci == Pstream::myProcNo()
proci == myRank
? index
: compactMap[proci][index]
);
......@@ -373,11 +385,14 @@ Foam::mapDistribute::mapDistribute
const List<labelPairList>& transformedElements,
labelListList& transformedIndices,
List<Map<label>>& compactMap,
const int tag
const int tag,
const label comm
)
:
mapDistributeBase()
mapDistributeBase(comm)
{
const label myRank = Pstream::myProcNo(comm_);
// Construct per processor compact addressing of the global elements
// needed. The ones from the local processor are not included since
// these are always all needed.
......@@ -396,7 +411,7 @@ Foam::mapDistribute::mapDistribute
forAll(elems, i)
{
label proci = globalTransforms.processor(elems[i]);
if (proci != Pstream::myProcNo())
if (proci != myRank)
{
label index = globalTransforms.index(elems[i]);
label nCompact = compactMap[proci].size();
......@@ -462,7 +477,7 @@ Foam::mapDistribute::mapDistribute
// Get compact index for untransformed element
label rawElemI =
(
proci == Pstream::myProcNo()
proci == myRank
? index
: compactMap[proci][index]
);
......@@ -487,10 +502,11 @@ Foam::mapDistribute::mapDistribute
(
labelListList&& subMap,
const bool subHasFlip,
const bool constructHasFlip
const bool constructHasFlip,
const label comm
)
:
mapDistributeBase(std::move(subMap), subHasFlip, constructHasFlip)
mapDistributeBase(std::move(subMap), subHasFlip, constructHasFlip, comm)
{}
......
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
View file @ 89f2cda3
......@@ -328,7 +328,7 @@ public:
// Constructors
//- Construct null
mapDistribute();
mapDistribute(const label comm = UPstream::worldComm);
//- Copy construct
explicit mapDistribute(const mapDistribute& map);
......@@ -343,7 +343,8 @@ public:
labelListList&& subMap,
labelListList&& constructMap,
const bool subHasFlip = false,
const bool constructHasFlip = false
const bool constructHasFlip = false,
const label comm = UPstream::worldComm
);
//- Move construct from components
......@@ -355,7 +356,8 @@ public:
labelListList&& transformElements,
labelList&& transformStart,
const bool subHasFlip = false,
const bool constructHasFlip = false
const bool constructHasFlip = false,
const label comm = UPstream::worldComm
);
//- Construct from reverse addressing: per data item the send
......@@ -365,7 +367,8 @@ public:
mapDistribute
(
const labelUList& sendProcs,
const labelUList& recvProcs
const labelUList& recvProcs,
const label comm = UPstream::worldComm
);
//- Construct from list of (possibly) remote elements in globalIndex
......@@ -378,7 +381,8 @@ public:
const globalIndex&,
labelList& elements,
List<Map<label>>& compactMap,
const int tag = Pstream::msgType()
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
//- Special variant that works with the info sorted into bins
......@@ -390,7 +394,8 @@ public:
const globalIndex&,
labelListList& cellCells,
List<Map<label>>& compactMap,
const int tag = Pstream::msgType()
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
//- Construct from list of (possibly remote) untransformed elements
......@@ -407,7 +412,8 @@ public:
const labelPairList& transformedElements,
labelList& transformedIndices,
List<Map<label>>& compactMap,
const int tag = Pstream::msgType()
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
//- As above but with ListLists.
......@@ -419,7 +425,8 @@ public:
const List<labelPairList>& transformedElements,
labelListList& transformedIndices,
List<Map<label>>& compactMap,
const int tag = Pstream::msgType()
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
//- Construct from my elements to send. Assumes layout is my elements
......@@ -429,7 +436,8 @@ public:
(
labelListList&& subMap,
const bool subHasFlip = false,
const bool constructHasFlip = false
const bool constructHasFlip = false,
const label comm = UPstream::worldComm
);
//- Construct from Istream
......
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.H
View file @ 89f2cda3
......@@ -122,6 +122,8 @@ protected:
//- Whether constructMap includes flip or not
bool constructHasFlip_;
//- Communicator to use for parallel operations
label comm_;
//- Schedule
mutable autoPtr<List<labelPair>> schedulePtr_;
......@@ -199,7 +201,7 @@ public:
// Constructors
//- Construct null
mapDistributeBase();
mapDistributeBase(const label comm = UPstream::worldComm);
//- Copy construct
mapDistributeBase(const mapDistributeBase& map);
......@@ -214,7 +216,8 @@ public:
labelListList&& subMap,
labelListList&& constructMap,