diff --git a/applications/test/parallel-comm2/Test-parallel-comm2.C b/applications/test/parallel-comm2/Test-parallel-comm2.C
index 7f5a4afa5a6c424dca6f6482b0993a06930883d6..17f8283d1325de589a539e03c4726dce99e640d9 100644
--- a/applications/test/parallel-comm2/Test-parallel-comm2.C
+++ b/applications/test/parallel-comm2/Test-parallel-comm2.C
@@ -64,6 +64,7 @@ int main(int argc, char *argv[])
argList::noBanner();
argList::noCheckProcessorDirectories();
argList::addBoolOption("verbose", "Set debug level");
+ argList::addBoolOption("print-tree", "Report tree(s) as graph");
argList::addBoolOption("comm-split", "Test simple comm split");
argList::addBoolOption("host-comm", "Test DIY host-comm split");
@@ -81,6 +82,8 @@ int main(int argc, char *argv[])
#include "setRootCase.H"
+ const bool optPrintTree = args.found("print-tree");
+
Info<< nl
<< "parallel:" << UPstream::parRun()
<< "nProcs = " << UPstream::nProcs()
@@ -102,6 +105,13 @@ int main(int argc, char *argv[])
labelList subRanks;
UPstream::communicator newComm;
+
+ if (UPstream::parRun() && optPrintTree)
+ {
+ Info<< "comms: " << UPstream::whichCommunication() << endl;
+ UPstream::printCommTree(UPstream::commWorld());
+ }
+
if (!args.found("comm-split") && !args.found("host-comm"))
{
#if 1
@@ -377,12 +387,7 @@ int main(int argc, char *argv[])
// From world to hostMaster
const label hostMasterComm =
- UPstream::allocateCommunicator
- (
- UPstream::commGlobal(),
- subRanks,
- true
- );
+ UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks);
const label myHostId =
@@ -400,12 +405,7 @@ int main(int argc, char *argv[])
// The intra-host ranks
const label hostComm =
- UPstream::allocateCommunicator
- (
- UPstream::commGlobal(),
- subRanks,
- true
- );
+ UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks);
Pout<< nl << "[manual split]" << nl
<< nl << "Host comm with "
@@ -419,8 +419,8 @@ int main(int argc, char *argv[])
<< " sub-rank:" << UPstream::is_subrank(hostMasterComm)
<< nl;
- UPstream::freeCommunicator(hostMasterComm, true);
- UPstream::freeCommunicator(hostComm, true);
+ UPstream::freeCommunicator(hostMasterComm);
+ UPstream::freeCommunicator(hostComm);
}
Info<< "\nEnd\n" << endl;
diff --git a/applications/test/treeComms/Test-treeComms.C b/applications/test/treeComms/Test-treeComms.C
index fe226b828a2a0716300182b7ba7a0d13fb881fdb..94588bd797664bff8e5d7dc6038f4982a20353d8 100644
--- a/applications/test/treeComms/Test-treeComms.C
+++ b/applications/test/treeComms/Test-treeComms.C
@@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
- Copyright (C) 2022 OpenCFD Ltd.
+ Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -59,7 +59,7 @@ void printRecvCount_gatherList
labelList nMesg;
labelList nRecv;
- if (UPstream::parRun() && np > 1 && Pstream::master(comm))
+ if (UPstream::parRun() && np > 1 && UPstream::master(comm))
{
nMesg.resize(np, Zero);
nRecv.resize(np, Zero);
@@ -99,7 +99,7 @@ void printSendCount_scatterList
labelList nMesg;
labelList nSend;
- if (UPstream::parRun() && np > 1 && Pstream::master(comm))
+ if (UPstream::parRun() && np > 1 && UPstream::master(comm))
{
nMesg.resize(np, Zero);
nSend.resize(np, Zero);
@@ -139,7 +139,7 @@ void printWidths
labelList maxBelow;
labelList maxNotBelow;
- if (UPstream::parRun() && np > 1 && Pstream::master(comm))
+ if (UPstream::parRun() && np > 1 && UPstream::master(comm))
{
maxBelow.resize(np, Zero);
maxNotBelow.resize(np, Zero);
@@ -194,7 +194,7 @@ int main(int argc, char *argv[])
// Info<< "allComms: " << comms << nl;
- if (Pstream::master())
+ if (UPstream::master())
{
OFstream os("treeComm.dot");
@@ -204,9 +204,9 @@ int main(int argc, char *argv[])
printConnection(os, 0, myComm.below());
// Pout<< flatOutput(myComm.allBelow()) << nl;
- for (const int proci : Pstream::subProcs())
+ for (const int proci : UPstream::subProcs())
{
- IPstream fromProc(Pstream::commsTypes::scheduled, proci);
+ IPstream fromProc(UPstream::commsTypes::scheduled, proci);
labelList below(fromProc);
printConnection(os, proci, below);
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
index e14bcfc066fe7c1cad40126931eebd5a7b371920..14a5cbc7d1f4b2295ffd16c8144668b43572637e 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
@@ -230,6 +230,10 @@ Foam::label Foam::UPstream::allocateCommunicator
procIds.resize(numSubRanks);
+ // Sizing and filling are demand-driven
+ linearCommunication_[index].clear();
+ treeCommunication_[index].clear();
+
if (doPstream && parRun())
{
allocatePstreamCommunicator(parentIndex, index);
@@ -249,13 +253,6 @@ Foam::label Foam::UPstream::allocateCommunicator
/// }
}
- // In case communicator allocation adjusted procIDs_
- numSubRanks = procIDs_[index].size();
-
- // Size but do not fill structure - this is done on-the-fly
- linearCommunication_[index] = List<commsStruct>(numSubRanks);
- treeCommunication_[index] = List<commsStruct>(numSubRanks);
-
return index;
}
@@ -349,112 +346,40 @@ Foam::label Foam::UPstream::procNo
}
-template<>
-Foam::UPstream::commsStruct&
-Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID)
+const Foam::List<Foam::UPstream::commsStruct>&
+Foam::UPstream::linearCommunication(const label communicator)
{
- UPstream::commsStruct& t = v_[procID];
-
- if (t.allBelow().size() + t.allNotBelow().size() + 1 != size())
+ if (linearCommunication_[communicator].empty())
{
- // Not yet allocated
+ linearCommunication_[communicator] =
+ List<commsStruct>(UPstream::nProcs(communicator));
+ }
- label above(-1);
- labelList below;
- labelList allBelow;
+ return linearCommunication_[communicator];
+}
- if (size() < UPstream::nProcsSimpleSum)
- {
- // Linear schedule
-
- if (procID == 0)
- {
- below.setSize(size()-1);
- for (label procI = 1; procI < size(); procI++)
- {
- below[procI-1] = procI;
- }
- }
- else
- {
- above = 0;
- }
- }
- else
- {
- // Use tree like schedule. For 8 procs:
- // (level 0)
- // 0 receives from 1
- // 2 receives from 3
- // 4 receives from 5
- // 6 receives from 7
- // (level 1)
- // 0 receives from 2
- // 4 receives from 6
- // (level 2)
- // 0 receives from 4
- //
- // The sends/receives for all levels are collected per processor
- // (one send per processor; multiple receives possible) creating
- // a table:
- //
- // So per processor:
- // proc receives from sends to
- // ---- ------------- --------
- // 0 1,2,4 -
- // 1 - 0
- // 2 3 0
- // 3 - 2
- // 4 5 0
- // 5 - 4
- // 6 7 4
- // 7 - 6
-
- label mod = 0;
-
- for (label step = 1; step < size(); step = mod)
- {
- mod = step * 2;
-
- if (procID % mod)
- {
- above = procID - (procID % mod);
- break;
- }
- else
- {
- for
- (
- label j = procID + step;
- j < size() && j < procID + mod;
- j += step
- )
- {
- below.append(j);
- }
- for
- (
- label j = procID + step;
- j < size() && j < procID + mod;
- j++
- )
- {
- allBelow.append(j);
- }
- }
- }
- }
- t = UPstream::commsStruct(size(), procID, above, below, allBelow);
+
+const Foam::List<Foam::UPstream::commsStruct>&
+Foam::UPstream::treeCommunication(const label communicator)
+{
+ if (treeCommunication_[communicator].empty())
+ {
+ treeCommunication_[communicator] =
+ List<commsStruct>(UPstream::nProcs(communicator));
}
- return t;
+
+ return treeCommunication_[communicator];
}
-template<>
-const Foam::UPstream::commsStruct&
-Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID) const
+void Foam::UPstream::printCommTree(const label communicator)
{
- return const_cast<UList<UPstream::commsStruct>&>(*this).operator[](procID);
+ const auto& comms = UPstream::whichCommunication(communicator);
+
+ if (UPstream::master(communicator))
+ {
+ commsStruct::printGraph(Info(), comms);
+ }
}
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
index 5030e9cef0655c3e8cfdecda9896590850bd0299..053eb8b9aa2ac5233ce22b9034c7b4c750213c5c 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
@@ -98,17 +98,18 @@ public:
{
// Private Data
- //- The procID of the processor directly above
+ //- The procID of the processor \em directly above
label above_;
- //- The procIDs of all processors directly below
+ //- The procIDs of processors \em directly below
labelList below_;
- //- procIDs of all processors below (so not just directly below)
+ //- The procIDs of all processors below myProcNo,
+ //- not just directly below
labelList allBelow_;
- //- procIDs of all processors not below.
- // Inverse set of allBelow_ without myProcNo.
+ //- The procIDs of all processors not below myProcNo
+ // (inverse of allBelow_ without myProcNo)
labelList allNotBelow_;
@@ -119,19 +120,19 @@ public:
//- Default construct with above == -1
commsStruct() noexcept : above_(-1) {}
- //- Construct from components
+ //- Move construct from components
commsStruct
(
const label above,
- const labelUList& below,
- const labelUList& allBelow,
- const labelUList& allNotBelow
+ labelList&& below,
+ labelList&& allBelow,
+ labelList&& allNotBelow
);
- //- Construct from components; construct allNotBelow_
+ //- Copy construct from below, allBelow components
commsStruct
(
- const label nProcs,
+ const label numProcs,
const label myProcID,
const label above,
const labelUList& below,
@@ -141,29 +142,31 @@ public:
// Member Functions
- //- Reset to default constructed state
- void clear();
+ //- Print un-directed graph in graphviz dot format
+ static void printGraph
+ (
+ Ostream& os,
+ const UList<UPstream::commsStruct>& comms,
+ const label proci = 0 // starting node
+ );
- //- The procID of the processor directly above
- label above() const noexcept
- {
- return above_;
- }
- //- The procIDs of all processors directly below
- const labelList& below() const noexcept
- {
- return below_;
- }
+ // Access
+
+ //- The number of processors addressed by the structure
+ label nProcs() const;
+
+ //- The procID of the processor \em directly above
+ label above() const noexcept { return above_; }
+
+ //- The procIDs of the processors \em directly below
+ const labelList& below() const noexcept { return below_; }
//- The procIDs of all processors below
//- (so not just directly below)
- const labelList& allBelow() const noexcept
- {
- return allBelow_;
- }
+ const labelList& allBelow() const noexcept { return allBelow_; }
- //- The procIDs of all processors that are above.
+ //- The procIDs of all processors not below myProcNo.
//- The inverse set of allBelow without myProcNo.
const labelList& allNotBelow() const noexcept
{
@@ -171,13 +174,19 @@ public:
}
- // Member Operators
+ // Edit
- bool operator==(const commsStruct&) const;
- bool operator!=(const commsStruct&) const;
+ //- Reset to default constructed state
+ void reset();
+
+ //- Reset with automatic linear/tree selection
+ void reset(const label procID, const label numProcs);
- // Ostream Operator
+ // Member / Friend Operators
+
+ bool operator==(const commsStruct&) const;
+ bool operator!=(const commsStruct&) const;
friend Ostream& operator<<(Ostream&, const commsStruct&);
};
@@ -335,6 +344,9 @@ public:
return (communicator > worldComm && communicator > selfComm);
}
+ //- Debugging: print the communication tree
+ static void printCommTree(const label communicator);
+
// Constructors
@@ -391,24 +403,21 @@ public:
//- Move construct, takes ownership
communicator(communicator&& c) : comm_(c.comm_) { c.comm_ = -1; }
- //- Allocate a communicator based on given parent
+ //- Allocate communicator for sub-ranks on given parent
communicator
(
//! The parent communicator
- const label parent,
+ const label parentComm,
- //! The sub-ranks of parent to use (ignore negative values)
- const labelUList& subRanks,
-
- //! Call allocatePstreamCommunicator
- const bool doPstream = true
+ //! The sub-ranks of parent to use (negative values ignored)
+ const labelUList& subRanks
)
:
- comm_(allocateCommunicator(parent, subRanks, doPstream))
+ comm_(UPstream::allocateCommunicator(parentComm, subRanks))
{}
//- Free allocated communicator and group
- ~communicator() { freeCommunicator(comm_); }
+ ~communicator() { UPstream::freeCommunicator(comm_); }
//- True if communicator is non-negative (ie, was allocated)
bool good() const noexcept { return (comm_ >= 0); }
@@ -417,19 +426,19 @@ public:
label comm() const noexcept { return comm_; }
//- Free allocated communicator and group
- void reset() { freeCommunicator(comm_); comm_ = -1; }
+ void reset() { UPstream::freeCommunicator(comm_); comm_ = -1; }
//- Allocate with subRanks of parent communicator
void reset(label parent, const labelUList& subRanks)
{
- freeCommunicator(comm_);
- comm_ = allocateCommunicator(parent, subRanks);
+ UPstream::freeCommunicator(comm_);
+ comm_ = UPstream::allocateCommunicator(parent, subRanks);
}
//- Take ownership, free allocated communicator and group.
void reset(communicator&& c)
{
- if (comm_ != c.comm_) freeCommunicator(comm_);
+ if (comm_ != c.comm_) UPstream::freeCommunicator(comm_);
comm_ = c.comm_;
c.comm_ = -1;
}
@@ -730,22 +739,17 @@ public:
}
//- Communication schedule for linear all-to-master (proc 0)
- static const List<commsStruct>& linearCommunication
+ static const List<commsStruct>&
+ linearCommunication
(
const label communicator = worldComm
- )
- {
- return linearCommunication_[communicator];
- }
+ );
//- Communication schedule for tree all-to-master (proc 0)
static const List<commsStruct>& treeCommunication
(
const label communicator = worldComm
- )
- {
- return treeCommunication_[communicator];
- }
+ );
//- Communication schedule for linear/tree all-to-master (proc 0).
//- Chooses based on the value of UPstream::nProcsSimpleSum
@@ -757,8 +761,8 @@ public:
return
(
nProcs(communicator) < nProcsSimpleSum
- ? linearCommunication_[communicator]
- : treeCommunication_[communicator]
+ ? linearCommunication(communicator)
+ : treeCommunication(communicator)
);
}
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstreamCommsStruct.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstreamCommsStruct.C
index 760c77d2bfbad7a73d5b711fb931d4394208d8cc..7aa432170bb2e2c281b0dfabfd9dfc1dfc95d381 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstreamCommsStruct.C
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstreamCommsStruct.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
- Copyright (C) 2021-2022 OpenCFD Ltd.
+ Copyright (C) 2021-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -33,21 +33,21 @@ License
Foam::UPstream::commsStruct::commsStruct
(
const label above,
- const labelUList& below,
- const labelUList& allBelow,
- const labelUList& allNotBelow
+ labelList&& below,
+ labelList&& allBelow,
+ labelList&& allNotBelow
)
:
above_(above),
- below_(below),
- allBelow_(allBelow),
- allNotBelow_(allNotBelow)
+ below_(std::move(below)),
+ allBelow_(std::move(allBelow)),
+ allNotBelow_(std::move(allNotBelow))
{}
Foam::UPstream::commsStruct::commsStruct
(
- const label nProcs,
+ const label numProcs,
const label myProcID,
const label above,
const labelUList& below,
@@ -57,33 +57,114 @@ Foam::UPstream::commsStruct::commsStruct
above_(above),
below_(below),
allBelow_(allBelow),
- allNotBelow_(nProcs - allBelow.size() - 1)
+ allNotBelow_(numProcs - allBelow.size() - 1)
+{
+ List<bool> isNotBelow(numProcs, true);
+
+ // Exclude self
+ isNotBelow[myProcID] = false;
+
+ // Exclude allBelow
+ for (const label proci : allBelow)
+ {
+ isNotBelow[proci] = false;
+ }
+
+ // Compacting to obtain allNotBelow_
+ label nNotBelow = 0;
+ forAll(isNotBelow, proci)
+ {
+ if (isNotBelow[proci])
+ {
+ allNotBelow_[nNotBelow++] = proci;
+ }
+ }
+
+ if (nNotBelow != allNotBelow_.size())
+ {
+ FatalErrorInFunction
+ << "Problem: " << nNotBelow << " != " << allNotBelow_.size() << nl
+ << Foam::abort(FatalError);
+ }
+}
+
+
+// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
+
+// This outputs as depth-first, but graphviz sorts that for us
+void Foam::UPstream::commsStruct::printGraph
+(
+ Ostream& os,
+ const UList<UPstream::commsStruct>& comms,
+ const label proci
+)
{
- boolList inBelow(nProcs, false);
+ // if (proci >= comms.size()) return; // Extreme safety!
+
+ const auto& below = comms[proci].below();
- forAll(allBelow, belowI)
+ if (proci == 0)
{
- inBelow[allBelow[belowI]] = true;
+ os << nl << "// communication graph:" << nl;
+ os.beginBlock("graph");
+
+ if (below.empty())
+ {
+ // A graph with a single-node (eg, self-comm)
+ os << indent << proci << nl;
+ }
}
- label notI = 0;
- forAll(inBelow, proci)
+ int pos = 0;
+
+ for (const label nbrProci : below)
{
- if ((proci != myProcID) && !inBelow[proci])
+ if (pos)
+ {
+ os << " ";
+ }
+ else
+ {
+ os << indent;
+ }
+ os << proci << " -- " << nbrProci;
+
+ if (++pos >= 4) // Max 4 items per line
{
- allNotBelow_[notI++] = proci;
+ pos = 0;
+ os << nl;
}
}
- if (notI != allNotBelow_.size())
+
+ if (pos)
+ {
+ os << nl;
+ }
+
+ for (const label nbrProci : below)
{
- FatalErrorInFunction << "problem!" << Foam::abort(FatalError);
+ // if (proci == nbrProci) continue; // Extreme safety!
+ printGraph(os, comms, nbrProci);
+ }
+
+ if (proci == 0)
+ {
+ os.endBlock();
+
+ os << "// end graph" << nl;
}
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
-void Foam::UPstream::commsStruct::clear()
+Foam::label Foam::UPstream::commsStruct::nProcs() const
+{
+ return (1 + allBelow_.size() + allNotBelow_.size());
+}
+
+
+void Foam::UPstream::commsStruct::reset()
{
above_ = -1;
below_.clear();
@@ -92,6 +173,127 @@ void Foam::UPstream::commsStruct::clear()
}
+void Foam::UPstream::commsStruct::reset
+(
+ const label procID,
+ const label numProcs
+)
+{
+ reset();
+
+ label above(-1);
+ DynamicList<label> below;
+ DynamicList<label> allBelow;
+
+ if (numProcs < UPstream::nProcsSimpleSum)
+ {
+ // Linear schedule
+
+ if (procID == 0)
+ {
+ below = identity(numProcs-1, 1);
+ allBelow = below;
+ }
+ else
+ {
+ above = 0;
+ }
+ }
+ else
+ {
+ // Use tree like schedule. For 8 procs:
+ // (level 0)
+ // 0 receives from 1
+ // 2 receives from 3
+ // 4 receives from 5
+ // 6 receives from 7
+ // (level 1)
+ // 0 receives from 2
+ // 4 receives from 6
+ // (level 2)
+ // 0 receives from 4
+ //
+ // The sends/receives for all levels are collected per processor
+ // (one send per processor; multiple receives possible) creating
+ // a table:
+ //
+ // So per processor:
+ // proc receives from sends to
+ // ---- ------------- --------
+ // 0 1,2,4 -
+ // 1 - 0
+ // 2 3 0
+ // 3 - 2
+ // 4 5 0
+ // 5 - 4
+ // 6 7 4
+ // 7 - 6
+
+ label mod = 0;
+
+ for (label step = 1; step < numProcs; step = mod)
+ {
+ mod = step * 2;
+
+ if (procID % mod)
+ {
+ above = procID - (procID % mod);
+ break;
+ }
+ else
+ {
+ for
+ (
+ label j = procID + step;
+ j < numProcs && j < procID + mod;
+ j += step
+ )
+ {
+ below.push_back(j);
+ }
+ for
+ (
+ label j = procID + step;
+ j < numProcs && j < procID + mod;
+ j++
+ )
+ {
+ allBelow.push_back(j);
+ }
+ }
+ }
+ }
+
+ *this = UPstream::commsStruct(numProcs, procID, above, below, allBelow);
+}
+
+
+// * * * * * * * * * * * * * * * Specializations * * * * * * * * * * * * * * //
+
+template<>
+Foam::UPstream::commsStruct&
+Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID)
+{
+ auto& val = this->v_[procID]; // or this->data()[procID]
+
+ if (val.nProcs() != size())
+ {
+ // Create/update
+ val.reset(procID, size());
+ }
+
+ return val;
+}
+
+
+template<>
+const Foam::UPstream::commsStruct&
+Foam::UList<Foam::UPstream::commsStruct>::operator[](const label procID) const
+{
+ return const_cast<UList<UPstream::commsStruct>&>(*this).operator[](procID);
+}
+
+
// * * * * * * * * * * * * * * * Member Operators * * * * * * * * * * * * * //
bool Foam::UPstream::commsStruct::operator==(const commsStruct& comm) const
@@ -100,8 +302,8 @@ bool Foam::UPstream::commsStruct::operator==(const commsStruct& comm) const
(
(above_ == comm.above())
&& (below_ == comm.below())
- && (allBelow_ == allBelow())
- && (allNotBelow_ == allNotBelow())
+ // && (allBelow_ == comm.allBelow())
+ // && (allNotBelow_ == comm.allNotBelow())
);
}
@@ -116,10 +318,10 @@ bool Foam::UPstream::commsStruct::operator!=(const commsStruct& comm) const
Foam::Ostream& Foam::operator<<(Ostream& os, const UPstream::commsStruct& comm)
{
- os << comm.above_ << token::SPACE
- << comm.below_ << token::SPACE
- << comm.allBelow_ << token::SPACE
- << comm.allNotBelow_;
+ os << comm.above() << nl << token::SPACE << token::SPACE;
+ comm.below().writeList(os) << nl << token::SPACE << token::SPACE;
+ comm.allBelow().writeList(os) << nl << token::SPACE << token::SPACE;
+ comm.allNotBelow().writeList(os);
os.check(FUNCTION_NAME);
return os;
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C
index f49db33303dc0f6198f35f9b1c92c16f7b2c91a3..386655461416ddf4613f00ae9c4a43dd9c8ee262 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/eagerGAMGProcAgglomeration/eagerGAMGProcAgglomeration.C
@@ -65,10 +65,7 @@ Foam::eagerGAMGProcAgglomeration::
{
forAllReverse(comms_, i)
{
- if (comms_[i] != -1)
- {
- UPstream::freeCommunicator(comms_[i]);
- }
+ UPstream::freeCommunicator(comms_[i]);
}
}
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
index 289f6a8aede0e948201f37af6fb241fd6f5cf3eb..a5d266015fc19391141cc788bec862403b957ad8 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/manualGAMGProcAgglomeration/manualGAMGProcAgglomeration.C
@@ -64,10 +64,7 @@ Foam::manualGAMGProcAgglomeration::
{
forAllReverse(comms_, i)
{
- if (comms_[i] != -1)
- {
- UPstream::freeCommunicator(comms_[i]);
- }
+ UPstream::freeCommunicator(comms_[i]);
}
}
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
index 20af4576cdc55410e86358e2419236b43871d827..180ef2d385ad8d6bc237082f5d875d177efe9b1c 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C
@@ -104,10 +104,7 @@ Foam::masterCoarsestGAMGProcAgglomeration::
{
forAllReverse(comms_, i)
{
- if (comms_[i] != -1)
- {
- UPstream::freeCommunicator(comms_[i]);
- }
+ UPstream::freeCommunicator(comms_[i]);
}
}
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/procFacesGAMGProcAgglomeration/procFacesGAMGProcAgglomeration.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/procFacesGAMGProcAgglomeration/procFacesGAMGProcAgglomeration.C
index 9202dd024188d48de7a23f67bee2a279d91664c6..3fe22a5e2fb98fe9c1e95f462bb7a490c1d0d71f 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/procFacesGAMGProcAgglomeration/procFacesGAMGProcAgglomeration.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/procFacesGAMGProcAgglomeration/procFacesGAMGProcAgglomeration.C
@@ -149,10 +149,10 @@ Foam::procFacesGAMGProcAgglomeration::processorAgglomeration
const lduMesh& mesh
) const
{
- label singleCellMeshComm = UPstream::allocateCommunicator
+ UPstream::communicator singleCellMeshComm
(
mesh.comm(),
- labelList(1, Zero) // only processor 0
+ labelList(Foam::one{}, 0) // only processor 0
);
scalarField faceWeights;
@@ -160,14 +160,14 @@ Foam::procFacesGAMGProcAgglomeration::processorAgglomeration
(
singleCellMesh
(
- singleCellMeshComm,
+ singleCellMeshComm.comm(),
mesh,
faceWeights
)
);
- tmp<labelField> tfineToCoarse(new labelField(0));
- labelField& fineToCoarse = tfineToCoarse.ref();
+ auto tfineToCoarse = tmp<labelField>::New();
+ auto& fineToCoarse = tfineToCoarse.ref();
if (singleCellMeshPtr)
{
@@ -197,7 +197,7 @@ Foam::procFacesGAMGProcAgglomeration::processorAgglomeration
}
Pstream::broadcast(fineToCoarse, mesh.comm());
- UPstream::freeCommunicator(singleCellMeshComm);
+ singleCellMeshComm.reset();
return tfineToCoarse;
}
@@ -234,10 +234,7 @@ Foam::procFacesGAMGProcAgglomeration::~procFacesGAMGProcAgglomeration()
{
forAllReverse(comms_, i)
{
- if (comms_[i] != -1)
- {
- UPstream::freeCommunicator(comms_[i]);
- }
+ UPstream::freeCommunicator(comms_[i]);
}
}
diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C
index b65115d7c745041e7059398a7ac774b2e97b7cab..feba605416d5f79c2c2a751ba6f469b4cf5a8f2d 100644
--- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C
+++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C
@@ -2714,13 +2714,14 @@ void Foam::globalMeshData::updateMesh()
// *** Temporary hack to avoid problems with overlapping communication
// *** between these reductions and the calculation of deltaCoeffs
- //const label comm = UPstream::worldComm + 1;
- const label comm = UPstream::allocateCommunicator
+
+ UPstream::communicator dupComm
(
UPstream::worldComm,
- identity(UPstream::nProcs(UPstream::worldComm)),
- true
+ identity(UPstream::nProcs(UPstream::worldComm))
);
+
+ const label comm = dupComm.comm();
const label oldWarnComm = UPstream::commWarn(comm);
@@ -2760,8 +2761,8 @@ void Foam::globalMeshData::updateMesh()
);
// Restore communicator settings
- UPstream::freeCommunicator(comm);
UPstream::commWarn(oldWarnComm);
+ dupComm.reset();
if (debug)
{
diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C
index 576626ab51614afe466ed1ca219ab457b1cf49f4..d8416661990d9ce1556fcc12065f298d3b19cba4 100644
--- a/src/Pstream/mpi/UPstream.C
+++ b/src/Pstream/mpi/UPstream.C
@@ -335,12 +335,7 @@ bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread)
// Allocate new communicator with globalComm as its parent
const label subComm =
- UPstream::allocateCommunicator
- (
- UPstream::globalComm, // parent
- subRanks,
- true
- );
+ UPstream::allocateCommunicator(UPstream::globalComm, subRanks);
// Override worldComm
diff --git a/src/meshTools/multiWorld/multiWorldConnectionsObject.C b/src/meshTools/multiWorld/multiWorldConnectionsObject.C
index 8b752d289a71d5aebefd067ff785e8dd7f93e707..2dc3d74deae1188c273ac9522fc19508f5707551 100644
--- a/src/meshTools/multiWorld/multiWorldConnectionsObject.C
+++ b/src/meshTools/multiWorld/multiWorldConnectionsObject.C
@@ -157,13 +157,12 @@ Foam::label Foam::multiWorldConnections::createCommunicator(const edge& worlds)
{
if (worlds.found(worldIDs[proci]))
{
- subRanks.append(proci);
+ subRanks.push_back(proci);
}
}
// Allocate new communicator with global world
- comm =
- UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks, true);
+ comm = UPstream::allocateCommunicator(UPstream::commGlobal(), subRanks);
if (debug & 2)
{