Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
openfoam
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Deploy
Releases
Model registry
Analyze
Contributor analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Development
openfoam
Commits
a01f3ed8
Commit
a01f3ed8
authored
3 months ago
by
Mark OLESEN
Committed by
Mark OLESEN
3 months ago
Browse files
Options
Downloads
Patches
Plain Diff
ENH: add node-based gatherList()
parent
c4b261c6
Branches
Branches containing commit
Tags
Tags containing commit
1 merge request
!731
additional topology-aware handling for Pstream
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H
+11
-0
11 additions, 0 deletions
src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H
src/OpenFOAM/db/IOstreams/Pstreams/PstreamGatherList.txx
+139
-1
139 additions, 1 deletion
src/OpenFOAM/db/IOstreams/Pstreams/PstreamGatherList.txx
with
150 additions
and
1 deletion
src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H
+
11
−
0
View file @
a01f3ed8
...
@@ -459,6 +459,17 @@ public:
...
@@ -459,6 +459,17 @@ public:
const
int
communicator
const
int
communicator
);
);
//- Gather data, keeping individual values separate.
// \returns True if topo algorithm was applied
template
<
class
T
>
static
bool
gatherList_topo_algorithm
(
//! [in,out]
UList
<
T
>&
values
,
const
int
tag
,
const
int
communicator
);
//- Implementation: inverse of gatherList_algorithm
//- Implementation: inverse of gatherList_algorithm
template
<
class
T
>
template
<
class
T
>
static
void
scatterList_algorithm
static
void
scatterList_algorithm
...
...
This diff is collapsed.
Click to expand it.
src/OpenFOAM/db/IOstreams/Pstreams/PstreamGatherList.txx
+
139
−
1
View file @
a01f3ed8
...
@@ -252,6 +252,127 @@ void Foam::Pstream::gatherList_algorithm
...
@@ -252,6 +252,127 @@ void Foam::Pstream::gatherList_algorithm
}
}
template<class T>
bool Foam::Pstream::gatherList_topo_algorithm
(
UList<T>& values,
const int tag,
const int communicator
)
{
const bool withTopo =
(
UPstream::is_parallel(communicator)
&& UPstream::usingTopoControl(UPstream::topoControls::gatherList)
&& UPstream::usingNodeComms(communicator)
);
if (withTopo)
{
// Topological gathering
if (FOAM_UNLIKELY(values.size() < UPstream::nProcs(communicator)))
{
FatalErrorInFunction
<< "List of values:" << values.size()
<< " < numProcs:" << UPstream::nProcs(communicator) << nl
<< Foam::abort(FatalError);
}
// Overall node-wise offsets
const auto& off = UPstream::interNode_offsets();
// The per-node processor range
const auto& nodeProcs = UPstream::localNode_parentProcs();
// The per-node sub-section of values
auto nodeValues = values.slice(nodeProcs.start(), nodeProcs.size());
// Stage 1: gather values within a node
// - linear for local-node (assume communication is fast)
if (UPstream::is_parallel(UPstream::commLocalNode()))
{
const auto subComm = UPstream::commLocalNode();
constexpr bool linear(true);
Pstream::gatherList_algorithm<T>
(
UPstream::whichCommunication(subComm, linear),
nodeValues,
tag,
subComm
);
}
// Stage 2: gather between node leaders
// - this unfortunately corresponds to a gatherv process
// (number of cores per node is not identical)
// - code strongly resembles globalIndex::gather
if (UPstream::is_parallel(UPstream::commInterNode()))
{
const auto subComm = UPstream::commInterNode();
if (UPstream::master(subComm))
{
for (const int proci : UPstream::subProcs(subComm))
{
auto slot =
values.slice(off[proci], off[proci+1]-off[proci]);
// Probably not contiguous though,
// otherwise would have used mpiGather()
if constexpr (is_contiguous_v<T>)
{
UIPstream::read
(
UPstream::commsTypes::scheduled,
proci,
slot,
tag,
subComm
);
}
else
{
IPstream::recv(slot, proci, tag, subComm);
}
}
}
else
{
if constexpr (is_contiguous_v<T>)
{
UOPstream::write
(
UPstream::commsTypes::scheduled,
UPstream::masterNo(),
nodeValues,
tag,
subComm
);
}
else
{
OPstream::send
(
nodeValues,
UPstream::commsTypes::scheduled,
UPstream::masterNo(),
tag,
subComm
);
}
}
}
}
return withTopo;
}
template<class T>
template<class T>
void Foam::Pstream::scatterList_algorithm
void Foam::Pstream::scatterList_algorithm
(
(
...
@@ -448,7 +569,15 @@ void Foam::Pstream::gatherList
...
@@ -448,7 +569,15 @@ void Foam::Pstream::gatherList
auto* ptr = values.data() + UPstream::myProcNo(communicator);
auto* ptr = values.data() + UPstream::myProcNo(communicator);
UPstream::mpiGather(ptr, ptr, 1, communicator);
UPstream::mpiGather(ptr, ptr, 1, communicator);
}
}
else
else if
(
!Pstream::gatherList_topo_algorithm
(
values,
tag,
communicator
)
)
{
{
// Communication order
// Communication order
const auto& commOrder = UPstream::whichCommunication(communicator);
const auto& commOrder = UPstream::whichCommunication(communicator);
...
@@ -486,6 +615,15 @@ void Foam::Pstream::allGatherList
...
@@ -486,6 +615,15 @@ void Foam::Pstream::allGatherList
}
}
else
else
{
{
// IMPORTANT: always call the *_algorithm() versions here and
// never the base versions [eg, Pstream::gatherList()] since
// the communcation order must be absolutely identical
// for the gatherList and scatterList, otherwise the results will
// not replicate the allGather behaviour.
//
// This also means that we must avoid the gatherList_topo_algorithm()
// as well, since this does not pair well with scatterList_algorithm()
// Communication order
// Communication order
const auto& commOrder = UPstream::whichCommunication(communicator);
const auto& commOrder = UPstream::whichCommunication(communicator);
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment