From c6a0beb91e81e66489f6d65b91fec3afb0f06c24 Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Mon, 14 Feb 2022 12:12:32 +0100
Subject: [PATCH] ENH: Pstream specialization for float/scalar, FixedList
(#2351)
- native MPI min/max/sum reductions for float/double
irrespective of WM_PRECISION_OPTION
- native MPI min/max/sum reductions for (u)int32_t/(u)int64_t types,
irrespective of WM_LABEL_SIZE
- replace rarely used vector2D sum reduction with FixedList as a
indicator of its intent and also generalizes to different lengths.
OLD:
vector2D values; values.x() = ...; values.y() = ...;
reduce(values, sumOp<vector2D>());
NEW:
FixedList<scalar,2> values; values[0] = ...; values[1] = ...;
reduce(values, sumOp<scalar>());
- allow returnReduce() to use native reductions. Previous code (with
linear/tree selector) would have bypassed them inadvertently.
ENH: added support for MPI broadcast (for a memory span)
ENH: select communication schedule as a static method
- UPstream::whichCommunication(comm) to select linear/tree
communication instead of ternary or
if (Pstream::nProcs() < Pstream::nProcsSimpleSum) ...
STYLE: align nProcsSimpleSum static value with etc/controlDict override
---
.../Test-parallel-communicators.C | 10 -
.../Pstreams/PstreamCombineReduceOps.H | 45 +--
.../db/IOstreams/Pstreams/PstreamReduceOps.H | 303 ++++++++++--------
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C | 4 +-
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H | 44 ++-
.../IOstreams/Pstreams/combineGatherScatter.C | 155 +++------
.../db/IOstreams/Pstreams/gatherScatter.C | 20 +-
.../db/IOstreams/Pstreams/gatherScatterList.C | 20 +-
.../masterUncollatedFileOperation.C | 18 +-
.../uncollatedFileOperation.C | 9 +-
.../lduMatrix/solvers/GAMG/GAMGSolverScale.C | 20 +-
src/Pstream/dummy/Make/files | 2 +
src/Pstream/dummy/UPstream.C | 97 +-----
src/Pstream/dummy/UPstreamBroadcast.C | 45 +++
src/Pstream/dummy/UPstreamReduce.C | 153 +++++++++
src/Pstream/mpi/Make/files | 2 +
src/Pstream/mpi/PstreamGlobals.H | 8 +-
src/Pstream/mpi/UPstream.C | 297 +----------------
src/Pstream/mpi/UPstreamBroadcast.C | 93 ++++++
src/Pstream/mpi/UPstreamReduce.C | 211 ++++++++++++
src/Pstream/mpi/allReduce.H | 43 ++-
src/Pstream/mpi/allReduceTemplates.C | 210 +++++-------
.../patchMeanVelocityForce.C | 39 +--
.../state/lumpedPointState.C | 10 +-
24 files changed, 925 insertions(+), 933 deletions(-)
create mode 100644 src/Pstream/dummy/UPstreamBroadcast.C
create mode 100644 src/Pstream/dummy/UPstreamReduce.C
create mode 100644 src/Pstream/mpi/UPstreamBroadcast.C
create mode 100644 src/Pstream/mpi/UPstreamReduce.C
diff --git a/applications/test/parallel-communicators/Test-parallel-communicators.C b/applications/test/parallel-communicators/Test-parallel-communicators.C
index 99a3158d060..6c4662d0118 100644
--- a/applications/test/parallel-communicators/Test-parallel-communicators.C
+++ b/applications/test/parallel-communicators/Test-parallel-communicators.C
@@ -163,16 +163,6 @@ int main(int argc, char *argv[])
if (Pstream::myProcNo(comm) != -1)
{
- //scalar sum = sumReduce(comm, localValue);
- //scalar sum = localValue;
- //reduce
- //(
- // UPstream::treeCommunication(comm),
- // sum,
- // sumOp<scalar>(),
- // Pstream::msgType(),
- // comm
- //);
scalar sum = returnReduce
(
localValue,
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamCombineReduceOps.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
index 23dd721a114..7ab8c9dbd9a 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
@@ -35,8 +35,8 @@ Description
\*---------------------------------------------------------------------------*/
-#ifndef PstreamCombineReduceOps_H
-#define PstreamCombineReduceOps_H
+#ifndef Foam_PstreamCombineReduceOps_H
+#define Foam_PstreamCombineReduceOps_H
#include "UPstream.H"
#include "Pstream.H"
@@ -73,42 +73,11 @@ void combineReduce
const label comm = Pstream::worldComm
)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- Pstream::combineGather
- (
- UPstream::linearCommunication(comm),
- Value,
- cop,
- tag,
- comm
- );
- Pstream::combineScatter
- (
- UPstream::linearCommunication(comm),
- Value,
- tag,
- comm
- );
- }
- else
- {
- Pstream::combineGather
- (
- UPstream::treeCommunication(comm),
- Value,
- cop,
- tag,
- comm
- );
- Pstream::combineScatter
- (
- UPstream::treeCommunication(comm),
- Value,
- tag,
- comm
- );
- }
+ const List<UPstream::commsStruct>& comms =
+ UPstream::whichCommunication(comm);
+
+ Pstream::combineGather(comms, Value, cop, tag, comm);
+ Pstream::combineScatter(comms, Value, tag, comm);
}
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
index 2741d562bd5..669a37d2487 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
- Copyright (C) 2016 OpenCFD Ltd.
+ Copyright (C) 2016-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -32,11 +32,11 @@ Description
\*---------------------------------------------------------------------------*/
-#ifndef PstreamReduceOps_H
-#define PstreamReduceOps_H
+#ifndef Foam_PstreamReduceOps_H
+#define Foam_PstreamReduceOps_H
#include "ops.H"
-#include "vector2D.H"
+#include "FixedList.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@@ -45,12 +45,12 @@ namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-// Reduce operation with user specified communication schedule
+//- Reduce operation with user specified communication schedule
template<class T, class BinaryOp>
void reduce
(
const List<UPstream::commsStruct>& comms,
- T& Value,
+ T& value,
const BinaryOp& bop,
const int tag,
const label comm
@@ -58,96 +58,86 @@ void reduce
{
if (UPstream::warnComm != -1 && comm != UPstream::warnComm)
{
- Pout<< "** reducing:" << Value << " with comm:" << comm
+ Pout<< "** reducing:" << value << " with comm:" << comm
<< endl;
error::printStack(Pout);
}
- Pstream::gather(comms, Value, bop, tag, comm);
- Pstream::scatter(comms, Value, tag, comm);
+ Pstream::gather(comms, value, bop, tag, comm);
+ Pstream::scatter(comms, value, tag, comm);
}
-// Reduce using either linear or tree communication schedule
+//- Reduce (inplace) using either linear or tree communication schedule
template<class T, class BinaryOp>
void reduce
(
- T& Value,
+ T& value,
const BinaryOp& bop,
- const int tag = Pstream::msgType(),
+ const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- reduce(UPstream::linearCommunication(comm), Value, bop, tag, comm);
- }
- else
+ if (UPstream::parRun())
{
- reduce(UPstream::treeCommunication(comm), Value, bop, tag, comm);
+ reduce(UPstream::whichCommunication(comm), value, bop, tag, comm);
}
}
-// Reduce using either linear or tree communication schedule
+//- Reduce (copy) and return value
template<class T, class BinaryOp>
T returnReduce
(
- const T& Value,
+ const T& value,
const BinaryOp& bop,
- const int tag = Pstream::msgType(),
+ const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
)
{
- T WorkValue(Value);
-
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- reduce
- (
- UPstream::linearCommunication(comm),
- WorkValue,
- bop,
- tag,
- comm
- );
- }
- else
- {
- reduce
- (
- UPstream::treeCommunication(comm),
- WorkValue,
- bop,
- tag,
- comm
- );
- }
-
- return WorkValue;
+ T work(value);
+ reduce(work, bop, tag, comm);
+ return work;
}
-// Reduce with sum of both value and count (for averaging)
+//- Reduce with sum of both value and count (for averaging)
template<class T>
void sumReduce
(
- T& Value,
- label& Count,
- const int tag = Pstream::msgType(),
+ T& value,
+ label& count,
+ const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
)
{
- reduce(Value, sumOp<T>(), tag, comm);
- reduce(Count, sumOp<label>(), tag, comm);
+ if (UPstream::parRun())
+ {
+ reduce(value, sumOp<T>(), tag, comm);
+ reduce(count, sumOp<label>(), tag, comm);
+ }
}
-// Non-blocking version of reduce. Sets request.
+//- Reduce multiple values (identical size on all processes!)
+template<class T, class BinaryOp>
+void reduce
+(
+ T values[],
+ const int size,
+ const BinaryOp&,
+ const int tag,
+ const label comm
+)
+{
+ NotImplemented;
+}
+
+//- Non-blocking reduce single value. Sets request.
template<class T, class BinaryOp>
void reduce
(
T& Value,
- const BinaryOp& bop,
+ const BinaryOp&,
const int tag,
const label comm,
label& request
@@ -156,13 +146,14 @@ void reduce
NotImplemented;
}
-// Non-blocking version of reduce. Sets request.
+//- Non-blocking reduce multiple values (identical size on all processes!)
+//- Sets request.
template<class T, class BinaryOp>
void reduce
(
- T Value[],
+ T values[],
const int size,
- const BinaryOp& bop,
+ const BinaryOp&,
const int tag,
const label comm,
label& request
@@ -172,111 +163,139 @@ void reduce
}
-// Insist there are specialisations for the common reductions of scalar(s)
-void reduce
-(
- scalar& Value,
- const sumOp<scalar>& bop,
- const int tag = Pstream::msgType(),
- const label comm = UPstream::worldComm
-);
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for bool
+//- Logical (and) reduction
void reduce
(
- scalar& Value,
- const minOp<scalar>& bop,
- const int tag = Pstream::msgType(),
+ bool& value,
+ const andOp<bool>&,
+ const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
);
+//- Logical (or) reduction
void reduce
(
- vector2D& Value,
- const sumOp<vector2D>& bop,
- const int tag = Pstream::msgType(),
+ bool& value,
+ const orOp<bool>&,
+ const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
);
-void sumReduce
-(
- scalar& Value,
- label& Count,
- const int tag = Pstream::msgType(),
- const label comm = UPstream::worldComm
-);
-void reduce
-(
- scalar& Value,
- const sumOp<scalar>& bop,
- const int tag,
- const label comm,
- label& request
-);
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-void reduce
-(
- scalar Value[],
- const int size,
- const sumOp<scalar>& bop,
- const int tag,
- const label comm,
- label& request
-);
+// Specialisations for common reduction types
+
+#undef Pstream_CommonReductions
+#define Pstream_CommonReductions(Native) \
+ \
+/*! \brief Reduce (min) single Native value */ \
+void reduce \
+( \
+ Native& value, \
+ const minOp<Native>&, \
+ const int tag = UPstream::msgType(), \
+ const label comm = UPstream::worldComm \
+); \
+ \
+/*! \brief Reduce (max) single Native value */ \
+void reduce \
+( \
+ Native& value, \
+ const maxOp<Native>&, \
+ const int tag = UPstream::msgType(), \
+ const label comm = UPstream::worldComm \
+); \
+ \
+/*! \brief Reduce (sum) single Native value */ \
+void reduce \
+( \
+ Native& value, \
+ const sumOp<Native>&, \
+ const int tag = UPstream::msgType(), \
+ const label comm = UPstream::worldComm \
+); \
+ \
+/*! \brief Reduce (sum) multiple Native values (identical size all procs!) */ \
+void reduce \
+( \
+ Native values[], \
+ const int size, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm \
+); \
+ \
+/*! \brief Reduce (sum) multiple Native values */ \
+template<unsigned N> \
+inline void reduce \
+( \
+ FixedList<Native, N>& values, \
+ const sumOp<Native>&, \
+ const int tag = UPstream::msgType(), \
+ const label comm = UPstream::worldComm \
+) \
+{ \
+ reduce(values.data(), int(values.size()), sumOp<Native>(), tag, comm); \
+}
-#if defined(WM_SPDP)
-void reduce
-(
- solveScalar& Value,
- const sumOp<solveScalar>& bop,
- const int tag = Pstream::msgType(),
- const label comm = UPstream::worldComm
-);
+Pstream_CommonReductions(int32_t);
+Pstream_CommonReductions(int64_t);
+Pstream_CommonReductions(uint32_t);
+Pstream_CommonReductions(uint64_t);
+Pstream_CommonReductions(float);
+Pstream_CommonReductions(double);
-void reduce
-(
- solveScalar& Value,
- const minOp<solveScalar>& bop,
- const int tag = Pstream::msgType(),
- const label comm = UPstream::worldComm
-);
+#undef Pstream_CommonReductions
-void reduce
-(
- Vector2D<solveScalar>& Value,
- const sumOp<Vector2D<solveScalar>>& bop,
- const int tag = Pstream::msgType(),
- const label comm = UPstream::worldComm
-);
-void sumReduce
-(
- solveScalar& Value,
- label& Count,
- const int tag = Pstream::msgType(),
- const label comm = UPstream::worldComm
-);
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-void reduce
-(
- solveScalar& Value,
- const sumOp<solveScalar>& bop,
- const int tag,
- const label comm,
- label& request
+// Specialisations for floating-point types
+
+#undef Pstream_FloatReductions
+#define Pstream_FloatReductions(Native) \
+ \
+/*! \brief Sum of both Native value and count (for averaging) */ \
+void sumReduce \
+( \
+ Native& value, \
+ label& count, \
+ const int tag = UPstream::msgType(), \
+ const label comm = UPstream::worldComm \
+); \
+ \
+/*! \brief Non-blocking reduce (sum) single Native value. Sets request */ \
+void reduce \
+( \
+ Native& value, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm, \
+ label& requestID \
+); \
+ \
+/*! \brief Non-blocking reduce (sum) multiple Native values. Sets request */ \
+void reduce \
+( \
+ Native values[], \
+ const int size, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm, \
+ label& requestID \
);
-void reduce
-(
- solveScalar Value[],
- const int size,
- const sumOp<solveScalar>& bop,
- const int tag,
- const label comm,
- label& request
-);
-#endif
+
+Pstream_FloatReductions(float);
+Pstream_FloatReductions(double);
+
+#undef Pstream_FloatReductions
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
index 936a50db55e..37ec9d57401 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
- Copyright (C) 2015-2021 OpenCFD Ltd.
+ Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -414,7 +414,7 @@ registerOptSwitch
int Foam::UPstream::nProcsSimpleSum
(
- Foam::debug::optimisationSwitch("nProcsSimpleSum", 16)
+ Foam::debug::optimisationSwitch("nProcsSimpleSum", 0)
);
registerOptSwitch
(
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
index 945bb60559d..44310c4c0d6 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
@@ -69,9 +69,9 @@ public:
//- Types of communications
enum class commsTypes : char
{
- blocking,
- scheduled,
- nonBlocking
+ blocking, //!< "blocking"
+ scheduled, //!< "scheduled"
+ nonBlocking //!< "nonBlocking"
};
//- Names of the communication types
@@ -198,7 +198,7 @@ private:
static labelList worldIDs_;
- // Communicator specific data
+ // Communicator specific data
//- My processor number
static DynamicList<int> myProcNo_;
@@ -413,11 +413,11 @@ public:
static int allocateTag(const char*);
- static int allocateTag(const word&);
+ static int allocateTag(const std::string&);
static void freeTag(const char*, const int tag);
- static void freeTag(const word&, const int tag);
+ static void freeTag(const std::string&, const int tag);
//- Set as parallel run on/off.
@@ -537,13 +537,28 @@ public:
return treeCommunication_[communicator];
}
+ //- Communication schedule for linear/tree all-to-master (proc 0).
+ //- Chooses based on the value of UPstream::nProcsSimpleSum
+ static const List<commsStruct>& whichCommunication
+ (
+ const label communicator = worldComm
+ )
+ {
+ return
+ (
+ nProcs(communicator) < nProcsSimpleSum
+ ? linearCommunication_[communicator]
+ : treeCommunication_[communicator]
+ );
+ }
+
+
//- Message tag of standard messages
static int& msgType() noexcept
{
return msgType_;
}
-
//- Get the communications type of the stream
commsTypes commsType() const noexcept
{
@@ -551,6 +566,7 @@ public:
}
//- Set the communications type of the stream
+ // \return the previous value
commsTypes commsType(const commsTypes ct) noexcept
{
commsTypes old(commsType_);
@@ -668,6 +684,20 @@ public:
);
+
+ // Broadcast Functions
+
+ //- Broadcast buffer contents, sizes must match on processors
+ // \return True on success
+ static bool broadcast
+ (
+ char* buf,
+ const std::streamsize bufSize,
+ const label communicator = worldComm,
+ const int rootProcNo = masterNo()
+ );
+
+
// Housekeeping
//- Process index of first sub-process
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C b/src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C
index e1c4f0cfc89..19a08cd4c0d 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
- Copyright (C) 2019-2021 OpenCFD Ltd.
+ Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -153,28 +153,14 @@ void Foam::Pstream::combineGather
const label comm
)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- combineGather
- (
- UPstream::linearCommunication(comm),
- Value,
- cop,
- tag,
- comm
- );
- }
- else
- {
- combineGather
- (
- UPstream::treeCommunication(comm),
- Value,
- cop,
- tag,
- comm
- );
- }
+ combineGather
+ (
+ UPstream::whichCommunication(comm),
+ Value,
+ cop,
+ tag,
+ comm
+ );
}
@@ -274,14 +260,7 @@ void Foam::Pstream::combineScatter
const label comm
)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- combineScatter(UPstream::linearCommunication(comm), Value, tag, comm);
- }
- else
- {
- combineScatter(UPstream::treeCommunication(comm), Value, tag, comm);
- }
+ combineScatter(UPstream::whichCommunication(comm), Value, tag, comm);
}
@@ -402,28 +381,14 @@ void Foam::Pstream::listCombineGather
const label comm
)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- listCombineGather
- (
- UPstream::linearCommunication(comm),
- Values,
- cop,
- tag,
- comm
- );
- }
- else
- {
- listCombineGather
- (
- UPstream::treeCommunication(comm),
- Values,
- cop,
- tag,
- comm
- );
- }
+ listCombineGather
+ (
+ UPstream::whichCommunication(comm),
+ Values,
+ cop,
+ tag,
+ comm
+ );
}
@@ -523,26 +488,13 @@ void Foam::Pstream::listCombineScatter
const label comm
)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- listCombineScatter
- (
- UPstream::linearCommunication(comm),
- Values,
- tag,
- comm
- );
- }
- else
- {
- listCombineScatter
- (
- UPstream::treeCommunication(comm),
- Values,
- tag,
- comm
- );
- }
+ listCombineScatter
+ (
+ UPstream::whichCommunication(comm),
+ Values,
+ tag,
+ comm
+ );
}
@@ -636,28 +588,14 @@ void Foam::Pstream::mapCombineGather
const label comm
)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- mapCombineGather
- (
- UPstream::linearCommunication(comm),
- Values,
- cop,
- tag,
- comm
- );
- }
- else
- {
- mapCombineGather
- (
- UPstream::treeCommunication(comm),
- Values,
- cop,
- tag,
- comm
- );
- }
+ mapCombineGather
+ (
+ UPstream::whichCommunication(comm),
+ Values,
+ cop,
+ tag,
+ comm
+ );
}
@@ -727,26 +665,13 @@ void Foam::Pstream::mapCombineScatter
const label comm
)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- mapCombineScatter
- (
- UPstream::linearCommunication(comm),
- Values,
- tag,
- comm
- );
- }
- else
- {
- mapCombineScatter
- (
- UPstream::treeCommunication(comm),
- Values,
- tag,
- comm
- );
- }
+ mapCombineScatter
+ (
+ UPstream::whichCommunication(comm),
+ Values,
+ tag,
+ comm
+ );
}
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C b/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C
index e4457ff2ec3..6eea2096987 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
- Copyright (C) 2019 OpenCFD Ltd.
+ Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -134,14 +134,7 @@ void Pstream::gather
const label comm
)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- gather(UPstream::linearCommunication(comm), Value, bop, tag, comm);
- }
- else
- {
- gather(UPstream::treeCommunication(comm), Value, bop, tag, comm);
- }
+ gather(UPstream::whichCommunication(comm), Value, bop, tag, comm);
}
@@ -225,14 +218,7 @@ void Pstream::scatter
template<class T>
void Pstream::scatter(T& Value, const int tag, const label comm)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- scatter(UPstream::linearCommunication(comm), Value, tag, comm);
- }
- else
- {
- scatter(UPstream::treeCommunication(comm), Value, tag, comm);
- }
+ scatter(UPstream::whichCommunication(comm), Value, tag, comm);
}
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C b/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C
index 7de5fd1224d..8ad37ff812e 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
- Copyright (C) 2015-2021 OpenCFD Ltd.
+ Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -199,14 +199,7 @@ void Pstream::gatherList
template<class T>
void Pstream::gatherList(List<T>& Values, const int tag, const label comm)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- gatherList(UPstream::linearCommunication(comm), Values, tag, comm);
- }
- else
- {
- gatherList(UPstream::treeCommunication(comm), Values, tag, comm);
- }
+ gatherList(UPstream::whichCommunication(comm), Values, tag, comm);
}
@@ -341,14 +334,7 @@ void Pstream::scatterList
template<class T>
void Pstream::scatterList(List<T>& Values, const int tag, const label comm)
{
- if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
- {
- scatterList(UPstream::linearCommunication(comm), Values, tag, comm);
- }
- else
- {
- scatterList(UPstream::treeCommunication(comm), Values, tag, comm);
- }
+ scatterList(UPstream::whichCommunication(comm), Values, tag, comm);
}
diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
index c7babbd03ab..01af9920882 100644
--- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation
- Copyright (C) 2019-2021 OpenCFD Ltd.
+ Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -2072,22 +2072,12 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
Pstream::scatter(io.note()); //, Pstream::msgType(), comm_);
- // scatter operation for regIOobjects
+ // Scatter operation for regIOobjects
// Get my communication order
- // const List<Pstream::commsStruct>& comms =
- //(
- // (Pstream::nProcs(comm_) < Pstream::nProcsSimpleSum)
- // ? Pstream::linearCommunication(comm_)
- // : Pstream::treeCommunication(comm_)
- //);
- // const Pstream::commsStruct& myComm = comms[Pstream::myProcNo(comm_)];
const List<Pstream::commsStruct>& comms =
- (
- (Pstream::nProcs(Pstream::worldComm) < Pstream::nProcsSimpleSum)
- ? Pstream::linearCommunication(Pstream::worldComm)
- : Pstream::treeCommunication(Pstream::worldComm)
- );
+ Pstream::whichCommunication(Pstream::worldComm);
+
const Pstream::commsStruct& myComm =
comms[Pstream::myProcNo(Pstream::worldComm)];
diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
index b1369111f61..f776b013a8e 100644
--- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation
- Copyright (C) 2020-2021 OpenCFD Ltd.
+ Copyright (C) 2020-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -655,12 +655,7 @@ bool Foam::fileOperations::uncollatedFileOperation::read
Pstream::scatter(io.note());
// Get my communication order
- const List<Pstream::commsStruct>& comms =
- (
- (Pstream::nProcs() < Pstream::nProcsSimpleSum)
- ? Pstream::linearCommunication()
- : Pstream::treeCommunication()
- );
+ const List<Pstream::commsStruct>& comms = Pstream::whichCommunication();
const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
// Receive from up
diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C
index 3563c4738b4..b616a1bfe48 100644
--- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C
+++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
- Copyright (C) 2017-2019 OpenCFD Ltd.
+ Copyright (C) 2017-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -27,7 +27,7 @@ License
\*---------------------------------------------------------------------------*/
#include "GAMGSolver.H"
-#include "vector2D.H"
+#include "FixedList.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
@@ -58,21 +58,21 @@ void Foam::GAMGSolver::scale
const solveScalar* const __restrict__ AcfPtr = Acf.begin();
- solveScalar scalingFactorNum = 0.0;
- solveScalar scalingFactorDenom = 0.0;
+ FixedList<solveScalar, 2> scalingFactor(Zero);
for (label i=0; i<nCells; i++)
{
- scalingFactorNum += sourcePtr[i]*fieldPtr[i];
- scalingFactorDenom += AcfPtr[i]*fieldPtr[i];
+ scalingFactor[0] += fieldPtr[i]*sourcePtr[i];
+ scalingFactor[1] += fieldPtr[i]*AcfPtr[i];
}
- Vector2D<solveScalar> scalingVector(scalingFactorNum, scalingFactorDenom);
- A.mesh().reduce(scalingVector, sumOp<Vector2D<solveScalar>>());
+ A.mesh().reduce(scalingFactor, sumOp<solveScalar>());
const solveScalar sf =
- scalingVector.x()
- /stabilise(scalingVector.y(), pTraits<solveScalar>::vsmall);
+ (
+ scalingFactor[0]
+ / stabilise(scalingFactor[1], pTraits<solveScalar>::vsmall)
+ );
if (debug >= 2)
{
diff --git a/src/Pstream/dummy/Make/files b/src/Pstream/dummy/Make/files
index d1d6ee32079..436bf6ffc40 100644
--- a/src/Pstream/dummy/Make/files
+++ b/src/Pstream/dummy/Make/files
@@ -1,4 +1,6 @@
UPstream.C
+UPstreamBroadcast.C
+UPstreamReduce.C
UIPstreamRead.C
UOPstreamWrite.C
diff --git a/src/Pstream/dummy/UPstream.C b/src/Pstream/dummy/UPstream.C
index 4f951e09ca0..0f73b4b89f2 100644
--- a/src/Pstream/dummy/UPstream.C
+++ b/src/Pstream/dummy/UPstream.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2018 OpenFOAM Foundation
- Copyright (C) 2016-2021 OpenCFD Ltd.
+ Copyright (C) 2016-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -27,7 +27,6 @@ License
\*---------------------------------------------------------------------------*/
#include "Pstream.H"
-#include "PstreamReduceOps.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@@ -74,99 +73,7 @@ void Foam::UPstream::abort()
}
-void Foam::reduce(scalar&, const sumOp<scalar>&, const int, const label)
-{}
-
-
-void Foam::reduce(scalar&, const minOp<scalar>&, const int, const label)
-{}
-
-
-void Foam::reduce(vector2D&, const sumOp<vector2D>&, const int, const label)
-{}
-
-
-void Foam::sumReduce
-(
- scalar&,
- label&,
- const int,
- const label
-)
-{}
-
-
-void Foam::reduce(scalar&, const sumOp<scalar>&, const int, const label, label&)
-{}
-
-
-void Foam::reduce
-(
- scalar[],
- const int,
- const sumOp<scalar>&,
- const int,
- const label,
- label&
-)
-{}
-
-
-#if defined(WM_SPDP)
-void Foam::reduce
-(
- solveScalar& Value,
- const sumOp<solveScalar>& bop,
- const int tag,
- const label comm
-)
-{}
-void Foam::reduce
-(
- solveScalar& Value,
- const minOp<solveScalar>& bop,
- const int tag,
- const label comm
-)
-{}
-void Foam::reduce
-(
- Vector2D<solveScalar>& Value,
- const sumOp<Vector2D<solveScalar>>& bop,
- const int tag,
- const label comm
-)
-{}
-void Foam::sumReduce
-(
- solveScalar& Value,
- label& Count,
- const int tag,
- const label comm
-)
-{}
-void Foam::reduce
-(
- solveScalar& Value,
- const sumOp<solveScalar>& bop,
- const int tag,
- const label comm,
- label& request
-)
-{}
-void Foam::reduce
-(
- solveScalar[],
- const int,
- const sumOp<solveScalar>&,
- const int,
- const label,
- label&
-)
-{}
-#endif
-
-
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
void Foam::UPstream::allToAll
(
diff --git a/src/Pstream/dummy/UPstreamBroadcast.C b/src/Pstream/dummy/UPstreamBroadcast.C
new file mode 100644
index 00000000000..b4ace0a112d
--- /dev/null
+++ b/src/Pstream/dummy/UPstreamBroadcast.C
@@ -0,0 +1,45 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | www.openfoam.com
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+ Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UPstream.H"
+
+// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
+
+bool Foam::UPstream::broadcast
+(
+ char* buf,
+ const std::streamsize bufSize,
+ const label communicator,
+ const int rootProcNo
+)
+{
+ // Nothing to do - ignore
+ return true;
+}
+
+
+// ************************************************************************* //
diff --git a/src/Pstream/dummy/UPstreamReduce.C b/src/Pstream/dummy/UPstreamReduce.C
new file mode 100644
index 00000000000..cef48527015
--- /dev/null
+++ b/src/Pstream/dummy/UPstreamReduce.C
@@ -0,0 +1,153 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | www.openfoam.com
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+ Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "Pstream.H"
+#include "PstreamReduceOps.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for bool
+
+void Foam::reduce
+(
+ bool& value,
+ const andOp<bool>&,
+ const int tag,
+ const label comm
+)
+{}
+
+void Foam::reduce
+(
+ bool& value,
+ const orOp<bool>&,
+ const int tag,
+ const label comm
+)
+{}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for common reduction types
+
+#undef Pstream_CommonReductions
+#define Pstream_CommonReductions(Native) \
+ \
+void Foam::reduce \
+( \
+ Native& value, \
+ const minOp<Native>&, \
+ const int tag, \
+ const label comm \
+) \
+{} \
+ \
+void Foam::reduce \
+( \
+ Native& value, \
+ const maxOp<Native>&, \
+ const int tag, \
+ const label comm \
+) \
+{} \
+ \
+void Foam::reduce \
+( \
+ Native& value, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm \
+) \
+{} \
+ \
+void Foam::reduce \
+( \
+ Native values[], \
+ const int size, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm \
+) \
+{}
+
+
+Pstream_CommonReductions(int32_t);
+Pstream_CommonReductions(int64_t);
+Pstream_CommonReductions(uint32_t);
+Pstream_CommonReductions(uint64_t);
+Pstream_CommonReductions(float);
+Pstream_CommonReductions(double);
+
+#undef Pstream_CommonReductions
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for floating-point types
+
+#undef Pstream_FloatReductions
+#define Pstream_FloatReductions(Native) \
+ \
+void Foam::sumReduce \
+( \
+ Native& value, \
+ label& count, \
+ const int tag, \
+ const label comm \
+) \
+{} \
+ \
+void Foam::reduce \
+( \
+ Native& value, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm, \
+ label& requestID \
+) \
+{} \
+ \
+void Foam::reduce \
+( \
+ Native values[], \
+ const int size, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm, \
+ label& requestID \
+) \
+{}
+
+
+Pstream_FloatReductions(float);
+Pstream_FloatReductions(double);
+
+#undef Pstream_FloatReductions
+
+
+// ************************************************************************* //
diff --git a/src/Pstream/mpi/Make/files b/src/Pstream/mpi/Make/files
index e16e96a25f3..551d8092bd7 100644
--- a/src/Pstream/mpi/Make/files
+++ b/src/Pstream/mpi/Make/files
@@ -1,5 +1,7 @@
PstreamGlobals.C
UPstream.C
+UPstreamBroadcast.C
+UPstreamReduce.C
UIPstreamRead.C
UOPstreamWrite.C
diff --git a/src/Pstream/mpi/PstreamGlobals.H b/src/Pstream/mpi/PstreamGlobals.H
index b3e9501d584..3f80c63a9d5 100644
--- a/src/Pstream/mpi/PstreamGlobals.H
+++ b/src/Pstream/mpi/PstreamGlobals.H
@@ -35,8 +35,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/
-#ifndef PstreamGlobals_H
-#define PstreamGlobals_H
+#ifndef Foam_PstreamGlobals_H
+#define Foam_PstreamGlobals_H
#include "DynamicList.H"
#include <mpi.h>
@@ -62,14 +62,12 @@ extern DynamicList<int> freedTags_;
extern DynamicList<MPI_Comm> MPICommunicators_;
extern DynamicList<MPI_Group> MPIGroups_;
-
void checkCommunicator(const label comm, const label toProcNo);
-};
-
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+} // End namespace PstreamGlobals
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C
index eb9061a8426..8b2763bdd2a 100644
--- a/src/Pstream/mpi/UPstream.C
+++ b/src/Pstream/mpi/UPstream.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
- Copyright (C) 2016-2021 OpenCFD Ltd.
+ Copyright (C) 2016-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -40,17 +40,6 @@ License
#include <cstdlib>
#include <csignal>
-#if defined(WM_SP)
- #define MPI_SCALAR MPI_FLOAT
- #define MPI_SOLVESCALAR MPI_FLOAT
-#elif defined(WM_SPDP)
- #define MPI_SCALAR MPI_FLOAT
- #define MPI_SOLVESCALAR MPI_DOUBLE
-#elif defined(WM_DP)
- #define MPI_SCALAR MPI_DOUBLE
- #define MPI_SOLVESCALAR MPI_DOUBLE
-#endif
-
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
// The min value and default for MPI buffers length
@@ -491,245 +480,7 @@ void Foam::UPstream::abort()
}
-void Foam::reduce
-(
- scalar& Value,
- const sumOp<scalar>& bop,
- const int tag,
- const label communicator
-)
-{
- if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
- {
- Pout<< "** reducing:" << Value << " with comm:" << communicator
- << " warnComm:" << UPstream::warnComm
- << endl;
- error::printStack(Pout);
- }
- allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
-}
-
-
-void Foam::reduce
-(
- scalar& Value,
- const minOp<scalar>& bop,
- const int tag,
- const label communicator
-)
-{
- if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
- {
- Pout<< "** reducing:" << Value << " with comm:" << communicator
- << " warnComm:" << UPstream::warnComm
- << endl;
- error::printStack(Pout);
- }
- allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, communicator);
-}
-
-
-void Foam::reduce
-(
- vector2D& Value,
- const sumOp<vector2D>& bop,
- const int tag,
- const label communicator
-)
-{
- if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
- {
- Pout<< "** reducing:" << Value << " with comm:" << communicator
- << " warnComm:" << UPstream::warnComm
- << endl;
- error::printStack(Pout);
- }
- allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
-}
-
-
-void Foam::sumReduce
-(
- scalar& Value,
- label& Count,
- const int tag,
- const label communicator
-)
-{
- if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
- {
- Pout<< "** sumReduce:" << Value << " with comm:" << communicator
- << " warnComm:" << UPstream::warnComm
- << endl;
- error::printStack(Pout);
- }
- vector2D twoScalars(Value, scalar(Count));
- reduce(twoScalars, sumOp<vector2D>(), tag, communicator);
-
- Value = twoScalars.x();
- Count = twoScalars.y();
-}
-
-
-void Foam::reduce
-(
- scalar& Value,
- const sumOp<scalar>& bop,
- const int tag,
- const label communicator,
- label& requestID
-)
-{
- iallReduce<scalar>(&Value, 1, MPI_SCALAR, MPI_SUM, communicator, requestID);
-}
-
-
-void Foam::reduce
-(
- scalar values[],
- const int size,
- const sumOp<scalar>& bop,
- const int tag,
- const label communicator,
- label& requestID
-)
-{
- iallReduce<scalar>
- (
- values,
- size,
- MPI_SCALAR,
- MPI_SUM,
- communicator,
- requestID
- );
-}
-
-
-#if defined(WM_SPDP)
-void Foam::reduce
-(
- solveScalar& Value,
- const sumOp<solveScalar>& bop,
- const int tag,
- const label communicator
-)
-{
- if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
- {
- Pout<< "** reducing:" << Value << " with comm:" << communicator
- << " warnComm:" << UPstream::warnComm
- << endl;
- error::printStack(Pout);
- }
- allReduce(Value, 1, MPI_SOLVESCALAR, MPI_SUM, bop, tag, communicator);
-}
-
-
-void Foam::reduce
-(
- solveScalar& Value,
- const minOp<solveScalar>& bop,
- const int tag,
- const label communicator
-)
-{
- if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
- {
- Pout<< "** reducing:" << Value << " with comm:" << communicator
- << " warnComm:" << UPstream::warnComm
- << endl;
- error::printStack(Pout);
- }
- allReduce(Value, 1, MPI_SOLVESCALAR, MPI_MIN, bop, tag, communicator);
-}
-
-
-void Foam::reduce
-(
- Vector2D<solveScalar>& Value,
- const sumOp<Vector2D<solveScalar>>& bop,
- const int tag,
- const label communicator
-)
-{
- if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
- {
- Pout<< "** reducing:" << Value << " with comm:" << communicator
- << " warnComm:" << UPstream::warnComm
- << endl;
- error::printStack(Pout);
- }
- allReduce(Value, 2, MPI_SOLVESCALAR, MPI_SUM, bop, tag, communicator);
-}
-
-
-void Foam::sumReduce
-(
- solveScalar& Value,
- label& Count,
- const int tag,
- const label communicator
-)
-{
- if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
- {
- Pout<< "** reducing:" << Value << " with comm:" << communicator
- << " warnComm:" << UPstream::warnComm
- << endl;
- error::printStack(Pout);
- }
- Vector2D<solveScalar> twoScalars(Value, solveScalar(Count));
- reduce(twoScalars, sumOp<Vector2D<solveScalar>>(), tag, communicator);
-
- Value = twoScalars.x();
- Count = twoScalars.y();
-}
-
-
-void Foam::reduce
-(
- solveScalar& Value,
- const sumOp<solveScalar>& bop,
- const int tag,
- const label communicator,
- label& requestID
-)
-{
- iallReduce<solveScalar>
- (
- &Value,
- 1,
- MPI_SOLVESCALAR,
- MPI_SUM,
- communicator,
- requestID
- );
-}
-
-
-void Foam::reduce
-(
- solveScalar values[],
- const int size,
- const sumOp<solveScalar>& bop,
- const int tag,
- const label communicator,
- label& requestID
-)
-{
- iallReduce<solveScalar>
- (
- values,
- size,
- MPI_SOLVESCALAR,
- MPI_SUM,
- communicator,
- requestID
- );
-}
-#endif
-
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
void Foam::UPstream::allToAll
(
@@ -1437,22 +1188,15 @@ int Foam::UPstream::allocateTag(const char* s)
if (debug)
{
- //if (UPstream::lateBlocking > 0)
- //{
- // string& poutp = Pout.prefix();
- // poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
- // Perr.prefix() = Pout.prefix();
- //}
- Pout<< "UPstream::allocateTag " << s
- << " : tag:" << tag
- << endl;
+ Pout<< "UPstream::allocateTag "
+ << s << " : tag:" << tag << endl;
}
return tag;
}
-int Foam::UPstream::allocateTag(const word& s)
+int Foam::UPstream::allocateTag(const std::string& s)
{
int tag;
if (PstreamGlobals::freedTags_.size())
@@ -1466,15 +1210,8 @@ int Foam::UPstream::allocateTag(const word& s)
if (debug)
{
- //if (UPstream::lateBlocking > 0)
- //{
- // string& poutp = Pout.prefix();
- // poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
- // Perr.prefix() = Pout.prefix();
- //}
- Pout<< "UPstream::allocateTag " << s
- << " : tag:" << tag
- << endl;
+ Pout<< "UPstream::allocateTag "
+ << s.c_str() << " : tag:" << tag << endl;
}
return tag;
@@ -1485,29 +1222,19 @@ void Foam::UPstream::freeTag(const char* s, const int tag)
{
if (debug)
{
- //if (UPstream::lateBlocking > 0)
- //{
- // string& poutp = Pout.prefix();
- // poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
- // Perr.prefix() = Pout.prefix();
- //}
- Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
+ Pout<< "UPstream::freeTag "
+ << s << " tag:" << tag << endl;
}
PstreamGlobals::freedTags_.append(tag);
}
-void Foam::UPstream::freeTag(const word& s, const int tag)
+void Foam::UPstream::freeTag(const std::string& s, const int tag)
{
if (debug)
{
- //if (UPstream::lateBlocking > 0)
- //{
- // string& poutp = Pout.prefix();
- // poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
- // Perr.prefix() = Pout.prefix();
- //}
- Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
+ Pout<< "UPstream::freeTag "
+ << s.c_str() << " tag:" << tag << endl;
}
PstreamGlobals::freedTags_.append(tag);
}
diff --git a/src/Pstream/mpi/UPstreamBroadcast.C b/src/Pstream/mpi/UPstreamBroadcast.C
new file mode 100644
index 00000000000..7678f136f5a
--- /dev/null
+++ b/src/Pstream/mpi/UPstreamBroadcast.C
@@ -0,0 +1,93 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | www.openfoam.com
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+ Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UPstream.H"
+#include "PstreamGlobals.H"
+#include "profilingPstream.H"
+
+#include <mpi.h>
+
+// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
+
+bool Foam::UPstream::broadcast
+(
+ char* buf,
+ const std::streamsize bufSize,
+ const label communicator,
+ const int rootProcNo
+)
+{
+ if (!UPstream::parRun() || UPstream::nProcs(communicator) < 2)
+ {
+ // Nothing to do - ignore
+ return true;
+ }
+
+ //Needed? PstreamGlobals::checkCommunicator(communicator, rootProcNo);
+
+ if (debug)
+ {
+ Pout<< "UPstream::broadcast : root:" << rootProcNo
+ << " comm:" << communicator
+ << " size:" << label(bufSize)
+ << Foam::endl;
+ }
+ if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+ {
+ Pout<< "UPstream::broadcast : root:" << rootProcNo
+ << " comm:" << communicator
+ << " size:" << label(bufSize)
+ << " warnComm:" << UPstream::warnComm
+ << Foam::endl;
+ error::printStack(Pout);
+ }
+
+ profilingPstream::beginTiming();
+
+ bool failed = MPI_Bcast
+ (
+ buf,
+ bufSize,
+ MPI_BYTE,
+ rootProcNo,
+ PstreamGlobals::MPICommunicators_[communicator]
+ );
+
+ if (rootProcNo == UPstream::myProcNo(communicator))
+ {
+ profilingPstream::addScatterTime();
+ }
+ else
+ {
+ profilingPstream::addGatherTime();
+ }
+
+ return !failed;
+}
+
+
+// ************************************************************************* //
diff --git a/src/Pstream/mpi/UPstreamReduce.C b/src/Pstream/mpi/UPstreamReduce.C
new file mode 100644
index 00000000000..4b112d3d92a
--- /dev/null
+++ b/src/Pstream/mpi/UPstreamReduce.C
@@ -0,0 +1,211 @@
+/*---------------------------------------------------------------------------*\
+ ========= |
+ \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
+ \\ / O peration |
+ \\ / A nd | www.openfoam.com
+ \\/ M anipulation |
+-------------------------------------------------------------------------------
+ Copyright (C) 2022 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+ This file is part of OpenFOAM.
+
+ OpenFOAM is free software: you can redistribute it and/or modify it
+ under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "Pstream.H"
+#include "PstreamReduceOps.H"
+#include "allReduce.H"
+
+#include <mpi.h>
+#include <cinttypes>
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for bool
+
+void Foam::reduce
+(
+ bool& value,
+ const andOp<bool>&,
+ const int tag,
+ const label comm
+)
+{
+ // This can also work:
+ // PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BAND, tag, comm);
+ PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LAND, tag, comm);
+}
+
+
+void Foam::reduce
+(
+ bool& value,
+ const orOp<bool>&,
+ const int tag,
+ const label comm
+)
+{
+ // This can also work:
+ // PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BOR, tag, comm);
+ PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LOR, tag, comm);
+}
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for common reduction types
+
+#undef Pstream_CommonReductions
+#define Pstream_CommonReductions(Native, TaggedType) \
+ \
+void Foam::reduce \
+( \
+ Native& value, \
+ const minOp<Native>&, \
+ const int tag, \
+ const label comm \
+) \
+{ \
+ PstreamDetail::allReduce<Native> \
+ ( \
+ &value, 1, TaggedType, MPI_MIN, tag, comm \
+ ); \
+} \
+ \
+void Foam::reduce \
+( \
+ Native& value, \
+ const maxOp<Native>&, \
+ const int tag, \
+ const label comm \
+) \
+{ \
+ PstreamDetail::allReduce<Native> \
+ ( \
+ &value, 1, TaggedType, MPI_MAX, tag, comm \
+ ); \
+} \
+ \
+void Foam::reduce \
+( \
+ Native& value, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm \
+) \
+{ \
+ PstreamDetail::allReduce<Native> \
+ ( \
+ &value, 1, TaggedType, MPI_SUM, tag, comm \
+ ); \
+} \
+ \
+void Foam::reduce \
+( \
+ Native values[], \
+ const int size, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm \
+) \
+{ \
+ PstreamDetail::allReduce<Native> \
+ ( \
+ values, size, TaggedType, MPI_SUM, tag, comm \
+ ); \
+} \
+
+
+Pstream_CommonReductions(int32_t, MPI_INT32_T);
+Pstream_CommonReductions(int64_t, MPI_INT64_T);
+Pstream_CommonReductions(uint32_t, MPI_UINT32_T);
+Pstream_CommonReductions(uint64_t, MPI_UINT64_T);
+Pstream_CommonReductions(float, MPI_FLOAT);
+Pstream_CommonReductions(double, MPI_DOUBLE);
+
+#undef Pstream_CommonReductions
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+// Specialisations for floating-point types
+
+#undef Pstream_FloatReductions
+#define Pstream_FloatReductions(Native, TaggedType) \
+ \
+void Foam::sumReduce \
+( \
+ Native& value, \
+ label& count, \
+ const int tag, \
+ const label comm \
+) \
+{ \
+ if (UPstream::parRun()) \
+ { \
+ Native values[2]; \
+ values[0] = value; \
+ values[1] = static_cast<Native>(count); \
+ \
+ PstreamDetail::allReduce<Native> \
+ ( \
+ values, 2, TaggedType, MPI_SUM, tag, comm \
+ ); \
+ \
+ value = values[0]; \
+ count = static_cast<label>(values[1]); \
+ } \
+} \
+ \
+void Foam::reduce \
+( \
+ Native& value, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm, \
+ label& requestID \
+) \
+{ \
+ PstreamDetail::iallReduce<Native> \
+ ( \
+ &value, 1, TaggedType, MPI_SUM, comm, requestID \
+ ); \
+} \
+ \
+void Foam::reduce \
+( \
+ Native values[], \
+ const int size, \
+ const sumOp<Native>&, \
+ const int tag, \
+ const label comm, \
+ label& requestID \
+) \
+{ \
+ PstreamDetail::iallReduce<Native> \
+ ( \
+ values, size, TaggedType, MPI_SUM, comm, requestID \
+ ); \
+}
+
+
+Pstream_FloatReductions(float, MPI_FLOAT);
+Pstream_FloatReductions(double, MPI_DOUBLE);
+
+#undef Pstream_FloatReductions
+
+
+// ************************************************************************* //
diff --git a/src/Pstream/mpi/allReduce.H b/src/Pstream/mpi/allReduce.H
index 5982bc2bec0..d81666da020 100644
--- a/src/Pstream/mpi/allReduce.H
+++ b/src/Pstream/mpi/allReduce.H
@@ -6,6 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2012-2016 OpenFOAM Foundation
+ Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -23,37 +24,52 @@ License
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
+Namespace
+ Foam::PstreamDetail
+
+Description
+ Some implementation details for Pstream and/or MPI.
+
InNamespace
- Foam
+ Foam::PstreamDetail
Description
- Various functions to wrap MPI_Allreduce
+ Functions to wrap MPI_Bcast, MPI_Allreduce, MPI_Iallreduce
SourceFiles
allReduceTemplates.C
\*---------------------------------------------------------------------------*/
-#ifndef allReduce_H
-#define allReduce_H
+#ifndef Foam_allReduce_H
+#define Foam_allReduce_H
#include "UPstream.H"
-
#include <mpi.h>
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{
+namespace PstreamDetail
+{
+
+template<class Type>
+void allBroadcast
+(
+ Type* values,
+ int count,
+ MPI_Datatype datatype,
+ const label communicator
+);
-template<class Type, class BinaryOp>
+template<class Type>
void allReduce
(
- Type& Value,
+ Type* values,
int count,
- MPI_Datatype MPIType,
- MPI_Op op,
- const BinaryOp& bop,
+ MPI_Datatype datatype,
+ MPI_Op optype,
const int tag,
const label communicator
);
@@ -61,16 +77,17 @@ void allReduce
template<class Type>
void iallReduce
(
- void* Value,
+ Type* values,
int count,
- MPI_Datatype MPIType,
- MPI_Op op,
+ MPI_Datatype datatype,
+ MPI_Op optype,
const label communicator,
label& requestID
);
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+} // End namespace PstreamDetail
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
diff --git a/src/Pstream/mpi/allReduceTemplates.C b/src/Pstream/mpi/allReduceTemplates.C
index 2c8e928407d..c052ab3b79d 100644
--- a/src/Pstream/mpi/allReduceTemplates.C
+++ b/src/Pstream/mpi/allReduceTemplates.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2012-2015 OpenFOAM Foundation
- Copyright (C) 2019-2020 OpenCFD Ltd.
+ Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -28,18 +28,16 @@ License
#include "allReduce.H"
#include "profilingPstream.H"
+#include "PstreamGlobals.H"
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //
-template<class Type, class BinaryOp>
-void Foam::allReduce
+template<class Type>
+void Foam::PstreamDetail::allBroadcast
(
- Type& Value,
- int MPICount,
- MPI_Datatype MPIType,
- MPI_Op MPIOp,
- const BinaryOp& bop,
- const int tag,
+ Type* values,
+ int count,
+ MPI_Datatype datatype,
const label communicator
)
{
@@ -50,129 +48,76 @@ void Foam::allReduce
profilingPstream::beginTiming();
- if (UPstream::nProcs(communicator) <= UPstream::nProcsSimpleSum)
- {
- if (UPstream::master(communicator))
- {
- for (const int proci : UPstream::subProcs(communicator))
- {
- Type value;
-
- if
- (
- MPI_Recv
- (
- &value,
- MPICount,
- MPIType,
- proci,
- tag,
- PstreamGlobals::MPICommunicators_[communicator],
- MPI_STATUS_IGNORE
- )
- )
- {
- FatalErrorInFunction
- << "MPI_Recv failed"
- << Foam::abort(FatalError);
- }
+ // const int retval =
+ MPI_Bcast
+ (
+ values,
+ count,
+ datatype,
+ 0, // (root process) is master == UPstream::masterNo()
+ PstreamGlobals::MPICommunicators_[communicator]
+ );
+
+ profilingPstream::addScatterTime();
+}
- Value = bop(Value, value);
- }
- }
- else
- {
- if
- (
- MPI_Send
- (
- &Value,
- MPICount,
- MPIType,
- UPstream::masterNo(),
- tag,
- PstreamGlobals::MPICommunicators_[communicator]
- )
- )
- {
- FatalErrorInFunction
- << "MPI_Send failed"
- << Foam::abort(FatalError);
- }
- }
+template<class Type>
+void Foam::PstreamDetail::allReduce
+(
+ Type* values,
+ int count,
+ MPI_Datatype datatype,
+ MPI_Op optype,
+ const int tag,
+ const label communicator
+)
+{
+ if (!UPstream::parRun())
+ {
+ return;
+ }
- if (UPstream::master(communicator))
+ if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
+ {
+ Pout<< "** reducing:";
+ if (count == 1)
{
- for (const int proci : UPstream::subProcs(communicator))
- {
- if
- (
- MPI_Send
- (
- &Value,
- MPICount,
- MPIType,
- proci,
- tag,
- PstreamGlobals::MPICommunicators_[communicator]
- )
- )
- {
- FatalErrorInFunction
- << "MPI_Send failed"
- << Foam::abort(FatalError);
- }
- }
+ Pout<< (*values);
}
else
{
- if
- (
- MPI_Recv
- (
- &Value,
- MPICount,
- MPIType,
- UPstream::masterNo(),
- tag,
- PstreamGlobals::MPICommunicators_[communicator],
- MPI_STATUS_IGNORE
- )
- )
- {
- FatalErrorInFunction
- << "MPI_Recv failed"
- << Foam::abort(FatalError);
- }
+ Pout<< UList<Type>(values, count);
}
+ Pout<< " with comm:" << communicator
+ << " warnComm:" << UPstream::warnComm << endl;
+ error::printStack(Pout);
}
- else
- {
- Type sum;
- MPI_Allreduce
- (
- &Value,
- &sum,
- MPICount,
- MPIType,
- MPIOp,
- PstreamGlobals::MPICommunicators_[communicator]
- );
- Value = sum;
- }
+
+ profilingPstream::beginTiming();
+
+ // const int retval =
+ MPI_Allreduce
+ (
+ MPI_IN_PLACE,
+ values,
+ count,
+ datatype,
+ optype,
+ PstreamGlobals::MPICommunicators_[communicator]
+ );
profilingPstream::addReduceTime();
}
template<class Type>
-void Foam::iallReduce
+void Foam::PstreamDetail::iallReduce
(
- void* recvBuf,
- int MPICount,
- MPI_Datatype MPIType,
- MPI_Op MPIOp,
+ Type* values,
+ int count,
+ MPI_Datatype datatype,
+ MPI_Op optype,
const label communicator,
label& requestID
)
@@ -184,9 +129,16 @@ void Foam::iallReduce
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
- Pout<< "** non-blocking reducing:"
- << UList<Type>(static_cast<Type*>(recvBuf), MPICount)
- << " with comm:" << communicator
+ Pout<< "** non-blocking reducing:";
+ if (count == 1)
+ {
+ Pout<< (*values);
+ }
+ else
+ {
+ Pout<< UList<Type>(values, count);
+ }
+ Pout<< " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm << endl;
error::printStack(Pout);
}
@@ -200,10 +152,10 @@ void Foam::iallReduce
MPI_Iallreduce
(
MPI_IN_PLACE,
- recvBuf,
- MPICount,
- MPIType,
- MPIOp,
+ values,
+ count,
+ datatype,
+ optype,
PstreamGlobals::MPICommunicators_[communicator],
&request
)
@@ -211,7 +163,7 @@ void Foam::iallReduce
{
FatalErrorInFunction
<< "MPI_Iallreduce failed for "
- << UList<Type>(static_cast<Type*>(recvBuf), MPICount)
+ << UList<Type>(values, count)
<< Foam::abort(FatalError);
}
@@ -238,17 +190,17 @@ void Foam::iallReduce
MPI_Allreduce
(
MPI_IN_PLACE,
- recvBuf,
- MPICount,
- MPIType,
- MPIOp,
+ values,
+ count,
+ datatype,
+ optype,
PstreamGlobals::MPICommunicators_[communicator]
)
)
{
FatalErrorInFunction
<< "MPI_Allreduce failed for "
- << UList<Type>(static_cast<Type*>(recvBuf), MPICount)
+ << UList<Type>(values, count)
<< Foam::abort(FatalError);
}
requestID = -1;
diff --git a/src/fvOptions/sources/derived/meanVelocityForce/patchMeanVelocityForce/patchMeanVelocityForce.C b/src/fvOptions/sources/derived/meanVelocityForce/patchMeanVelocityForce/patchMeanVelocityForce.C
index 8684878b4b4..d2095839c29 100644
--- a/src/fvOptions/sources/derived/meanVelocityForce/patchMeanVelocityForce/patchMeanVelocityForce.C
+++ b/src/fvOptions/sources/derived/meanVelocityForce/patchMeanVelocityForce/patchMeanVelocityForce.C
@@ -6,6 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2015-2016 OpenFOAM Foundation
+ Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -71,15 +72,15 @@ Foam::scalar Foam::fv::patchMeanVelocityForce::magUbarAve
const volVectorField& U
) const
{
- vector2D sumAmagUsumA
- (
+ FixedList<scalar, 2> sumAmagUsumA(Zero);
+
+ sumAmagUsumA[0] +=
sum
(
(flowDir_ & U.boundaryField()[patchi_])
- *mesh_.boundary()[patchi_].magSf()
- ),
- sum(mesh_.boundary()[patchi_].magSf())
- );
+ * mesh_.boundary()[patchi_].magSf()
+ );
+ sumAmagUsumA[1] += sum(mesh_.boundary()[patchi_].magSf());
// If the mean velocity force is applied to a cyclic patch
@@ -89,29 +90,29 @@ Foam::scalar Foam::fv::patchMeanVelocityForce::magUbarAve
if (Pstream::parRun() && isA<cyclicPolyPatch>(patches[patchi_]))
{
- labelList processorCyclicPatches
+ for
(
- processorCyclicPolyPatch::patchIDs(patch_, patches)
- );
-
- forAll(processorCyclicPatches, pcpi)
+ const label patchi
+ : processorCyclicPolyPatch::patchIDs(patch_, patches)
+ )
{
- const label patchi = processorCyclicPatches[pcpi];
-
- sumAmagUsumA.x() +=
+ sumAmagUsumA[0] +=
sum
(
(flowDir_ & U.boundaryField()[patchi])
- *mesh_.boundary()[patchi].magSf()
+ * mesh_.boundary()[patchi].magSf()
);
-
- sumAmagUsumA.y() += sum(mesh_.boundary()[patchi].magSf());
+ sumAmagUsumA[1] += sum(mesh_.boundary()[patchi].magSf());
}
}
- mesh_.reduce(sumAmagUsumA, sumOp<vector2D>());
+ mesh_.reduce(sumAmagUsumA, sumOp<scalar>());
- return sumAmagUsumA.x()/sumAmagUsumA.y();
+ return
+ (
+ sumAmagUsumA[0]
+ / stabilise(sumAmagUsumA[1], VSMALL)
+ );
}
diff --git a/src/lumpedPointMotion/state/lumpedPointState.C b/src/lumpedPointMotion/state/lumpedPointState.C
index 5dc1c5e4344..f84f39e0b6a 100644
--- a/src/lumpedPointMotion/state/lumpedPointState.C
+++ b/src/lumpedPointMotion/state/lumpedPointState.C
@@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
- Copyright (C) 2016-2020 OpenCFD Ltd.
+ Copyright (C) 2016-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -405,14 +405,8 @@ bool Foam::lumpedPointState::readData
{
// Scatter master data using communication scheme
- const List<Pstream::commsStruct>& comms =
- (
- (Pstream::nProcs() < Pstream::nProcsSimpleSum)
- ? Pstream::linearCommunication()
- : Pstream::treeCommunication()
- );
-
// Get my communication order
+ const List<Pstream::commsStruct>& comms = Pstream::whichCommunication();
const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
// Receive from up
--
GitLab