diff --git a/applications/test/gather-scatter1/Make/files b/applications/test/gather-scatter1/Make/files new file mode 100644 index 0000000000000000000000000000000000000000..90ddd25b741cc9b72eee2a36d0e4d258e9894928 --- /dev/null +++ b/applications/test/gather-scatter1/Make/files @@ -0,0 +1,3 @@ +Test-gather-scatter1.cxx + +EXE = $(FOAM_USER_APPBIN)/Test-gather-scatter1 diff --git a/applications/test/gather-scatter1/Make/options b/applications/test/gather-scatter1/Make/options new file mode 100644 index 0000000000000000000000000000000000000000..18e6fe47afacb902cddccf82632772447704fd88 --- /dev/null +++ b/applications/test/gather-scatter1/Make/options @@ -0,0 +1,2 @@ +/* EXE_INC = */ +/* EXE_LIBS = */ diff --git a/applications/test/gather-scatter1/Test-gather-scatter1.cxx b/applications/test/gather-scatter1/Test-gather-scatter1.cxx new file mode 100644 index 0000000000000000000000000000000000000000..bcee94387b7f1785a466604933af9cef3d076934 --- /dev/null +++ b/applications/test/gather-scatter1/Test-gather-scatter1.cxx @@ -0,0 +1,282 @@ +/*---------------------------------------------------------------------------*\ + ========= | + \\ / F ield | OpenFOAM: The Open Source CFD Toolbox + \\ / O peration | + \\ / A nd | www.openfoam.com + \\/ M anipulation | +------------------------------------------------------------------------------- + Copyright (C) 2025 OpenCFD Ltd. +------------------------------------------------------------------------------- +License + This file is part of OpenFOAM. + + OpenFOAM is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + OpenFOAM is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + for more details. + + You should have received a copy of the GNU General Public License + along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>. + +Application + Test-gather-scatter1 + +Description + Simple tests for gather/scatter + +\*---------------------------------------------------------------------------*/ + +#include "List.H" +#include "argList.H" +#include "Time.H" +#include "Pstream.H" +#include "IOstreams.H" + +using namespace Foam; + + +//- Ostensibly the inverse of gatherList, but actually works like +//- a broadcast that skips overwriting the local rank! +template<class T> +void real_scatterList +( + //! [in,out] + UList<T>& values, + [[maybe_unused]] const int tag = UPstream::msgType(), + const int communicator = UPstream::worldComm +) +{ + if (!UPstream::is_parallel(communicator)) + { + // Nothing to do + return; + } + else if constexpr (is_contiguous_v<T>) + { + // This part is a real in-place scatter: + + // In-place scatter for contiguous types - one element per rank + // - on master: + // * send pointer is the full list + // * recv pointer is first destination + // - on rank: + // * send pointer is irrelevant + // * recv pointer is destination in the list + // + // So can simply use identical pointers for send/recv + + auto* ptr = values.data() + UPstream::myProcNo(communicator); + UPstream::mpiScatter(ptr, ptr, 1, communicator); + } + else + { + // Communication order + const auto& commOrder = UPstream::whichCommunication(communicator); + + Pstream::scatterList_algorithm(commOrder, values, tag, communicator); + } +} + + +//- gatherList_algorithm, but with specific communication style +template<class T> +void gatherList_algo +( + const bool linear, + //! [in,out] + UList<T>& values, + [[maybe_unused]] const int tag = UPstream::msgType(), + const int communicator = UPstream::worldComm +) +{ + if (UPstream::is_parallel(communicator)) + { + Pstream::gatherList_algorithm + ( + UPstream::whichCommunication(communicator, linear), + values, + tag, + communicator + ); + } +} + + +//- scatterList_algorithm, but with specific communication style +template<class T> +void scatterList_algo +( + const bool linear, + //! [in,out] + UList<T>& values, + [[maybe_unused]] const int tag = UPstream::msgType(), + const int communicator = UPstream::worldComm +) +{ + if (UPstream::is_parallel(communicator)) + { + Pstream::scatterList_algorithm + ( + UPstream::whichCommunication(communicator, linear), + values, + tag, + communicator + ); + } +} + + +// Perform tests +template<class ListType, class ResetCode> +void doTest(ResetCode reset) +{ + ListType values; + + reset(values); + + Pout<< nl << "before:" << flatOutput(values) << endl; + Pstream::broadcastList(values); + Pout<< "broadcast:" << flatOutput(values) << endl; + + reset(values); + + Pout<< nl << "before:" << flatOutput(values) << endl; + Pstream::scatterList(values); + Pout<< "scatter:" << flatOutput(values) << endl; + + reset(values); + + Pout<< "before:" << flatOutput(values) << endl; + real_scatterList(values); + Pout<< "inplace:" << flatOutput(values) << endl; + + using control = std::pair<int, int>; + + const char* algoType[2] = { "tree", "linear" }; + + for + ( + auto [gather, scatter] : + { + control{0, 0}, + control{1, 1}, + control{0, 1}, + control{1, 0} + } + ) + { + reset(values); + + Pout<< nl << "before:" << flatOutput(values) << endl; + + gatherList_algo(gather, values); + Pout<< "gather[" << algoType[gather] << "]:" + << flatOutput(values) << endl; + + scatterList_algo(scatter, values); + Pout<< "scatter[" << algoType[scatter] << "]:" + << flatOutput(values) << endl; + } +} + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +int main(int argc, char *argv[]) +{ + argList::noCheckProcessorDirectories(); + argList::addVerboseOption("increase UPstream::debug level"); + + #include "setRootCase.H" + + const int optVerbose = args.verbose(); + + if (optVerbose) + { + UPstream::debug = optVerbose; + } + + Pout<< nl << "Test contiguous" << endl; + { + doTest<labelList> + ( + [](auto& values){ + if (UPstream::master()) + { + values = identity(UPstream::nProcs()); + } + else + { + values.resize(UPstream::nProcs()); + values = -1; + values[UPstream::myProcNo()] = 10 * UPstream::myProcNo(); + } + } + ); + } + + Pout<< nl << "Test non-contiguous" << endl; + { + doTest<wordList> + ( + [](auto& values) { + values.resize(UPstream::nProcs()); + if (UPstream::master()) + { + forAll(values, i) + { + values[i] = "proc" + Foam::name(i); + } + } + else + { + values = "none"; + values[UPstream::myProcNo()] = + "_" + Foam::name(UPstream::myProcNo()); + } + } + ); + } + + // Test dummy broadcast as well + Pout<< nl << "Test broadcastList" << endl; + { + wordList list; + + Pout<< nl << "before: " << flatOutput(list) << endl; + + Pstream::broadcastList(list); + Pout<< "-> " << flatOutput(list) << endl; + } + + // Test in-place reduce + Pout<< nl << "Test in-place reduce" << endl; + { + FixedList<label, 6> list; + list = UPstream::myProcNo(); + + Pout<< nl << "before: " << flatOutput(list) << endl; + + UPstream::mpiReduce + ( + list.data(), + list.size(), + UPstream::opCodes::op_sum, + UPstream::worldComm + ); + + Pout<< "-> " << flatOutput(list) << endl; + } + + Info<< nl << "End\n" << endl; + + return 0; +} + + +// ************************************************************************* // diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H index 91329be13da8104fe0ddb8e8edcecfabe4f84834..70294111f3c130afade1807564a44c994be23ced 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H @@ -456,19 +456,6 @@ public: ); - // Scatter - - //- Inverse of gatherList. - template<class T> - static void scatterList - ( - //! [in,out] - UList<T>& values, - const int tag = UPstream::msgType(), - const int communicator = UPstream::worldComm - ); - - // Exchange //- Helper: exchange sizes of sendBufs for specified send/recv ranks @@ -700,6 +687,31 @@ public: { Pstream::broadcast(values, comm); } + + //- The inverse of gatherList, but when combined with gatherList + //- it effectively acts like a partial broadcast... + // \deprecated(2025-03) Normally prefer broadcast + template<class T> + FOAM_DEPRECATED_FOR(2025-03, "broadcast() or broadcastList()") + static void scatterList + ( + //! [in,out] + UList<T>& values, + const int tag = UPstream::msgType(), + const int communicator = UPstream::worldComm + ) + { + if (UPstream::is_parallel(communicator)) + { + Pstream::scatterList_algorithm + ( + UPstream::whichCommunication(communicator), + values, + tag, + communicator + ); + } + } }; diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamGatherList.txx b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamGatherList.txx index 5a3de56ad5db344fba09a4a6a691cfa3a9c16294..d850fe348e9e06356e743ae9e66da6ae2a08b43b 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamGatherList.txx +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamGatherList.txx @@ -109,7 +109,8 @@ void Foam::Pstream::gatherList_algorithm ( UPstream::commsTypes::scheduled, belowID, - values[belowID], + &(values[belowID]), + 1, tag, communicator ); @@ -194,7 +195,8 @@ void Foam::Pstream::gatherList_algorithm ( UPstream::commsTypes::scheduled, myComm.above(), - values[myProci], + &(values[myProci]), + 1, tag, communicator ); @@ -434,7 +436,17 @@ void Foam::Pstream::gatherList } // In-place gather for contiguous types - one element per rank - UPstream::mpiGather(nullptr, values.data(), 1, communicator); + // - all ranks: + // * send pointer is source location from within the list + // - on master: + // * recv pointer is the full list (same address as first location) + // - on rank: + // * recv pointer is irrelevant + // + // So can simply use identical pointers for send/recv + + auto* ptr = values.data() + UPstream::myProcNo(communicator); + UPstream::mpiGather(ptr, ptr, 1, communicator); } else { @@ -446,34 +458,6 @@ void Foam::Pstream::gatherList } -template<class T> -void Foam::Pstream::scatterList -( - UList<T>& values, - [[maybe_unused]] const int tag, - const int communicator -) -{ - if (!UPstream::is_parallel(communicator)) - { - // Nothing to do - return; - } - else if constexpr (is_contiguous_v<T>) - { - // In-place scatter for contiguous types - one element per rank - UPstream::mpiScatter(nullptr, values.data(), 1, communicator); - } - else - { - // Communication order - const auto& commOrder = UPstream::whichCommunication(communicator); - - Pstream::scatterList_algorithm(commOrder, values, tag, communicator); - } -} - - template<class T> void Foam::Pstream::allGatherList ( diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index 8caf255853fd814852436a50bdaf16d48312fd24..1ac2249f3cdad33436037dab35495f95152ad174 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -170,9 +170,10 @@ public: { broadcast = 1, //!< broadcast [MPI] reduce = 2, //!< reduce/all-reduce [MPI] - gather = 16, //!< gather (reduction) [manual algorithm] + gather = 4, //!< gather/all-gather [MPI] + combine = 16, //!< combine/gather (reduction) [manual algorithm] mapGather = 32, //!< mapGather (reduction) [manual algorithm] - gatherList = 64, //!< gatherList/scatterList [manual algorithm] + gatherList = 64, //!< gatherList [manual algorithm] }; @@ -600,16 +601,17 @@ protected: UPstream::Request* req = nullptr ); - //- Receive identically-sized (contiguous) data from all ranks + //- Receive identically-sized (contiguous) data from all ranks, + //- placing the result on rank 0. // Includes internal parallel guard. // For non-parallel, does not copy any data. // If needed, this must be done by the caller. static void mpi_gather ( - //! On rank: individual value to send (or nullptr for inplace) + //! All ranks: location of individual value to send const void* sendData, - //! Master: receive buffer with all values - //! Or for in-place send/recv when sendData is nullptr + //! Master: receive buffer with all values. + //! Other ranks: ignored void* recvData, //! Number of send/recv data per rank. Globally consistent! int count, @@ -619,14 +621,15 @@ protected: UPstream::Request* req = nullptr ); - //- Send identically-sized (contiguous) data to all ranks + //- Send identically-sized (contiguous) data from rank 0 + //- to all other ranks. // Includes internal parallel guard. static void mpi_scatter ( - //! On master: send buffer with all values (nullptr for inplace) + //! Master: send buffer with all values. + //! Other ranks: ignored const void* sendData, - //! On rank: individual value to receive - //! Or for in-place send/recv when sendData is nullptr + //! All ranks: location to receive individual value void* recvData, //! Number of send/recv data per rank. Globally consistent! int count, @@ -640,7 +643,7 @@ protected: // Send data from proc slot, receive into all slots static void mpi_allgather ( - //! On all ranks: the base of the data locations + //! All ranks: the base of the data locations void* allData, //! Number of send/recv data per rank. Globally consistent! int count, @@ -650,7 +653,8 @@ protected: UPstream::Request* req = nullptr ); - //- Receive variable length data from all ranks. + //- Receive variable length data from all ranks, + //- placing the result on rank 0. //- (caution: known to scale poorly) static void mpi_gatherv ( @@ -663,7 +667,7 @@ protected: const int communicator ); - //- Send variable length data to all ranks + //- Send variable length data from rank 0 to all ranks. //- (caution: known to scale poorly) static void mpi_scatterv ( @@ -1583,10 +1587,10 @@ public: template<class Type> static void mpiGather ( - //! On rank: individual value to send (or nullptr for inplace) + //! All ranks: location of individual value to send const Type* sendData, - //! Master: receive buffer with all values - //! Or for in-place send/recv when sendData is nullptr + //! Master: receive buffer with all values. + //! Other ranks: ignored Type* recvData, //! Number of send/recv data per rank. Globally consistent! int count, @@ -1597,10 +1601,10 @@ public: template<class Type> static void mpiScatter ( - //! On master: send buffer with all values (nullptr for inplace) + //! Master: send buffer with all values. + //! Other ranks: ignored const Type* sendData, - //! On rank: individual value to receive - //! Or for in-place send/recv when sendData is nullptr + //! All ranks: location to receive individual value Type* recvData, //! Number of send/recv data per rank. Globally consistent! int count, @@ -1612,7 +1616,7 @@ public: template<class Type> static void mpiAllGather ( - //! On all ranks: the base of the data locations + //! All ranks: the base of the data locations Type* allData, //! Number of send/recv data per rank. Globally consistent! int count, diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.txx b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.txx index 1e529ac018e5444e5c3d8977c373f38dfde88a1a..35ab5448b848e2b0e57f63226b906f1638e8dff1 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.txx +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.txx @@ -335,7 +335,7 @@ void Foam::UPstream::mpiGatherv { if constexpr (is_contiguous_v<Type>) { - if (sendData && recvData) + if (sendData && recvData && (sendData != recvData)) { // recvCounts[0] may be invalid - use sendCount instead std::memmove(recvData, sendData, sendCount*sizeof(Type)); @@ -385,7 +385,7 @@ void Foam::UPstream::mpiScatterv { if constexpr (is_contiguous_v<Type>) { - if (sendData && recvData) + if (sendData && recvData && (sendData != recvData)) { std::memmove(recvData, sendData, recvCount*sizeof(Type)); } diff --git a/src/Pstream/mpi/UPstreamBroadcast.C b/src/Pstream/mpi/UPstreamBroadcast.C index 91f4b1abfc66f671379f7f4c81019e2fd027b4f7..5a916441b15dd340ea709f217e18f38e89575c5a 100644 --- a/src/Pstream/mpi/UPstreamBroadcast.C +++ b/src/Pstream/mpi/UPstreamBroadcast.C @@ -74,7 +74,7 @@ bool Foam::UPstream::mpi_broadcast { // Regular broadcast - // OR: PstreamDetail::broadcast0(buf, count, datatype, communicator); + // OR: PstreamDetail::broadcast(buf, count, datatype, communicator); returnCode = MPI_Bcast ( diff --git a/src/Pstream/mpi/UPstreamGatherScatter.C b/src/Pstream/mpi/UPstreamGatherScatter.C index 561a1d41c982ef631a16c06bbb83e78a4187570e..ec82a807219caed311ebf84a7d626c2d52663e5c 100644 --- a/src/Pstream/mpi/UPstreamGatherScatter.C +++ b/src/Pstream/mpi/UPstreamGatherScatter.C @@ -78,8 +78,19 @@ void Foam::UPstream::mpi_gather if (FOAM_UNLIKELY(UPstream::debug)) { - Perr<< "[mpi_gather] : " - << " type:" << int(dataTypeId) << " count:" << count + Perr<< "[mpi_gather] :"; + + // Appears to be an in-place request + if + ( + UPstream::master(communicator) + && (!sendData || (sendData == recvData)) + ) + { + Perr<< " (inplace)"; + } + + Perr<< " type:" << int(dataTypeId) << " count:" << count << " comm:" << communicator << Foam::endl; } @@ -117,8 +128,19 @@ void Foam::UPstream::mpi_scatter if (FOAM_UNLIKELY(UPstream::debug)) { - Perr<< "[mpi_scatter] : " - << " type:" << int(dataTypeId) << " count:" << count + Perr<< "[mpi_scatter] :"; + + // Appears to be an in-place request + if + ( + UPstream::master(communicator) + && (!recvData || (sendData == recvData)) + ) + { + Perr<< " (inplace)"; + } + + Perr<< " type:" << int(dataTypeId) << " count:" << count << " comm:" << communicator << Foam::endl; } @@ -155,7 +177,7 @@ void Foam::UPstream::mpi_allgather if (FOAM_UNLIKELY(UPstream::debug)) { - Perr<< "[mpi_allgather] : " + Perr<< "[mpi_allgather] :" << " type:" << int(dataTypeId) << " count:" << count << " comm:" << communicator << Foam::endl; diff --git a/src/Pstream/mpi/UPstreamReduce.C b/src/Pstream/mpi/UPstreamReduce.C index 8cdbceeeb5380b9d6cebb877039127f8b3bd553c..fc87380ba5313df8bb97660938d67c26bab07009 100644 --- a/src/Pstream/mpi/UPstreamReduce.C +++ b/src/Pstream/mpi/UPstreamReduce.C @@ -174,7 +174,7 @@ void Foam::UPstream::mpi_reduce if (FOAM_UNLIKELY(UPstream::debug)) { - Perr<< "[mpi_reduce] : " + Perr<< "[mpi_reduce] : (inplace)" << " op:" << int(opCodeId) << " type:" << int(dataTypeId) << " count:" << count << " comm:" << communicator @@ -214,7 +214,7 @@ void Foam::UPstream::mpi_reduce { // Regular reduce - PstreamDetail::reduce0 + PstreamDetail::reduce ( send_buffer, values, @@ -263,7 +263,7 @@ void Foam::UPstream::mpi_allreduce if (FOAM_UNLIKELY(UPstream::debug)) { - Perr<< "[mpi_allreduce] : " + Perr<< "[mpi_allreduce] :" << " op:" << int(opCodeId) << " type:" << int(dataTypeId) << " count:" << count << " comm:" << communicator diff --git a/src/Pstream/mpi/UPstreamWrapping.H b/src/Pstream/mpi/UPstreamWrapping.H index 2587d840dc2a9de4369b622114e7e3a798fef461..d7d7a7f3a7f445bc7b57e44ba33076e51d233b72 100644 --- a/src/Pstream/mpi/UPstreamWrapping.H +++ b/src/Pstream/mpi/UPstreamWrapping.H @@ -50,7 +50,7 @@ namespace PstreamDetail // MPI_Bcast, using root=0 // No fail/abort handling template<class Type> -bool broadcast0 +bool broadcast ( Type* values, int count, @@ -60,7 +60,7 @@ bool broadcast0 // MPI_Reduce, using root=0 template<class Type> -void reduce0 +void reduce ( const Type* sendData, // Use nullptr for in-place operation Type* values, diff --git a/src/Pstream/mpi/UPstreamWrapping.txx b/src/Pstream/mpi/UPstreamWrapping.txx index 9d15f1b8e12f9745ca73e63367b96450fb87e101..2129de6b0e824cc53ad38649c8ee09b686fa9c03 100644 --- a/src/Pstream/mpi/UPstreamWrapping.txx +++ b/src/Pstream/mpi/UPstreamWrapping.txx @@ -35,7 +35,7 @@ License // * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * // template<class Type> -bool Foam::PstreamDetail::broadcast0 +bool Foam::PstreamDetail::broadcast ( Type* values, int count, @@ -72,7 +72,7 @@ bool Foam::PstreamDetail::broadcast0 template<class Type> -void Foam::PstreamDetail::reduce0 +void Foam::PstreamDetail::reduce ( const Type* sendData, Type* values, @@ -94,9 +94,14 @@ void Foam::PstreamDetail::reduce0 } const void* send_buffer = sendData; - if (sendData == nullptr || (sendData == values)) + if + ( + UPstream::master(communicator) + && (!sendData || (sendData == values)) + ) { - // Appears to be an in-place request + // Appears to be an in-place request. + // - this setting only relevant (or usable) on the root rank send_buffer = MPI_IN_PLACE; } @@ -110,7 +115,7 @@ void Foam::PstreamDetail::reduce0 { Perr<< "** MPI_Reduce (blocking):"; } - if (sendData == nullptr || (sendData == values)) + if (UPstream::master(communicator) && (send_buffer == MPI_IN_PLACE)) { Perr<< " [inplace]"; } @@ -976,13 +981,25 @@ void Foam::PstreamDetail::gather // Cannot copy data here since we don't know the number of bytes // - must be done by the caller. } - else if (sendData && recvData) + else if (sendData && recvData && (sendData != recvData)) { std::memmove(recvData, sendData, count*sizeof(Type)); } return; } + const void* send_buffer = sendData; + if + ( + UPstream::master(communicator) + && (!sendData || (sendData == recvData)) + ) + { + // Appears to be an in-place request. + // - this setting only relevant (or usable) on the root rank + send_buffer = MPI_IN_PLACE; + } + if (FOAM_UNLIKELY(PstreamGlobals::warnCommunicator(communicator))) { if (immediate) @@ -993,7 +1010,7 @@ void Foam::PstreamDetail::gather { Perr<< "** MPI_Gather (blocking):"; } - if (sendData == nullptr || (sendData == recvData)) + if (UPstream::master(communicator) && (send_buffer == MPI_IN_PLACE)) { Perr<< " [inplace]"; } @@ -1005,14 +1022,6 @@ void Foam::PstreamDetail::gather error::printStack(Perr); } - const void* send_buffer = sendData; - if (sendData == nullptr || (sendData == recvData)) - { - // Appears to be an in-place request - send_buffer = MPI_IN_PLACE; - } - - int returnCode(MPI_ERR_UNKNOWN); #if defined(MPI_VERSION) && (MPI_VERSION >= 3) @@ -1094,13 +1103,25 @@ void Foam::PstreamDetail::scatter // Cannot copy data here since we don't know the number of bytes // - must be done by the caller. } - else if (sendData && recvData) + else if (sendData && recvData && (sendData != recvData)) { std::memmove(recvData, sendData, count*sizeof(Type)); } return; } + void* recv_buffer = recvData; + + if + ( + UPstream::master(communicator) + && (!recvData || (sendData == recvData))) + { + // Appears to be an in-place request. + // - this setting only relevant (or usable) on the root rank + recv_buffer = MPI_IN_PLACE; + } + if (FOAM_UNLIKELY(PstreamGlobals::warnCommunicator(communicator))) { if (immediate) @@ -1111,7 +1132,7 @@ void Foam::PstreamDetail::scatter { Perr<< "** MPI_Scatter (blocking):"; } - if (sendData == nullptr || (sendData == recvData)) + if (UPstream::master(communicator) && (recv_buffer == MPI_IN_PLACE)) { Perr<< " [inplace]"; } @@ -1122,16 +1143,6 @@ void Foam::PstreamDetail::scatter << endl; error::printStack(Perr); } - - - const void* send_buffer = sendData; - if (sendData == nullptr || (sendData == recvData)) - { - // Appears to be an in-place request - send_buffer = MPI_IN_PLACE; - } - - int returnCode(MPI_ERR_UNKNOWN); #if defined(MPI_VERSION) && (MPI_VERSION >= 3) @@ -1144,8 +1155,8 @@ void Foam::PstreamDetail::scatter returnCode = MPI_Iscatter ( - send_buffer, count, datatype, - recvData, count, datatype, + sendData, count, datatype, + recv_buffer, count, datatype, 0, // root: UPstream::masterNo() PstreamGlobals::MPICommunicators_[communicator], &request @@ -1162,8 +1173,8 @@ void Foam::PstreamDetail::scatter returnCode = MPI_Scatter ( - send_buffer, count, datatype, - recvData, count, datatype, + sendData, count, datatype, + recv_buffer, count, datatype, 0, // root: UPstream::masterNo() PstreamGlobals::MPICommunicators_[communicator] );