diff --git a/applications/test/parallel/Test-parallel.C b/applications/test/parallel/Test-parallel.C index 7a61b77f72e169d3086b0e17ad68375bb9f7a46d..14ba97f7a561a39dd3ea5952b15553e04647f543 100644 --- a/applications/test/parallel/Test-parallel.C +++ b/applications/test/parallel/Test-parallel.C @@ -51,7 +51,7 @@ void testMapDistribute() List<Tuple2<label, List<scalar>>> complexData(100); forAll(complexData, i) { - complexData[i].first() = rndGen.integer(0, Pstream::nProcs()-1); + complexData[i].first() = rndGen.position(0, Pstream::nProcs()-1); complexData[i].second().setSize(3); complexData[i].second()[0] = 1; complexData[i].second()[1] = 2; diff --git a/etc/config.csh/mpi b/etc/config.csh/mpi index c4df2b627e79ec357c4d6a4f0c8b1c0abeb912f7..e1e7fba8e1ebea4079808fad70d8465a7f0cc297 100644 --- a/etc/config.csh/mpi +++ b/etc/config.csh/mpi @@ -257,19 +257,7 @@ if ( "$FOAM_MPI" != dummy ) then _foamAddLib ${FOAM_LIBBIN}/${FOAM_MPI} endif - -# Minimum MPI buffer size (used by all except SGI MPI) -if ( ! $?minBufferSize ) set minBufferSize=20000000 -if ( $?MPI_BUFFER_SIZE ) then - if ( $MPI_BUFFER_SIZE < $minBufferSize ) then - setenv MPI_BUFFER_SIZE $minBufferSize - endif -else - setenv MPI_BUFFER_SIZE $minBufferSize -endif - -# Cleanup environment -# ~~~~~~~~~~~~~~~~~~~ -unset minBufferSize +# Alternative to 'mpiBufferSize' controlDict entry +#setenv MPI_BUFFER_SIZE 20000000 #------------------------------------------------------------------------------ diff --git a/etc/config.sh/mpi b/etc/config.sh/mpi index 4c663ce276fd5a33312c171ab1d42f25e4083464..9d22c3e700660c76c185abe48da83cacfeb675f9 100644 --- a/etc/config.sh/mpi +++ b/etc/config.sh/mpi @@ -274,18 +274,7 @@ then _foamAddLib $FOAM_LIBBIN/$FOAM_MPI fi - -# Minimum MPI buffer size (used by all except SGI MPI) -: ${minBufferSize:=20000000} -if [ "${MPI_BUFFER_SIZE:=$minBufferSize}" -lt $minBufferSize ] -then - MPI_BUFFER_SIZE=$minBufferSize -fi -export MPI_BUFFER_SIZE - - -# Cleanup environment -# ~~~~~~~~~~~~~~~~~~~ -unset minBufferSize +# Alternative to 'mpiBufferSize' controlDict entry +#export MPI_BUFFER_SIZE=20000000 #------------------------------------------------------------------------------ diff --git a/etc/controlDict b/etc/controlDict index 72112b0ddcfd576deffabe02d66bfa39d49bdba1..c73b3875107a5b6f5846fbb4c86e00d952e46af5 100644 --- a/etc/controlDict +++ b/etc/controlDict @@ -65,6 +65,11 @@ OptimisationSwitches floatTransfer 0; nProcsSimpleSum 0; + // MPI buffer size (bytes) + // Can override with the MPI_BUFFER_SIZE env variable. + // The default and minimum is (20000000). + mpiBufferSize 0; + // Optional max size (bytes) for unstructured data exchanges. In some // phases of OpenFOAM it can send over very large data chunks // (e.g. in parallel load balancing) and some Pstream implementations have diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C index 7230b5c5973cc322d2b6dfa29514741f6af55f74..9bd97a134993e2b87430cc2e6e33a257974952eb 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C @@ -475,4 +475,10 @@ registerOptSwitch ); +const int Foam::UPstream::mpiBufferSize +( + Foam::debug::optimisationSwitch("mpiBufferSize", 0) +); + + // ************************************************************************* // diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index ba791916d0b43b050b49c1b3b852ba83b9b78e2e..7e71fc543c211a7d1db8a2305023233b33cf9eb5 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -272,6 +272,9 @@ public: //- Optional maximum message size (bytes) static int maxCommsSize; + //- MPI buffer-size (bytes) + static const int mpiBufferSize; + //- Default communicator (all processors) static label worldComm; diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C index 1f6ebca09ae3b9c6d10078187c2a819ec801b099..9eadaf48e23c0e5465e911962f4f2e807e5b80da 100644 --- a/src/Pstream/mpi/UPstream.C +++ b/src/Pstream/mpi/UPstream.C @@ -29,6 +29,7 @@ License #include "PstreamGlobals.H" #include "SubList.H" #include "allReduce.H" +#include "int.H" #include <mpi.h> @@ -42,6 +43,12 @@ License #define MPI_SCALAR MPI_DOUBLE #endif +// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // + +// file-scope: min value and default for mpiBufferSize +static const int minBufferSize = 20000000; + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // NOTE: @@ -87,33 +94,32 @@ bool Foam::UPstream::init(int& argc, char**& argv) setParRun(numprocs); #ifndef SGIMPI - string bufferSizeName = getEnv("MPI_BUFFER_SIZE"); - - if (bufferSizeName.size()) { - int bufferSize = atoi(bufferSizeName.c_str()); + // Normally use UPstream::mpiBufferSize (optimisationSwitch), + // but allow override with the MPI_BUFFER_SIZE env variable + // which has an int value + int bufSize = 0; - if (bufferSize) + const std::string str = Foam::getEnv("MPI_BUFFER_SIZE"); + if (str.empty() || !Foam::read(str.c_str(), bufSize) || bufSize <= 0) { - MPI_Buffer_attach(new char[bufferSize], bufferSize); + bufSize = mpiBufferSize; } - } - else - { - FatalErrorInFunction - << "UPstream::init(int& argc, char**& argv) : " - << "environment variable MPI_BUFFER_SIZE not defined" - << Foam::abort(FatalError); + + if (bufSize < minBufferSize) + { + bufSize = minBufferSize; + } + + if (debug) + { + Pout<< "UPstream::init : mpi-buffer-size " << bufSize << endl; + } + + MPI_Buffer_attach(new char[bufSize], bufSize); } #endif - //int processorNameLen; - //char processorName[MPI_MAX_PROCESSOR_NAME]; - // - //MPI_Get_processor_name(processorName, &processorNameLen); - //processorName[processorNameLen] = '\0'; - //Pout<< "Processor name:" << processorName << endl; - return true; } @@ -126,10 +132,12 @@ void Foam::UPstream::exit(int errnum) } #ifndef SGIMPI - int size; - char* buff; - MPI_Buffer_detach(&buff, &size); - delete[] buff; + { + int size; + char* buff; + MPI_Buffer_detach(&buff, &size); + delete[] buff; + } #endif if (PstreamGlobals::outstandingRequests_.size())