Commit 9e463b0d authored by Mark Olesen's avatar Mark Olesen
Browse files

ENH: add mpiBufferSize optimisationSwitch (issue #517)

- allows configuration without an environment variable.
  For compatibility still respect MPI_BUFFER_SIZE env-variable.
parent ab8715a0
......@@ -51,7 +51,7 @@ void testMapDistribute()
List<Tuple2<label, List<scalar>>> complexData(100);
forAll(complexData, i)
{
complexData[i].first() = rndGen.integer(0, Pstream::nProcs()-1);
complexData[i].first() = rndGen.position(0, Pstream::nProcs()-1);
complexData[i].second().setSize(3);
complexData[i].second()[0] = 1;
complexData[i].second()[1] = 2;
......
......@@ -257,19 +257,7 @@ if ( "$FOAM_MPI" != dummy ) then
_foamAddLib ${FOAM_LIBBIN}/${FOAM_MPI}
endif
# Minimum MPI buffer size (used by all except SGI MPI)
if ( ! $?minBufferSize ) set minBufferSize=20000000
if ( $?MPI_BUFFER_SIZE ) then
if ( $MPI_BUFFER_SIZE < $minBufferSize ) then
setenv MPI_BUFFER_SIZE $minBufferSize
endif
else
setenv MPI_BUFFER_SIZE $minBufferSize
endif
# Cleanup environment
# ~~~~~~~~~~~~~~~~~~~
unset minBufferSize
# Alternative to 'mpiBufferSize' controlDict entry
#setenv MPI_BUFFER_SIZE 20000000
#------------------------------------------------------------------------------
......@@ -274,18 +274,7 @@ then
_foamAddLib $FOAM_LIBBIN/$FOAM_MPI
fi
# Minimum MPI buffer size (used by all except SGI MPI)
: ${minBufferSize:=20000000}
if [ "${MPI_BUFFER_SIZE:=$minBufferSize}" -lt $minBufferSize ]
then
MPI_BUFFER_SIZE=$minBufferSize
fi
export MPI_BUFFER_SIZE
# Cleanup environment
# ~~~~~~~~~~~~~~~~~~~
unset minBufferSize
# Alternative to 'mpiBufferSize' controlDict entry
#export MPI_BUFFER_SIZE=20000000
#------------------------------------------------------------------------------
......@@ -65,6 +65,11 @@ OptimisationSwitches
floatTransfer 0;
nProcsSimpleSum 0;
// MPI buffer size (bytes)
// Can override with the MPI_BUFFER_SIZE env variable.
// The default and minimum is (20000000).
mpiBufferSize 0;
// Optional max size (bytes) for unstructured data exchanges. In some
// phases of OpenFOAM it can send over very large data chunks
// (e.g. in parallel load balancing) and some Pstream implementations have
......
......@@ -475,4 +475,10 @@ registerOptSwitch
);
const int Foam::UPstream::mpiBufferSize
(
Foam::debug::optimisationSwitch("mpiBufferSize", 0)
);
// ************************************************************************* //
......@@ -272,6 +272,9 @@ public:
//- Optional maximum message size (bytes)
static int maxCommsSize;
//- MPI buffer-size (bytes)
static const int mpiBufferSize;
//- Default communicator (all processors)
static label worldComm;
......
......@@ -29,6 +29,7 @@ License
#include "PstreamGlobals.H"
#include "SubList.H"
#include "allReduce.H"
#include "int.H"
#include <mpi.h>
......@@ -42,6 +43,12 @@ License
#define MPI_SCALAR MPI_DOUBLE
#endif
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
// file-scope: min value and default for mpiBufferSize
static const int minBufferSize = 20000000;
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// NOTE:
......@@ -87,33 +94,32 @@ bool Foam::UPstream::init(int& argc, char**& argv)
setParRun(numprocs);
#ifndef SGIMPI
string bufferSizeName = getEnv("MPI_BUFFER_SIZE");
if (bufferSizeName.size())
{
int bufferSize = atoi(bufferSizeName.c_str());
// Normally use UPstream::mpiBufferSize (optimisationSwitch),
// but allow override with the MPI_BUFFER_SIZE env variable
// which has an int value
int bufSize = 0;
if (bufferSize)
const std::string str = Foam::getEnv("MPI_BUFFER_SIZE");
if (str.empty() || !Foam::read(str.c_str(), bufSize) || bufSize <= 0)
{
MPI_Buffer_attach(new char[bufferSize], bufferSize);
bufSize = mpiBufferSize;
}
}
else
{
FatalErrorInFunction
<< "UPstream::init(int& argc, char**& argv) : "
<< "environment variable MPI_BUFFER_SIZE not defined"
<< Foam::abort(FatalError);
if (bufSize < minBufferSize)
{
bufSize = minBufferSize;
}
if (debug)
{
Pout<< "UPstream::init : mpi-buffer-size " << bufSize << endl;
}
MPI_Buffer_attach(new char[bufSize], bufSize);
}
#endif
//int processorNameLen;
//char processorName[MPI_MAX_PROCESSOR_NAME];
//
//MPI_Get_processor_name(processorName, &processorNameLen);
//processorName[processorNameLen] = '\0';
//Pout<< "Processor name:" << processorName << endl;
return true;
}
......@@ -126,10 +132,12 @@ void Foam::UPstream::exit(int errnum)
}
#ifndef SGIMPI
int size;
char* buff;
MPI_Buffer_detach(&buff, &size);
delete[] buff;
{
int size;
char* buff;
MPI_Buffer_detach(&buff, &size);
delete[] buff;
}
#endif
if (PstreamGlobals::outstandingRequests_.size())
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment