ENH: add mpiBufferSize optimisationSwitch (issue #517)

- allows configuration without an environment variable.
  For compatibility still respect MPI_BUFFER_SIZE env-variable.

applications/test/parallel/Test-parallel.C

@@ -51,7 +51,7 @@ void testMapDistribute()
     List<Tuple2<label, List<scalar>>> complexData(100);
     forAll(complexData, i)
     {
-        complexData[i].first() = rndGen.integer(0, Pstream::nProcs()-1);
+        complexData[i].first() = rndGen.position(0, Pstream::nProcs()-1);
         complexData[i].second().setSize(3);
         complexData[i].second()[0] = 1;
         complexData[i].second()[1] = 2;

etc/config.csh/mpi

@@ -257,19 +257,7 @@ if ( "$FOAM_MPI" != dummy ) then
     _foamAddLib ${FOAM_LIBBIN}/${FOAM_MPI}
 endif
 
-
-# Minimum MPI buffer size (used by all except SGI MPI)
-if ( ! $?minBufferSize ) set minBufferSize=20000000
-if ( $?MPI_BUFFER_SIZE ) then
-    if ( $MPI_BUFFER_SIZE < $minBufferSize ) then
-        setenv MPI_BUFFER_SIZE $minBufferSize
-    endif
-else
-    setenv MPI_BUFFER_SIZE $minBufferSize
-endif
-
-# Cleanup environment
-# ~~~~~~~~~~~~~~~~~~~
-unset minBufferSize
+# Alternative to 'mpiBufferSize' controlDict entry
+#setenv MPI_BUFFER_SIZE 20000000
 
 #------------------------------------------------------------------------------

etc/config.sh/mpi

@@ -274,18 +274,7 @@ then
     _foamAddLib $FOAM_LIBBIN/$FOAM_MPI
 fi
 
-
-# Minimum MPI buffer size (used by all except SGI MPI)
-: ${minBufferSize:=20000000}
-if [ "${MPI_BUFFER_SIZE:=$minBufferSize}" -lt $minBufferSize ]
-then
-    MPI_BUFFER_SIZE=$minBufferSize
-fi
-export MPI_BUFFER_SIZE
-
-
-# Cleanup environment
-# ~~~~~~~~~~~~~~~~~~~
-unset minBufferSize
+# Alternative to 'mpiBufferSize' controlDict entry
+#export MPI_BUFFER_SIZE=20000000
 
 #------------------------------------------------------------------------------

etc/controlDict

@@ -65,6 +65,11 @@ OptimisationSwitches
     floatTransfer   0;
     nProcsSimpleSum 0;
 
+    // MPI buffer size (bytes)
+    // Can override with the MPI_BUFFER_SIZE env variable.
+    // The default and minimum is (20000000).
+    mpiBufferSize   0;
+
     // Optional max size (bytes) for unstructured data exchanges. In some
     // phases of OpenFOAM it can send over very large data chunks
     // (e.g. in parallel load balancing) and some Pstream implementations have

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C

@@ -475,4 +475,10 @@ registerOptSwitch
 );
 
 
+const int Foam::UPstream::mpiBufferSize
+(
+    Foam::debug::optimisationSwitch("mpiBufferSize", 0)
+);
+
+
 // ************************************************************************* //

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H

@@ -272,6 +272,9 @@ public:
         //- Optional maximum message size (bytes)
         static int maxCommsSize;
 
+        //- MPI buffer-size (bytes)
+        static const int mpiBufferSize;
+
         //- Default communicator (all processors)
         static label worldComm;
 

src/Pstream/mpi/UPstream.C

@@ -29,6 +29,7 @@ License
 #include "PstreamGlobals.H"
 #include "SubList.H"
 #include "allReduce.H"
+#include "int.H"
 
 #include <mpi.h>
 
@@ -42,6 +43,12 @@ License
     #define MPI_SCALAR MPI_DOUBLE
 #endif
 
+// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
+
+// file-scope: min value and default for mpiBufferSize
+static const int minBufferSize = 20000000;
+
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 // NOTE:
@@ -87,33 +94,32 @@ bool Foam::UPstream::init(int& argc, char**& argv)
     setParRun(numprocs);
 
     #ifndef SGIMPI
-    string bufferSizeName = getEnv("MPI_BUFFER_SIZE");
-
-    if (bufferSizeName.size())
     {
-        int bufferSize = atoi(bufferSizeName.c_str());
+        // Normally use UPstream::mpiBufferSize (optimisationSwitch),
+        // but allow override with the MPI_BUFFER_SIZE env variable
+        // which has an int value
+        int bufSize = 0;
 
-        if (bufferSize)
+        const std::string str = Foam::getEnv("MPI_BUFFER_SIZE");
+        if (str.empty() || !Foam::read(str.c_str(), bufSize) || bufSize <= 0)
         {
-            MPI_Buffer_attach(new char[bufferSize], bufferSize);
+            bufSize = mpiBufferSize;
         }
-    }
-    else
-    {
-        FatalErrorInFunction
-            << "UPstream::init(int& argc, char**& argv) : "
-            << "environment variable MPI_BUFFER_SIZE not defined"
-            << Foam::abort(FatalError);
+
+        if (bufSize < minBufferSize)
+        {
+            bufSize = minBufferSize;
+        }
+
+        if (debug)
+        {
+            Pout<< "UPstream::init : mpi-buffer-size " << bufSize << endl;
+        }
+
+        MPI_Buffer_attach(new char[bufSize], bufSize);
     }
     #endif
 
-    //int processorNameLen;
-    //char processorName[MPI_MAX_PROCESSOR_NAME];
-    //
-    //MPI_Get_processor_name(processorName, &processorNameLen);
-    //processorName[processorNameLen] = '\0';
-    //Pout<< "Processor name:" << processorName << endl;
-
     return true;
 }
 
@@ -126,10 +132,12 @@ void Foam::UPstream::exit(int errnum)
     }
 
     #ifndef SGIMPI
-    int size;
-    char* buff;
-    MPI_Buffer_detach(&buff, &size);
-    delete[] buff;
+    {
+        int size;
+        char* buff;
+        MPI_Buffer_detach(&buff, &size);
+        delete[] buff;
+    }
     #endif
 
     if (PstreamGlobals::outstandingRequests_.size())