ENH: improve MPI configuration possibilities

- permit SYSTEMMPI user adjustments via
      etc/config.{csh,sh}/mpi-system
  This can be a convenient place for setting up SYSTEMMPI for OpenFOAM
  without adjusting bashrc, prefs.sh ...

- add a USERMPI type. This represents any generic mpi implementation.
  The user is responsible for supplying an appropriate
  wmake/rules/General/mplibUSERMPI file and managing all settings.

  This type of setup can be useful in combination with specific build
  systems (SPACK, EASYBUILD, etc) or module systems for which the MPI
  variant is part of the installed configuration.

.gitignore

@@ -62,6 +62,7 @@ doc/Doxygen/DTAGS
 /etc/prefs.sh
 /etc/config.csh/prefs.csh
 /etc/config.sh/prefs.sh
+/wmake/rules/General/mplibUSER*
 
 # Source packages - anywhere
 *.tar.bz2

etc/bashrc

@@ -89,7 +89,7 @@ export WM_COMPILE_OPTION=Opt
 
 #- MPI implementation:
 #    WM_MPLIB = SYSTEMOPENMPI | OPENMPI | SYSTEMMPI | MPICH | MPICH-GM | HPMPI
-#               | MPI | FJMPI | QSMPI | SGIMPI | INTELMPI
+#               | MPI | FJMPI | QSMPI | SGIMPI | INTELMPI | USERMPI
 export WM_MPLIB=SYSTEMOPENMPI
 
 #- Operating System:

etc/config.csh/mpi

@@ -3,7 +3,7 @@
 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
 #  \\    /   O peration     |
 #   \\  /    A nd           | Copyright (C) 2011-2016 OpenFOAM Foundation
-#    \\/     M anipulation  |
+#    \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 #------------------------------------------------------------------------------
 # License
 #     This file is part of OpenFOAM.
@@ -25,11 +25,13 @@
 #     etc/config.csh/mpi
 #
 # Description
-#     Startup file for communications library (MPI) for OpenFOAM
+#     Setup file for communications library (MPI) for OpenFOAM
 #     Sourced from OpenFOAM-<VERSION>/etc/cshrc
 #
+#     For USERMPI, the user is responsible for supplying an appropriate
+#     wmake/rules/General/mplibUSERMPI file and managing all settings
+#
 #------------------------------------------------------------------------------
-
 unsetenv MPI_ARCH_PATH MPI_HOME FOAM_MPI_LIBBIN
 
 switch ("$WM_MPLIB")
@@ -48,8 +50,7 @@ case SYSTEMOPENMPI:
 
 case OPENMPI:
     setenv FOAM_MPI openmpi-2.0.2
-    # Optional configuration tweaks:
-    _foamEtc config.csh/openmpi
+    _foamEtc config.csh/openmpi                 # <- Adjustments (optional)
 
     setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
 
@@ -67,8 +68,15 @@ case OPENMPI:
     _foamAddMan     $MPI_ARCH_PATH/share/man
     breaksw
 
+case USERMPI:
+    # Use an arbitrary, user-specified mpi implementation
+    setenv FOAM_MPI mpi-user
+    _foamEtc config.csh/mpi-user                # <- Adjustments
+    breaksw
+
 case SYSTEMMPI:
     setenv FOAM_MPI mpi-system
+    _foamEtc config.csh/mpi-system              # <- Adjustments (optional)
 
     if ( ! $?MPI_ROOT ) then
         echo

etc/config.sh/mpi

@@ -3,7 +3,7 @@
 # \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
 #  \\    /   O peration     |
 #   \\  /    A nd           | Copyright (C) 2011-2016 OpenFOAM Foundation
-#    \\/     M anipulation  |
+#    \\/     M anipulation  | Copyright (C) 2017 OpenCFD Ltd.
 #------------------------------------------------------------------------------
 # License
 #     This file is part of OpenFOAM.
@@ -25,11 +25,13 @@
 #     etc/config.sh/mpi
 #
 # Description
-#     Startup file for communications library (MPI) for OpenFOAM
+#     Setup file for communications library (MPI) for OpenFOAM
 #     Sourced from OpenFOAM-<VERSION>/etc/bashrc
 #
+#     For USERMPI, the user is responsible for supplying an appropriate
+#     wmake/rules/General/mplibUSERMPI file and managing all settings
+#
 #------------------------------------------------------------------------------
-
 unset MPI_ARCH_PATH MPI_HOME FOAM_MPI_LIBBIN
 
 case "$WM_MPLIB" in
@@ -54,8 +56,7 @@ SYSTEMOPENMPI)
 
 OPENMPI)
     export FOAM_MPI=openmpi-2.0.2
-    # Optional configuration tweaks:
-    _foamEtc config.sh/openmpi
+    _foamEtc config.sh/openmpi                  # <- Adjustments (optional)
 
     export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
 
@@ -78,8 +79,15 @@ OPENMPI)
     fi
     ;;
 
+USERMPI)
+    # Use an arbitrary, user-specified mpi implementation
+    export FOAM_MPI=mpi-user
+    _foamEtc config.sh/mpi-user                 # <- Adjustments
+    ;;
+
 SYSTEMMPI)
     export FOAM_MPI=mpi-system
+    _foamEtc config.sh/mpi-system               # <- Adjustments (optional)
 
     if [ -z "$MPI_ROOT" ]
     then

etc/cshrc

@@ -86,7 +86,7 @@ setenv WM_COMPILE_OPTION Opt
 
 #- MPI implementation:
 #    WM_MPLIB = SYSTEMOPENMPI | OPENMPI | SYSTEMMPI | MPICH | MPICH-GM | HPMPI
-#               | MPI | FJMPI | QSMPI | SGIMPI | INTELMPI
+#               | MPI | FJMPI | QSMPI | SGIMPI | INTELMPI | USERMPI
 setenv WM_MPLIB SYSTEMOPENMPI
 
 #- Operating System: