BUG: missing parallel-aware for metis-like decomposition

applications/test/decomposePar/Make/options

@@ -8,5 +8,6 @@ EXE_INC = \
 EXE_LIBS = \
     -ldecompose \
     -ldecompositionMethods \
-    -L$(FOAM_LIBBIN)/dummy -lmetisDecomp -lscotchDecomp -lkahipDecomp \
+    -L$(FOAM_LIBBIN)/dummy \
+    -lkahipDecomp -lmetisDecomp -lscotchDecomp \
     -lregionModels

applications/utilities/mesh/generation/foamyMesh/cellSizeAndAlignmentGrid/Make/options

@@ -28,5 +28,6 @@ EXE_LIBS =  \
     -lmeshTools \
     -lsampling \
     -ldecompositionMethods \
-    -L$(FOAM_LIBBIN)/dummy -lscotchDecomp -lptscotchDecomp \
+    -L$(FOAM_LIBBIN)/dummy \
+    -lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp \
     -ldynamicMesh

applications/utilities/mesh/generation/foamyMesh/foamyHexMesh/Make/options

@@ -31,5 +31,6 @@ EXE_LIBS = \
     -lsampling \
     -ldecompositionMethods \
     -ldecompose \
-    -L$(FOAM_LIBBIN)/dummy -lptscotchDecomp -lscotchDecomp \
+    -L$(FOAM_LIBBIN)/dummy \
+    -lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp \
     -ldynamicMesh

applications/utilities/mesh/generation/foamyMesh/foamyHexMeshSurfaceSimplify/Make/options

@@ -18,6 +18,8 @@ EXE_LIBS = \
     -L$(FASTDUALOCTREE_SRC_PATH) -lperf_main \
     -lGL \
     -lconformalVoronoiMesh \
-    -ldecompositionMethods -L$(FOAM_LIBBIN)/dummy -lscotchDecomp \
+    -ldecompositionMethods \
+    -L$(FOAM_LIBBIN)/dummy \
+    -lkahipDecomp -lmetisDecomp -lscotchDecomp \
     -lmeshTools \
     -ldynamicMesh

applications/utilities/mesh/generation/foamyMesh/foamyQuadMesh/Make/options

@@ -31,4 +31,5 @@ EXE_LIBS = \
     -lsampling \
     -ldynamicMesh \
     -ldecompositionMethods \
-    -L$(FOAM_LIBBIN)/dummy -lscotchDecomp -lptscotchDecomp
+    -L$(FOAM_LIBBIN)/dummy \
+    -lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp

applications/utilities/mesh/generation/snappyHexMesh/Make/options

@@ -13,9 +13,8 @@ EXE_INC = \
 EXE_LIBS = \
     -lfiniteVolume \
     -ldecompositionMethods \
-    -L$(FOAM_LIBBIN)/dummy -lptscotchDecomp \
-    /* note: scotch < 6.0 does not like both scotch and ptscotch together */ \
-    -lscotchDecomp \
+    -L$(FOAM_LIBBIN)/dummy \
+    -lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp \
     -lmeshTools \
     -ldynamicMesh \
     -ldecompose \

applications/utilities/mesh/manipulation/renumberMesh/Make/options

@@ -18,4 +18,5 @@ EXE_LIBS = \
     -lreconstruct \
     $(LINK_FLAGS) \
     -ldecompositionMethods \
-    -L$(FOAM_LIBBIN)/dummy -lmetisDecomp -lscotchDecomp -lkahipDecomp
+    -L$(FOAM_LIBBIN)/dummy \
+    -lkahipDecomp -lmetisDecomp -lscotchDecomp

applications/utilities/parallelProcessing/decomposePar/Make/options

@@ -12,7 +12,8 @@ EXE_LIBS = \
     -ldecompose \
     -lgenericPatchFields \
     -ldecompositionMethods \
-    -L$(FOAM_LIBBIN)/dummy -lmetisDecomp -lscotchDecomp -lkahipDecomp \
+    -L$(FOAM_LIBBIN)/dummy \
+    -lkahipDecomp -lmetisDecomp -lscotchDecomp \
     -llagrangian \
     -ldynamicMesh \
     -lregionModels

applications/utilities/parallelProcessing/redistributePar/Make/options

@@ -11,7 +11,8 @@ EXE_LIBS = \
     -lfiniteVolume \
     -lgenericPatchFields \
     -ldecompositionMethods \
-    -L$(FOAM_LIBBIN)/dummy -lptscotchDecomp -lscotchDecomp \
+    -L$(FOAM_LIBBIN)/dummy \
+    -lkahipDecomp -lmetisDecomp -lptscotchDecomp -lscotchDecomp \
     -ldecompose \
     -lmeshTools \
     -llagrangian \

src/parallel/decompose/decompositionMethods/metisLikeDecomp/metisLikeDecomp.C

@@ -181,7 +181,7 @@ Foam::labelList Foam::metisLikeDecomp::decompose
         mesh,
         identity(mesh.nCells()),
         mesh.nCells(),
-        false,
+        true,
         cellCells
     );
 
@@ -214,7 +214,14 @@ Foam::labelList Foam::metisLikeDecomp::decompose
     //   xadj(celli) : start of information in adjncy for celli
 
     CompactListList<label> cellCells;
-    calcCellCells(mesh, agglom, agglomPoints.size(), false, cellCells);
+    calcCellCells
+    (
+        mesh,
+        agglom,
+        agglomPoints.size(),
+        true,
+        cellCells
+    );
 
     // Decompose using default weights
     labelList decomp;

src/parallel/decompose/kahipDecomp/kahipDecomp.C

@@ -92,7 +92,9 @@ Foam::label Foam::kahipDecomp::decomposeSerial
     List<int> cellWeights;
 
     // Check for externally provided cellweights and if so initialise weights
-    const scalar minWeights = gMin(cWeights);
+    // Note: min, not gMin since routine runs on master only.
+    const scalar minWeights = min(cWeights);
+
     if (!cWeights.empty())
     {
         if (minWeights <= 0)

src/parallel/decompose/metisDecomp/metisDecomp.C

@@ -77,10 +77,11 @@ Foam::label Foam::metisDecomp::decomposeSerial
     // Face weights (so on the edges of the dual)
     List<label> faceWeights;
 
-
     // Check for externally provided cellweights and if so initialise weights
-    const scalar minWeights = gMin(cWeights);
-    if (cWeights.size() > 0)
+    // Note: min, not gMin since routine runs on master only.
+    const scalar minWeights = min(cWeights);
+
+    if (!cWeights.empty())
     {
         if (minWeights <= 0)
         {

tutorials/heatTransfer/chtMultiRegionFoam/externalSolarLoad/Allrun

@@ -12,20 +12,21 @@ cd ${0%/*} || exit 1    # Run from this directory
 # Agglomerate patch faces
 for i in air
 do
-    faceAgglomerate -region $i -dict constant/viewFactorsDict > log.faceAgglomerate.$i 2>&1
+    runApplication -s $i \
+        faceAgglomerate -region $i -dict constant/viewFactorsDict
 done
 
 # Generate view factors
 for i in air
 do
-    viewFactorsGen -region $i > log.viewFactorsGen.$i 2>&1
+    runApplication -s $i \
+        viewFactorsGen -region $i
 done
 
 runApplication $(getApplication)
 
-
 echo
-echo "creating files for paraview post-processing"
+echo "Creating files for paraview post-processing"
 echo
 paraFoam -touchAll
 

tutorials/heatTransfer/chtMultiRegionFoam/externalSolarLoad/Allrun-parallel

 #!/bin/sh
-cd ${0%/*} || exit 1    # Run from this directory
-
-# Source tutorial run functions
-. $WM_PROJECT_DIR/bin/tools/RunFunctions
+cd ${0%/*} || exit 1                        # Run from this directory
+. $WM_PROJECT_DIR/bin/tools/RunFunctions    # Tutorial run functions
 
 # Setup case
 ./Allrun.pre
@@ -15,18 +13,18 @@ runApplication decomposePar -allRegions -constant
 # Agglomerate patch faces
 for i in air
 do
-    runParallel -s $i -np 4 \
+    runParallel -s $i \
         faceAgglomerate -region $i -dict constant/viewFactorsDict
 done
 
 # Generate view factors
 for i in air
 do
-     runParallel -s $i -np 4 \
+    runParallel -s $i \
         viewFactorsGen -region $i
 done
 
-# set the initial fields
+# Set the initial fields
 restore0Dir
 
 runParallel $(getApplication)
@@ -35,7 +33,7 @@ runParallel $(getApplication)
 runApplication reconstructPar -allRegions
 
 echo
-echo "creating files for paraview post-processing"
+echo "Creating files for paraview post-processing"
 echo
 paraFoam -touchAll
 

tutorials/heatTransfer/chtMultiRegionFoam/externalSolarLoad/Allrun.pre

 #!/bin/sh
-cd ${0%/*} || exit 1    # Run from this directory
-
-# Source tutorial run functions
-. $WM_PROJECT_DIR/bin/tools/RunFunctions
+cd ${0%/*} || exit 1                        # Run from this directory
+. $WM_PROJECT_DIR/bin/tools/RunFunctions    # Tutorial run functions
 
 runApplication blockMesh
 runApplication topoSet
 runApplication splitMeshRegions -cellZones -overwrite
 
 # Remove unwanted region
-rm -r 0/domain3
-rm -r constant/domain3
-rm -r system/domain3
+rm -rf 0/domain3 constant/domain3 system/domain3
 
-# remove fluid fields from solid regions (important for post-processing)
+# Remove fluid fields from solid regions (important for post-processing)
 for i in solid floor
 do
    rm -f 0*/$i/{rho,mut,alphat,epsilon,k,U,p_rgh,qr,G,IDefault}
 done
 
-# set the initial fields
+# Set the initial fields
 restore0Dir
 
 for i in air solid floor
 do
-    runApplication -s $i changeDictionary -region $i -subDict dictionaryReplacement
+    runApplication -s $i changeDictionary \
+        -region $i -subDict dictionaryReplacement
 done
 
 # -----------------------------------------------------------------------------

tutorials/heatTransfer/chtMultiRegionSimpleFoam/multiRegionHeaterRadiation/Allrun-parallel

@@ -4,11 +4,9 @@ cd ${0%/*} || exit 1    # Run from this directory
 # Source tutorial run functions
 . $WM_PROJECT_DIR/bin/tools/RunFunctions
 
-
 # Setup case
 ./Allrun.pre
 
-
 #-- Run in parallel
 
 # Decompose
@@ -17,14 +15,14 @@ runApplication decomposePar -allRegions
 # Agglomerate patch faces
 for i in bottomAir topAir
 do
-    runParallel -s $i -np 4 \
+    runParallel -s $i \
         faceAgglomerate -region $i -dict constant/viewFactorsDict
 done
 
 # Generate view factors
 for i in bottomAir topAir
 do
-    runParallel -s $i -np 4 \
+    runParallel -s $i \
         viewFactorsGen -region $i
 done
 
@@ -34,9 +32,8 @@ runParallel $(getApplication)
 # Reconstruct
 runApplication reconstructPar -allRegions
 
-
 echo
-echo "creating files for paraview post-processing"
+echo "Creating files for paraview post-processing"
 echo
 paraFoam -touchAll
 

tutorials/mesh/parallel/cavity/Allclean

 #!/bin/sh
-cd ${0%/*} || exit 1    # Run from this directory
-
-# Source tutorial clean functions
-. $WM_PROJECT_DIR/bin/tools/CleanFunctions
+cd ${0%/*} || exit 1                        # Run from this directory
+. $WM_PROJECT_DIR/bin/tools/CleanFunctions  # Tutorial clean functions
 
 cleanCase
 
 # Restore default dictionaries
-cp system/decomposeParDict-2 system/decomposeParDict
 cp system/controlDict-startTime system/controlDict
 
 # -----------------------------------------------------------------------------

tutorials/mesh/parallel/cavity/Allrun

 #!/bin/sh
-cd ${0%/*} || exit 1    # Run from this directory
-
-# Source tutorial run functions
-. $WM_PROJECT_DIR/bin/tools/RunFunctions
+cd ${0%/*} || exit 1                        # Run from this directory
+. $WM_PROJECT_DIR/bin/tools/RunFunctions    # Tutorial run functions
 
 runApplication blockMesh
 
-#cp system/decomposeParDict-2 system/decomposeParDict
-#runApplication decomposePar
+decompDict5="-decomposeParDict system/decomposeParDict-5"
 
 # redistributePar to do decomposition
 runParallel -s decompose redistributePar -decompose
@@ -25,15 +22,12 @@ runParallel -s random icoFoam
 cp system/controlDict-latestTime system/controlDict
 
 # Redistribute to 5 processors
-runParallel -s 5 -np 5 redistributePar \
-    -decomposeParDict system/decomposeParDict-5 -cellDist
+runParallel -s 5 $decompDict5 redistributePar -cellDist
 
 # Run a bit more
-runParallel -s 5 -np 5 icoFoam \
-    -decomposeParDict system/decomposeParDict-5
+runParallel -s 5 $decompDict5 icoFoam
 
 # Reconstruct mesh and results
-runParallel -s reconstruct  -np 5 redistributePar -reconstruct
-
+runParallel -s reconstruct -np 5 redistributePar -reconstruct
 
 # -----------------------------------------------------------------------------

tutorials/mesh/parallel/cavity/system/decomposeParDict

@@ -43,13 +43,14 @@ numberOfSubdomains  2;
 //  for a balanced number of particles in a lagrangian simulation.
 // weightField dsmcRhoNMean;
 
-method          scotch;
-//method          hierarchical;
-// method          simple;
-// method          metis;
-// method          manual;
-// method          multiLevel;
-// method          structured;  // does 2D decomposition of structured mesh
+method      scotch;
+// method      kahip;
+// method      metis;
+// method      hierarchical;
+// method      simple;
+// method      manual;
+// method      multiLevel;
+// method      structured;  // does 2D decomposition of structured mesh
 
 multiLevelCoeffs
 {

tutorials/mesh/parallel/cavity/system/decomposeParDict-2

-/*--------------------------------*- C++ -*----------------------------------*\
-| =========                 |                                                 |
-| \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox           |
-|  \\    /   O peration     | Version:  plus                                  |
-|   \\  /    A nd           | Web:      www.OpenFOAM.com                      |
-|    \\/     M anipulation  |                                                 |
-\*---------------------------------------------------------------------------*/
-FoamFile
-{
-    version     2.0;
-    format      ascii;
-    class       dictionary;
-    note        "mesh decomposition control dictionary";
-    object      decomposeParDict;
-}
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-numberOfSubdomains  2;
-
-//- Keep owner and neighbour on same processor for faces in zones:
-// preserveFaceZones (heater solid1 solid3);
-
-//- Keep owner and neighbour on same processor for faces in patches:
-//  (makes sense only for cyclic patches)
-//preservePatches (cyclic_half0 cyclic_half1);
-
-//- Keep all of faceSet on a single processor. This puts all cells
-//  connected with a point, edge or face on the same processor.
-//  (just having face connected cells might not guarantee a balanced
-//  decomposition)
-// The processor can be -1 (the decompositionMethod chooses the processor
-// for a good load balance) or explicitly provided (upsets balance).
-//singleProcessorFaceSets ((f0 -1));
-
-
-//- Keep owner and neighbour of baffles on same processor (i.e. keep it
-//  detectable as a baffle). Baffles are two boundary face sharing the
-//  same points.
-//preserveBaffles true;
-
-//- Use the volScalarField named here as a weight for each cell in the
-//  decomposition.  For example, use a particle population field to decompose
-//  for a balanced number of particles in a lagrangian simulation.
-// weightField dsmcRhoNMean;
-
-method          scotch;
-//method          hierarchical;
-// method          simple;
-// method          metis;
-// method          manual;
-// method          multiLevel;
-// method          structured;  // does 2D decomposition of structured mesh
-
-multiLevelCoeffs
-{
-    // Decomposition methods to apply in turn. This is like hierarchical but
-    // fully general - every method can be used at every level.
-
-    level0
-    {
-        numberOfSubdomains  64;
-        //method simple;
-        //simpleCoeffs
-        //{
-        //    n           (2 1 1);
-        //    delta       0.001;
-        //}
-        method scotch;
-    }
-    level1
-    {
-        numberOfSubdomains  4;
-        method scotch;
-    }
-}
-
-// Desired output
-
-simpleCoeffs
-{
-    n           (2 1 1);
-    delta       0.001;
-}
-
-hierarchicalCoeffs
-{
-    n           (1 2 1);
-    delta       0.001;
-    order       xyz;
-}
-
-metisCoeffs
-{
- /*
-    processorWeights
-    (
-        1
-        1
-        1
-        1
-    );
-  */
-}
-
-scotchCoeffs
-{
-    //processorWeights
-    //(
-    //    1
-    //    1
-    //    1
-    //    1
-    //);
-    //writeGraph  true;
-    //strategy "b";
-}
-
-manualCoeffs
-{
-    dataFile    "decompositionData";
-}
-
-structuredCoeffs
-{
-    // Patches to do 2D decomposition on. Structured mesh only; cells have
-    // to be in 'columns' on top of patches.
-    patches     (movingWall);
-
-    // Method to use on the 2D subset
-    method      scotch;
-}
-
-//// Is the case distributed? Note: command-line argument -roots takes
-//// precedence
-//distributed     yes;
-//// Per slave (so nProcs-1 entries) the directory above the case.
-//roots
-//(
-//    "/tmp"
-//    "/tmp"
-//);
-
-// ************************************************************************* //