Feature master coarsest multi masters

Processor agglomeration in the GAMG linear solver

The current GAMG linear solver supports processor agglomeration on the coarsest level using the masterCoarsest processor agglomerator method. This removes communication, increases implicitness but increases the size of the coarsest level and at some core count will not be beneficial.

Extend masterCoarsest for multiple master processors

A test was done varying the number of master processors - in below table it starts at a single master (responsible for 1728 processors) and increases the number of masters to 48 (each responsible for 36 processors).

Coarsest level procs	run1 (s)	run2 (s)
1 (1728)	193	219
2 (864)	141	156
4 (432)	167	126
8 (216)	135	114
16 (108)	140	126
48 (36)	247	--

Especially run2 shows a benefit of using more than one master processor. Other testing on a different cluster has shown no benefit of master-coarsest on one or more than one master processor - it all depends on cost of local computation versus communication/explicitness.

Above testing was done using complicated dictionary scripting to manually agglomerate processors. In this version this has been integrated into the masterCoarsest processor agglomeration using the new nMasters or nProcssorsPerMaster keyword:

{
    solver          GAMG;
    ..
    processorAgglomerator   masterCoarsest;
    nCellsInCoarsestLevel   1;
    nMasters                2;
}

With debug switches in the system/controlDict

DebugSwitches
{
    // Print number of processors per master
    masterCoarsest      1;
    // Print agglomeration
    GAMGAgglomeration   1;
}

we can see the effect of using processor agglomeration on a simple case decomposed onto 17 processors:

masterCoarsest : agglomerating
    master  procs
    0       9 (1 2 3 4 5 6 7 8)
    9       8 (10 11 12 13 14 15 16)
GAMGAgglomeration:
    local agglomerator     : faceAreaPair
    processor agglomerator : masterCoarsest

                              nCells       nFaces/nCells         nInterfaces    nIntFaces/nCells     profile
   Level  nProcs         avg     max         avg     max         avg     max         avg     max         avg
   -----  ------         ---     ---         ---     ---         ---     ---         ---     ---         ---
       0      17         719     725       1.922   1.926       3.529       5      0.1093  0.1335   1.797e+04
       1      17         359     362       1.966   2.109       3.529       5      0.1864  0.2632        7291
       2      17         176     181       2.352   2.769       3.529       5       0.272   0.436        2810
       3      17          86      90       2.344   2.593       3.529       5      0.4415  0.7738       930.4
       4      17          42      44       2.291   2.442       3.529       5      0.6747    1.22       320.9
       5      17          20      22       2.094   2.286       3.529       5       0.967   1.895       98.41
       6      17           9      11       1.741       2       3.529       5        1.35   2.444          29
       7      17           4       5       1.149     1.6       3.529       5       2.082     3.5           6
       8       2          15      18       1.585   1.615           1       1      0.5962  0.6923          40
       9       2           8       9       1.417     1.5           1       1      0.7083    0.75          15

Tutorials:

• compressible/rhoSimpleFoam/squareBendLiq (use of masterCoarsest)

Source code

• src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGProcAgglomerations/masterCoarsestGAMGProcAgglomeration/masterCoarsestGAMGProcAgglomeration.C

@@ -6,6 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013-2014 OpenFOAM Foundation
+    Copyright (C) 2022 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -52,8 +53,48 @@ Foam::masterCoarsestGAMGProcAgglomeration::masterCoarsestGAMGProcAgglomeration
     const dictionary& controlDict
 )
 :
-    GAMGProcAgglomeration(agglom, controlDict)
-{}
+    GAMGProcAgglomeration(agglom, controlDict),
+    nProcessorsPerMaster_
+    (
+        controlDict.getOrDefault<label>
+        (
+            "nProcessorsPerMaster",
+            0,
+            keyType::LITERAL
+        )
+    )
+{
+    const auto* ePtr = controlDict.findEntry("nMasters", keyType::LITERAL);
+    if (ePtr)
+    {
+        if (nProcessorsPerMaster_ > 0)
+        {
+            FatalIOErrorInFunction(controlDict)
+                << "Cannot specify both \"nMasters\" and"
+                << " \"nProcessorsPerMaster\"" << exit(FatalIOError);
+        }
+
+        const label nMasters(readLabel(ePtr->stream()));
+
+        if (nMasters <= 0)
+        {
+            FatalIOErrorInFunction(controlDict)
+                << "Illegal value \"nMasters\" "
+                << nMasters << exit(FatalIOError);
+        }
+
+        nProcessorsPerMaster_ =
+            (Pstream::nProcs(agglom.mesh().comm())+nMasters-1)
+          / nMasters;
+    }
+
+    if (nProcessorsPerMaster_ < 0)
+    {
+        FatalIOErrorInFunction(controlDict)
+            << "Illegal value \"nProcessorsPerMaster\" "
+            << nProcessorsPerMaster_ << exit(FatalIOError);
+    }
+}
 
 
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
@@ -75,7 +116,7 @@ Foam::masterCoarsestGAMGProcAgglomeration::
 
 bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
 {
-    if (debug)
+    if (debug & 2)
     {
         Pout<< nl << "Starting mesh overview" << endl;
         printStats(Pout, agglom_);
@@ -97,7 +138,23 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
             if (nProcs > 1)
             {
                 // Processor restriction map: per processor the coarse processor
-                labelList procAgglomMap(nProcs, Zero);
+                labelList procAgglomMap(nProcs);
+
+                if (nProcessorsPerMaster_ > 0)
+                {
+                    forAll(procAgglomMap, fineProci)
+                    {
+                        procAgglomMap[fineProci] =
+                        (
+                            fineProci
+                          / nProcessorsPerMaster_
+                        );
+                    }
+                }
+                else
+                {
+                    procAgglomMap = Zero;
+                }
 
                 // Master processor
                 labelList masterProcs;
@@ -112,6 +169,32 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
                     agglomProcIDs
                 );
 
+                if (debug)
+                {
+                    if (masterProcs.size())
+                    {
+                        labelListList masterToProcs
+                        (
+                            invertOneToMany
+                            (
+                                masterProcs.size(),
+                                procAgglomMap
+                            )
+                        );
+                        Info<< typeName << " : agglomerating" << nl
+                            << "\tmaster\tnProcs\tprocIDs" << endl;
+                        for (const auto& p : masterToProcs)
+                        {
+                            Info<< '\t' << p[0]
+                                << '\t' << p.size()
+                                << '\t'
+                                << flatOutput(SubList<label>(p, p.size()-1, 1))
+                                << endl;
+                        }
+                    }
+                }
+
+
                 // Allocate a communicator for the processor-agglomerated matrix
                 comms_.append
                 (
@@ -139,7 +222,7 @@ bool Foam::masterCoarsestGAMGProcAgglomeration::agglomerate()
     }
 
     // Print a bit
-    if (debug)
+    if (debug & 2)
     {
         Pout<< nl << "Agglomerated mesh overview" << endl;
         printStats(Pout, agglom_);