BUG: GAMG: processorAgglomeration not merging proc coeffs. Fixes #3172

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2023 OpenCFD Ltd.
+    Copyright (C) 2023-2024 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -310,13 +310,7 @@ void Foam::GAMGSolver::gatherMatrices
 
     const auto& boundaryMap = agglomeration_.boundaryMap(destLevel);
 
-    // Use PstreamBuffers
-    PstreamBuffers pBufs
-    (
-        UPstream::commsTypes::nonBlocking,
-        UPstream::msgType(),
-        comm
-    );
+    PstreamBuffers pBufs(comm);
 
     // Send to master
     if (!UPstream::master(comm))
@@ -332,35 +326,49 @@ void Foam::GAMGSolver::gatherMatrices
 
         const label proci = UPstream::myProcNo(comm);
 
-        labelList validInterface(interfaces.size(), -1);
+        // All interfaceBouCoeffs need to be sent across
+        bitSet validCoeffs(interfaces.size());
+        forAll(interfaceBouCoeffs, intI)
+        {
+            if (interfaceBouCoeffs.set(intI))
+            {
+                validCoeffs.set(intI);
+            }
+        }
+
+        // Only preserved interfaces need to be sent across
+        bitSet validInterface(interfaces.size());
         forAll(interfaces, intI)
         {
             const label allIntI = boundaryMap[proci][intI];
             if (interfaces.set(intI) && allIntI != -1)
             {
-                validInterface[intI] = intI;
+                validInterface.set(intI);
             }
         }
 
         UOPstream toMaster(UPstream::masterNo(), pBufs);
 
-        toMaster<< mat << token::SPACE << validInterface;
+        toMaster
+            << mat
+            << token::SPACE << validCoeffs
+            << token::SPACE << validInterface;
 
-        forAll(validInterface, intI)
+        for (const label intI : validCoeffs)
         {
-            if (validInterface[intI] != -1)
-            {
-                const auto& interface = refCast<const GAMGInterfaceField>
-                (
-                    interfaces[intI]
-                );
+            toMaster
+                << interfaceBouCoeffs[intI]
+                << interfaceIntCoeffs[intI];
+        }
+        for (const label intI : validInterface)
+        {
+            const auto& interface = refCast<const GAMGInterfaceField>
+            (
+                interfaces[intI]
+            );
 
-                toMaster
-                    << interfaceBouCoeffs[intI]
-                    << interfaceIntCoeffs[intI]
-                    << interface.type();
-                interface.write(toMaster);
-            }
+            toMaster << interface.type();
+            interface.write(toMaster);
         }
     }
 
@@ -376,10 +384,10 @@ void Foam::GAMGSolver::gatherMatrices
         lduInterfacePtrsList destInterfaces = destMesh.interfaces();
 
         // Master.
-        otherMats.setSize(nProcs-1);
-        otherBouCoeffs.setSize(nProcs-1);
-        otherIntCoeffs.setSize(nProcs-1);
-        otherInterfaces.setSize(nProcs-1);
+        otherMats.resize(nProcs-1);
+        otherBouCoeffs.resize(nProcs-1);
+        otherIntCoeffs.resize(nProcs-1);
+        otherInterfaces.resize(nProcs-1);
 
         for (const int proci : UPstream::subProcs(comm))
         {
@@ -389,60 +397,41 @@ void Foam::GAMGSolver::gatherMatrices
 
             otherMats.set(otherI, new lduMatrix(destMesh, fromProc));
 
-            // Receive number of/valid interfaces
-            // >= 0 : remote interface index
-            // -1   : invalid interface
-            const labelList validInterface(fromProc);
+            // Receive bit-sets of valid interfaceCoeffs/interfaces
+            const bitSet validCoeffs(fromProc);
+            const bitSet validInterface(fromProc);
 
-            otherBouCoeffs.set
-            (
-                otherI,
-                new FieldField<Field, scalar>(validInterface.size())
-            );
-            otherIntCoeffs.set
-            (
-                otherI,
-                new FieldField<Field, scalar>(validInterface.size())
-            );
-            otherInterfaces.set
-            (
-                otherI,
-                new PtrList<lduInterfaceField>(validInterface.size())
-            );
+            otherBouCoeffs.emplace_set(otherI, validCoeffs.size());
+            otherIntCoeffs.emplace_set(otherI, validCoeffs.size());
+            otherInterfaces.emplace_set(otherI, validInterface.size());
 
-            forAll(validInterface, intI)
+            // Receive individual interface contributions
+            for (const label intI : validCoeffs)
             {
-                if (validInterface[intI] != -1)
-                {
-                    otherBouCoeffs[otherI].set
-                    (
-                        intI,
-                        new scalarField(fromProc)
-                    );
-                    otherIntCoeffs[otherI].set
-                    (
-                        intI,
-                        new scalarField(fromProc)
-                    );
+                otherBouCoeffs[otherI].emplace_set(intI, fromProc);
+                otherIntCoeffs[otherI].emplace_set(intI, fromProc);
+            }
 
-                    const word coupleType(fromProc);
+            // Receive individual interface contributions
+            for (const label intI : validInterface)
+            {
+                const word coupleType(fromProc);
 
-                    const label allIntI = boundaryMap[proci][intI];
+                const label allIntI = boundaryMap[proci][intI];
 
-                    otherInterfaces[otherI].set
+                otherInterfaces[otherI].set
+                (
+                    intI,
+                    GAMGInterfaceField::New
                     (
-                        intI,
-                        GAMGInterfaceField::New
+                        coupleType,
+                        refCast<const GAMGInterface>
                         (
-                            coupleType,
-                            refCast<const GAMGInterface>
-                            (
-                                destInterfaces[allIntI]
-                            ),
-                            fromProc
-                        ).release()
-                    );
-                }
+                            destInterfaces[allIntI]
+                        ),
+                        fromProc
+                    ).release()
+                );
             }
         }
     }