ENH: Creating nonBlocking comms to exchange referredParticles.

Currently operating within one function call and not doing local comms
separately.

Using inplaceSubset instead of inplaceReorder and setSize.

src/lagrangian/basic/InteractionLists/InteractionLists.C

@@ -307,6 +307,44 @@ void Foam::InteractionLists<ParticleType>::buildMap
     );
 }
 
+
+template<class ParticleType>
+void Foam::InteractionLists<ParticleType>::prepareParticlesToRefer
+(
+    const List<DynamicList<ParticleType*> >& cellOccupancy
+)
+{
+    // Clear all existing referred particles
+
+    forAll(referredParticles_, i)
+    {
+        referredParticles_[i].clear();
+    }
+
+    referredParticles_.setSize(cellIndexAndTransformToDistribute_.size());
+
+    forAll(cellIndexAndTransformToDistribute_, i)
+    {
+        const labelPair giat = cellIndexAndTransformToDistribute_[i];
+
+        label cellIndex = globalTransforms_.index(giat);
+
+        List<ParticleType*> realParticles = cellOccupancy[cellIndex];
+
+        IDLList<ParticleType>& particlesToRefer = referredParticles_[i];
+
+        forAll (realParticles, rM)
+        {
+            const ParticleType& particle = *realParticles[rM];
+
+            particlesToRefer.append(particle.clone().ptr());
+
+            prepareParticleToBeReferred(particlesToRefer.last(), giat);
+        }
+    }
+}
+
+
 template<class ParticleType>
 void InteractionLists<ParticleType>::prepareParticleToBeReferred
 (
@@ -323,6 +361,32 @@ void InteractionLists<ParticleType>::prepareParticleToBeReferred
 }
 
 
+template<class ParticleType>
+void InteractionLists<ParticleType>::writeReferredParticleCloud()
+{
+    bool writeCloud = true;
+
+    if (mesh_.time().outputTime() && writeCloud)
+    {
+        cloud_.clear();
+
+        forAll(referredParticles_, refCellI)
+        {
+            const IDLList<ParticleType>& refCell = referredParticles_[refCellI];
+
+            forAllConstIter(typename IDLList<ParticleType>, refCell, iter)
+            {
+                cloud_.addParticle(iter().clone().ptr());
+            }
+        }
+
+        Particle<ParticleType>::writeFields(cloud_);
+
+        cloud_.clear();
+    }
+}
+
+
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 template<class ParticleType>
@@ -525,21 +589,23 @@ Foam::InteractionLists<ParticleType>::InteractionLists
         // the order and structure is preserved, i.e. it, is as if the
         // cell had never been referred in the first place.
 
-        labelList oldToNew(globalIAndTToExchange.size(), -1);
+        // labelList oldToNew(globalIAndTToExchange.size(), -1);
 
-        label refCellI = 0;
+        // label refCellI = 0;
 
-        forAll(bbRequiredByAnyCell, bbReqI)
-        {
-            if (bbRequiredByAnyCell[bbReqI])
-            {
-                oldToNew[bbReqI] = refCellI++;
-            }
-        }
+        // forAll(bbRequiredByAnyCell, bbReqI)
+        // {
+        //     if (bbRequiredByAnyCell[bbReqI])
+        //     {
+        //         oldToNew[bbReqI] = refCellI++;
+        //     }
+        // }
+
+        // inplaceReorder(oldToNew, ril_);
 
-        inplaceReorder(oldToNew, ril_);
+        // ril_.setSize(refCellI);
 
-        ril_.setSize(refCellI);
+        inplaceSubset(bbRequiredByAnyCell, ril_);
     }
 
     // Send information about which cells are actually required back
@@ -571,32 +637,36 @@ Foam::InteractionLists<ParticleType>::InteractionLists
     // and truncate it
 
     {
-        labelList oldToNew(globalIAndTToExchange.size(), -1);
+        // labelList oldToNew(globalIAndTToExchange.size(), -1);
 
-        label refCellI = 0;
+        // label refCellI = 0;
 
-        forAll(bbRequiredByAnyCell, bbReqI)
-        {
-            if (bbRequiredByAnyCell[bbReqI])
-            {
-                oldToNew[bbReqI] = refCellI++;
-            }
-        }
+        // forAll(bbRequiredByAnyCell, bbReqI)
+        // {
+        //     if (bbRequiredByAnyCell[bbReqI])
+        //     {
+        //         oldToNew[bbReqI] = refCellI++;
+        //     }
+        // }
 
-        inplaceReorder
-        (
-            oldToNew,
-            static_cast<List<labelPair>&>(globalIAndTToExchange)
-        );
+        // inplaceReorder
+        // (
+        //     oldToNew,
+        //     globalIAndTToExchange
+        // );
 
-        inplaceReorder
-        (
-            oldToNew,
-            static_cast<List<label>&>(procToDistributeTo)
-        );
+        // inplaceReorder
+        // (
+        //     oldToNew,
+        //     procToDistributeTo
+        // );
+
+        // globalIAndTToExchange.setSize(refCellI);
+        // procToDistributeTo.setSize(refCellI);
 
-        globalIAndTToExchange.setSize(refCellI);
-        procToDistributeTo.setSize(refCellI);
+        inplaceSubset(bbRequiredByAnyCell, globalIAndTToExchange);
+
+        inplaceSubset(bbRequiredByAnyCell, procToDistributeTo);
     }
 
     preDistributionSize = procToDistributeTo.size();
@@ -699,62 +769,73 @@ Foam::InteractionLists<ParticleType>::~InteractionLists()
 // * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * * //
 
 template<class ParticleType>
-void Foam::InteractionLists<ParticleType>::prepareParticlesToRefer
+void Foam::InteractionLists<ParticleType>::sendReferredParticles
 (
     const List<DynamicList<ParticleType*> >& cellOccupancy
 )
 {
-    // Clear all existing referred particles
+    prepareParticlesToRefer(cellOccupancy);
 
-    forAll(referredParticles_, i)
-    {
-        referredParticles_[i].clear();
-    }
-
-    referredParticles_.setSize(cellIndexAndTransformToDistribute_.size());
+    PstreamBuffers pBufs(Pstream::nonBlocking);
 
-    forAll(cellIndexAndTransformToDistribute_, i)
+    // Stream data into buffer
+    for (label domain = 0; domain < Pstream::nProcs(); domain++)
     {
-        const labelPair giat = cellIndexAndTransformToDistribute_[i];
-
-        label cellIndex = globalTransforms_.index(giat);
+        const labelList& subMap = map().subMap()[domain];
 
-        List<ParticleType*> realParticles = cellOccupancy[cellIndex];
-
-        IDLList<ParticleType>& particlesToRefer = referredParticles_[i];
-
-        forAll (realParticles, rM)
+        if (subMap.size())
         {
-            const ParticleType& particle = *realParticles[rM];
+            // Put data into send buffer
+            UOPstream toDomain(domain, pBufs);
 
-            particlesToRefer.append(particle.clone().ptr());
+            UIndirectList<IDLList<ParticleType> > subMappedParticles
+            (
+                referredParticles_,
+                subMap
+            );
 
-            prepareParticleToBeReferred(particlesToRefer.last(), giat);
+            forAll(subMappedParticles, i)
+            {
+                toDomain << subMappedParticles[i];
+            }
         }
     }
 
-    // map().distribute(referredParticles_);
+    // Start sending and receiving but do not block.
+    pBufs.finishedSends(false);
 
-    bool writeCloud = true;
+    // DO OTHER STUFF HERE;
 
-    if (mesh_.time().outputTime() && writeCloud)
+    Pstream::waitRequests();
+
+    referredParticles_.setSize(map().constructSize());
+
+    for (label domain = 0; domain < Pstream::nProcs(); domain++)
     {
-        cloud_.clear();
+        const labelList& constructMap = map().constructMap()[domain];
 
-        forAll(referredParticles_, refCellI)
+        if (constructMap.size())
         {
-            const IDLList<ParticleType>& refCell = referredParticles_[refCellI];
+            UIPstream str(domain, pBufs);
 
-            forAllConstIter(typename IDLList<ParticleType>, refCell, iter)
+            forAll (constructMap, i)
             {
-                cloud_.addParticle(iter().clone().ptr());
+                referredParticles_[constructMap[i]] = IDLList<ParticleType>
+                (
+                    str,
+                    typename ParticleType::iNew(cloud_)
+                );
             }
         }
+    }
 
-        Particle<ParticleType>::writeFields(cloud_);
+    writeReferredParticleCloud();
+};
 
-        cloud_.clear();
-    }
+
+template<class ParticleType>
+void Foam::InteractionLists<ParticleType>::receiveReferredParticles()
+{
 }
 
 

src/lagrangian/basic/InteractionLists/InteractionLists.H

@@ -112,6 +112,13 @@ class InteractionLists
         //- Build the mapDistribute
         void buildMap(const List<label>& toProc);
 
+        //- Fill the referredParticles container with particles that
+        //  will be referred
+        void prepareParticlesToRefer
+        (
+            const List<DynamicList<ParticleType*> >& cellOccupancy
+        );
+
         //- Prepare particle to be referred
         void prepareParticleToBeReferred
         (
@@ -119,6 +126,9 @@ class InteractionLists
             labelPair giat
         );
 
+        //- Write the referredParticles out to visualise
+        void writeReferredParticleCloud();
+
         //- Disallow default bitwise copy construct
         InteractionLists(const InteractionLists&);
 
@@ -144,13 +154,15 @@ public:
 
     // Member Functions
 
-        //- Fill the referredParticles container with particles that
-        //  will be referred
-        void prepareParticlesToRefer
+        //- Prepare and send referred particles, nonBlocking communication
+        void sendReferredParticles
         (
             const List<DynamicList<ParticleType*> >& cellOccupancy
         );
 
+        //- Receive referred particles
+        void receiveReferredParticles();
+
         // Access
 
             //- Return access to the mesh