From c22fce2006406f7a5788f8249a37dd06f7a84330 Mon Sep 17 00:00:00 2001
From: graham <g.macpherson@opencfd.co.uk>
Date: Wed, 7 Apr 2010 16:29:29 +0100
Subject: [PATCH] ENH: Creating nonBlocking comms to exchange
referredParticles.
Currently operating within one function call and not doing local comms
separately.
Using inplaceSubset instead of inplaceReorder and setSize.
---
.../basic/InteractionLists/InteractionLists.C | 207 ++++++++++++------
.../basic/InteractionLists/InteractionLists.H | 18 +-
2 files changed, 159 insertions(+), 66 deletions(-)
diff --git a/src/lagrangian/basic/InteractionLists/InteractionLists.C b/src/lagrangian/basic/InteractionLists/InteractionLists.C
index 1a3133e6536..6613b5a4392 100644
--- a/src/lagrangian/basic/InteractionLists/InteractionLists.C
+++ b/src/lagrangian/basic/InteractionLists/InteractionLists.C
@@ -307,6 +307,44 @@ void Foam::InteractionLists<ParticleType>::buildMap
);
}
+
+template<class ParticleType>
+void Foam::InteractionLists<ParticleType>::prepareParticlesToRefer
+(
+ const List<DynamicList<ParticleType*> >& cellOccupancy
+)
+{
+ // Clear all existing referred particles
+
+ forAll(referredParticles_, i)
+ {
+ referredParticles_[i].clear();
+ }
+
+ referredParticles_.setSize(cellIndexAndTransformToDistribute_.size());
+
+ forAll(cellIndexAndTransformToDistribute_, i)
+ {
+ const labelPair giat = cellIndexAndTransformToDistribute_[i];
+
+ label cellIndex = globalTransforms_.index(giat);
+
+ List<ParticleType*> realParticles = cellOccupancy[cellIndex];
+
+ IDLList<ParticleType>& particlesToRefer = referredParticles_[i];
+
+ forAll (realParticles, rM)
+ {
+ const ParticleType& particle = *realParticles[rM];
+
+ particlesToRefer.append(particle.clone().ptr());
+
+ prepareParticleToBeReferred(particlesToRefer.last(), giat);
+ }
+ }
+}
+
+
template<class ParticleType>
void InteractionLists<ParticleType>::prepareParticleToBeReferred
(
@@ -323,6 +361,32 @@ void InteractionLists<ParticleType>::prepareParticleToBeReferred
}
+template<class ParticleType>
+void InteractionLists<ParticleType>::writeReferredParticleCloud()
+{
+ bool writeCloud = true;
+
+ if (mesh_.time().outputTime() && writeCloud)
+ {
+ cloud_.clear();
+
+ forAll(referredParticles_, refCellI)
+ {
+ const IDLList<ParticleType>& refCell = referredParticles_[refCellI];
+
+ forAllConstIter(typename IDLList<ParticleType>, refCell, iter)
+ {
+ cloud_.addParticle(iter().clone().ptr());
+ }
+ }
+
+ Particle<ParticleType>::writeFields(cloud_);
+
+ cloud_.clear();
+ }
+}
+
+
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
template<class ParticleType>
@@ -525,21 +589,23 @@ Foam::InteractionLists<ParticleType>::InteractionLists
// the order and structure is preserved, i.e. it, is as if the
// cell had never been referred in the first place.
- labelList oldToNew(globalIAndTToExchange.size(), -1);
+ // labelList oldToNew(globalIAndTToExchange.size(), -1);
- label refCellI = 0;
+ // label refCellI = 0;
- forAll(bbRequiredByAnyCell, bbReqI)
- {
- if (bbRequiredByAnyCell[bbReqI])
- {
- oldToNew[bbReqI] = refCellI++;
- }
- }
+ // forAll(bbRequiredByAnyCell, bbReqI)
+ // {
+ // if (bbRequiredByAnyCell[bbReqI])
+ // {
+ // oldToNew[bbReqI] = refCellI++;
+ // }
+ // }
+
+ // inplaceReorder(oldToNew, ril_);
- inplaceReorder(oldToNew, ril_);
+ // ril_.setSize(refCellI);
- ril_.setSize(refCellI);
+ inplaceSubset(bbRequiredByAnyCell, ril_);
}
// Send information about which cells are actually required back
@@ -571,32 +637,36 @@ Foam::InteractionLists<ParticleType>::InteractionLists
// and truncate it
{
- labelList oldToNew(globalIAndTToExchange.size(), -1);
+ // labelList oldToNew(globalIAndTToExchange.size(), -1);
- label refCellI = 0;
+ // label refCellI = 0;
- forAll(bbRequiredByAnyCell, bbReqI)
- {
- if (bbRequiredByAnyCell[bbReqI])
- {
- oldToNew[bbReqI] = refCellI++;
- }
- }
+ // forAll(bbRequiredByAnyCell, bbReqI)
+ // {
+ // if (bbRequiredByAnyCell[bbReqI])
+ // {
+ // oldToNew[bbReqI] = refCellI++;
+ // }
+ // }
- inplaceReorder
- (
- oldToNew,
- static_cast<List<labelPair>&>(globalIAndTToExchange)
- );
+ // inplaceReorder
+ // (
+ // oldToNew,
+ // globalIAndTToExchange
+ // );
- inplaceReorder
- (
- oldToNew,
- static_cast<List<label>&>(procToDistributeTo)
- );
+ // inplaceReorder
+ // (
+ // oldToNew,
+ // procToDistributeTo
+ // );
+
+ // globalIAndTToExchange.setSize(refCellI);
+ // procToDistributeTo.setSize(refCellI);
- globalIAndTToExchange.setSize(refCellI);
- procToDistributeTo.setSize(refCellI);
+ inplaceSubset(bbRequiredByAnyCell, globalIAndTToExchange);
+
+ inplaceSubset(bbRequiredByAnyCell, procToDistributeTo);
}
preDistributionSize = procToDistributeTo.size();
@@ -699,62 +769,73 @@ Foam::InteractionLists<ParticleType>::~InteractionLists()
// * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * * //
template<class ParticleType>
-void Foam::InteractionLists<ParticleType>::prepareParticlesToRefer
+void Foam::InteractionLists<ParticleType>::sendReferredParticles
(
const List<DynamicList<ParticleType*> >& cellOccupancy
)
{
- // Clear all existing referred particles
+ prepareParticlesToRefer(cellOccupancy);
- forAll(referredParticles_, i)
- {
- referredParticles_[i].clear();
- }
-
- referredParticles_.setSize(cellIndexAndTransformToDistribute_.size());
+ PstreamBuffers pBufs(Pstream::nonBlocking);
- forAll(cellIndexAndTransformToDistribute_, i)
+ // Stream data into buffer
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
- const labelPair giat = cellIndexAndTransformToDistribute_[i];
-
- label cellIndex = globalTransforms_.index(giat);
+ const labelList& subMap = map().subMap()[domain];
- List<ParticleType*> realParticles = cellOccupancy[cellIndex];
-
- IDLList<ParticleType>& particlesToRefer = referredParticles_[i];
-
- forAll (realParticles, rM)
+ if (subMap.size())
{
- const ParticleType& particle = *realParticles[rM];
+ // Put data into send buffer
+ UOPstream toDomain(domain, pBufs);
- particlesToRefer.append(particle.clone().ptr());
+ UIndirectList<IDLList<ParticleType> > subMappedParticles
+ (
+ referredParticles_,
+ subMap
+ );
- prepareParticleToBeReferred(particlesToRefer.last(), giat);
+ forAll(subMappedParticles, i)
+ {
+ toDomain << subMappedParticles[i];
+ }
}
}
- // map().distribute(referredParticles_);
+ // Start sending and receiving but do not block.
+ pBufs.finishedSends(false);
- bool writeCloud = true;
+ // DO OTHER STUFF HERE;
- if (mesh_.time().outputTime() && writeCloud)
+ Pstream::waitRequests();
+
+ referredParticles_.setSize(map().constructSize());
+
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
- cloud_.clear();
+ const labelList& constructMap = map().constructMap()[domain];
- forAll(referredParticles_, refCellI)
+ if (constructMap.size())
{
- const IDLList<ParticleType>& refCell = referredParticles_[refCellI];
+ UIPstream str(domain, pBufs);
- forAllConstIter(typename IDLList<ParticleType>, refCell, iter)
+ forAll (constructMap, i)
{
- cloud_.addParticle(iter().clone().ptr());
+ referredParticles_[constructMap[i]] = IDLList<ParticleType>
+ (
+ str,
+ typename ParticleType::iNew(cloud_)
+ );
}
}
+ }
- Particle<ParticleType>::writeFields(cloud_);
+ writeReferredParticleCloud();
+};
- cloud_.clear();
- }
+
+template<class ParticleType>
+void Foam::InteractionLists<ParticleType>::receiveReferredParticles()
+{
}
diff --git a/src/lagrangian/basic/InteractionLists/InteractionLists.H b/src/lagrangian/basic/InteractionLists/InteractionLists.H
index cd341116ac6..7803c797fd4 100644
--- a/src/lagrangian/basic/InteractionLists/InteractionLists.H
+++ b/src/lagrangian/basic/InteractionLists/InteractionLists.H
@@ -112,6 +112,13 @@ class InteractionLists
//- Build the mapDistribute
void buildMap(const List<label>& toProc);
+ //- Fill the referredParticles container with particles that
+ // will be referred
+ void prepareParticlesToRefer
+ (
+ const List<DynamicList<ParticleType*> >& cellOccupancy
+ );
+
//- Prepare particle to be referred
void prepareParticleToBeReferred
(
@@ -119,6 +126,9 @@ class InteractionLists
labelPair giat
);
+ //- Write the referredParticles out to visualise
+ void writeReferredParticleCloud();
+
//- Disallow default bitwise copy construct
InteractionLists(const InteractionLists&);
@@ -144,13 +154,15 @@ public:
// Member Functions
- //- Fill the referredParticles container with particles that
- // will be referred
- void prepareParticlesToRefer
+ //- Prepare and send referred particles, nonBlocking communication
+ void sendReferredParticles
(
const List<DynamicList<ParticleType*> >& cellOccupancy
);
+ //- Receive referred particles
+ void receiveReferredParticles();
+
// Access
//- Return access to the mesh
--
GitLab