diff --git a/applications/test/extendedStencil/testExtendedStencil.C b/applications/test/extendedStencil/testExtendedStencil.C
index b8db2b89bf993552dd8582ba0c77c2cef58b121a..876576b29eab8666ae74c85e9801fb00c7f22324 100644
--- a/applications/test/extendedStencil/testExtendedStencil.C
+++ b/applications/test/extendedStencil/testExtendedStencil.C
@@ -34,7 +34,7 @@ Description
#include "fvMesh.H"
#include "volFields.H"
#include "Time.H"
-#include "mapDistribute.H"
+//#include "mapDistribute.H"
#include "OFstream.H"
#include "meshTools.H"
//#include "FECCellToFaceStencil.H"
diff --git a/applications/test/parallel/parallelTest.C b/applications/test/parallel/parallelTest.C
index bf8440ece62a6e1ac7d488ae8db434c6d08f90ae..6120de31a51670a1d2f2e71ea77004fa45613e61 100644
--- a/applications/test/parallel/parallelTest.C
+++ b/applications/test/parallel/parallelTest.C
@@ -23,19 +23,23 @@ License
Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Application
- icoFoam
+ parallelTest
Description
- Incompressible laminar CFD code.
+ Test for various parallel routines.
\*---------------------------------------------------------------------------*/
+#include "List.H"
+#include "mapDistribute.H"
#include "argList.H"
#include "Time.H"
#include "IPstream.H"
#include "OPstream.H"
#include "vector.H"
#include "IOstreams.H"
+#include "Random.H"
+#include "Tuple2.H"
using namespace Foam;
@@ -47,6 +51,99 @@ int main(int argc, char *argv[])
# include "setRootCase.H"
# include "createTime.H"
+
+ // Test mapDistribute
+ // ~~~~~~~~~~~~~~~~~~
+
+ if (false)
+ {
+ Random rndGen(43544*Pstream::myProcNo());
+
+ // Generate random data.
+ List<Tuple2<label, List<scalar> > > complexData(100);
+ forAll(complexData, i)
+ {
+ complexData[i].first() = rndGen.integer(0, Pstream::nProcs()-1);
+ complexData[i].second().setSize(3);
+ complexData[i].second()[0] = 1;
+ complexData[i].second()[1] = 2;
+ complexData[i].second()[2] = 3;
+ }
+
+ // Send all ones to processor indicated by .first()
+
+
+ // Count how many to send
+ labelList nSend(Pstream::nProcs(), 0);
+ forAll(complexData, i)
+ {
+ label procI = complexData[i].first();
+ nSend[procI]++;
+ }
+
+ // Sync how many to send
+ labelListList allNTrans(Pstream::nProcs());
+ allNTrans[Pstream::myProcNo()] = nSend;
+ combineReduce(allNTrans, mapDistribute::listEq());
+
+ // Collect items to be sent
+ labelListList sendMap(Pstream::nProcs());
+ forAll(sendMap, procI)
+ {
+ sendMap[procI].setSize(nSend[procI]);
+ }
+ nSend = 0;
+ forAll(complexData, i)
+ {
+ label procI = complexData[i].first();
+ sendMap[procI][nSend[procI]++] = i;
+ }
+
+ // Collect items to be received
+ labelListList recvMap(Pstream::nProcs());
+ forAll(recvMap, procI)
+ {
+ recvMap[procI].setSize(allNTrans[procI][Pstream::myProcNo()]);
+ }
+
+ label constructSize = 0;
+ // Construct with my own elements first
+ forAll(recvMap[Pstream::myProcNo()], i)
+ {
+ recvMap[Pstream::myProcNo()][i] = constructSize++;
+ }
+ // Construct from other processors
+ forAll(recvMap, procI)
+ {
+ if (procI != Pstream::myProcNo())
+ {
+ forAll(recvMap[procI], i)
+ {
+ recvMap[procI][i] = constructSize++;
+ }
+ }
+ }
+
+
+
+ // Construct distribute map (destructively)
+ mapDistribute map(constructSize, sendMap.xfer(), recvMap.xfer());
+
+ // Distribute complexData
+ mapDistribute::distribute
+ (
+ Pstream::nonBlocking,
+ List<labelPair>(),
+ map.constructSize(),
+ map.subMap(),
+ map.constructMap(),
+ complexData
+ );
+
+ Pout<< "complexData:" << complexData << endl;
+ }
+
+
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
Perr<< "\nStarting transfers\n" << endl;
@@ -60,13 +157,13 @@ int main(int argc, char *argv[])
{
Perr<< "slave sending to master "
<< Pstream::masterNo() << endl;
- OPstream toMaster(Pstream::masterNo());
+ OPstream toMaster(Pstream::blocking, Pstream::masterNo());
toMaster << data;
}
Perr<< "slave receiving from master "
<< Pstream::masterNo() << endl;
- IPstream fromMaster(Pstream::masterNo());
+ IPstream fromMaster(Pstream::blocking, Pstream::masterNo());
fromMaster >> data;
Perr<< data << endl;
@@ -81,7 +178,7 @@ int main(int argc, char *argv[])
)
{
Perr << "master receiving from slave " << slave << endl;
- IPstream fromSlave(slave);
+ IPstream fromSlave(Pstream::blocking, slave);
fromSlave >> data;
Perr<< data << endl;
@@ -95,7 +192,7 @@ int main(int argc, char *argv[])
)
{
Perr << "master sending to slave " << slave << endl;
- OPstream toSlave(slave);
+ OPstream toSlave(Pstream::blocking, slave);
toSlave << data;
}
}
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
index 1bbeb3d077deede92a108a524491dc1b1ff709cf..e2c2dbf773dae044c2af38cea5c8317a0f600c2f 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.C
@@ -172,8 +172,8 @@ const Foam::List<Foam::labelPair>& Foam::mapDistribute::schedule() const
Foam::mapDistribute::mapDistribute
(
const label constructSize,
- const labelListList& subMap,
- const labelListList& constructMap
+ const Xfer<labelListList>& subMap,
+ const Xfer<labelListList>& constructMap
)
:
constructSize_(constructSize),
@@ -183,22 +183,6 @@ Foam::mapDistribute::mapDistribute
{}
-//- (optionally destructively) construct from components
-Foam::mapDistribute::mapDistribute
-(
- const label constructSize,
- labelListList& subMap,
- labelListList& constructMap,
- const bool reUse // clone or reuse
-)
-:
- constructSize_(constructSize),
- subMap_(subMap, reUse),
- constructMap_(constructMap, reUse),
- schedulePtr_()
-{}
-
-
Foam::mapDistribute::mapDistribute
(
const labelList& sendProcs,
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
index d2cfe64ca53c0dde02c3d424d5236b415728151d..c6962355fdd5c24ce58ce70dcc54b557a72a0fe0 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistribute.H
@@ -83,23 +83,36 @@ class mapDistribute
public:
+ // Public classes
+
+ //- combineReduce operator for lists. Used for counting.
+ class listEq
+ {
+
+ public:
+
+ template<class T>
+ void operator()(T& x, const T& y) const
+ {
+ forAll(y, i)
+ {
+ if (y[i].size())
+ {
+ x[i] = y[i];
+ }
+ }
+ }
+ };
+
+
// Constructors
//- Construct from components
mapDistribute
(
const label constructSize,
- const labelListList& subMap,
- const labelListList& constructMap
- );
-
- //- (optionally destructively) construct from components
- mapDistribute
- (
- const label constructSize,
- labelListList& subMap,
- labelListList& constructMap,
- const bool reUse // clone or reuse
+ const Xfer<labelListList>& subMap,
+ const Xfer<labelListList>& constructMap
);
//- Construct from reverse addressing: per data item the send
@@ -205,11 +218,7 @@ public:
template<class T>
void distribute(List<T>& fld) const
{
- if
- (
- Pstream::defaultCommsType == Pstream::nonBlocking
- && contiguous<T>()
- )
+ if (Pstream::defaultCommsType == Pstream::nonBlocking)
{
distribute
(
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeLagrangian.H b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeLagrangian.H
index 5782308d196e59cdabcae247fcc6922a5d8721ec..ce0567390827c8bb12d38b164d8b46cba9b5888d 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeLagrangian.H
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeLagrangian.H
@@ -68,35 +68,15 @@ public:
mapDistributeLagrangian
(
const label nNewParticles,
- const labelListList& subParticleMap,
- const labelListList& constructParticleMap,
- const labelListList& constructCellLabels
+ const Xfer<labelListList>& subParticleMap,
+ const Xfer<labelListList>& constructParticleMap,
+ const Xfer<labelListList>& constructCellLabels
)
:
particleMap_(nNewParticles, subParticleMap, constructParticleMap),
constructCellLabels_(constructCellLabels)
{}
- //- Construct from components and steal storage
- mapDistributeLagrangian
- (
- const label nNewParticles,
- labelListList& subParticleMap,
- labelListList& constructParticleMap,
- labelListList& constructCellLabels,
- const bool reUse
- )
- :
- particleMap_
- (
- nNewParticles,
- subParticleMap,
- constructParticleMap,
- reUse
- ),
- constructCellLabels_(constructCellLabels, reUse)
- {}
-
// Member Functions
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMesh.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMesh.C
index a5e4f163417ca44e664898569450e5836c2154ec..29649665b4b1473181013802b315d43aa9d8ee6c 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMesh.C
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMesh.C
@@ -66,27 +66,27 @@ Foam::mapDistributePolyMesh::mapDistributePolyMesh
const label nOldPoints,
const label nOldFaces,
const label nOldCells,
- const labelList& oldPatchStarts,
- const labelList& oldPatchNMeshPoints,
+ const Xfer<labelList>& oldPatchStarts,
+ const Xfer<labelList>& oldPatchNMeshPoints,
// how to subset pieces of mesh to send across
- const labelListList& subPointMap,
- const labelListList& subFaceMap,
- const labelListList& subCellMap,
- const labelListList& subPatchMap,
+ const Xfer<labelListList>& subPointMap,
+ const Xfer<labelListList>& subFaceMap,
+ const Xfer<labelListList>& subCellMap,
+ const Xfer<labelListList>& subPatchMap,
// how to reconstruct received mesh
- const labelListList& constructPointMap,
- const labelListList& constructFaceMap,
- const labelListList& constructCellMap,
- const labelListList& constructPatchMap
+ const Xfer<labelListList>& constructPointMap,
+ const Xfer<labelListList>& constructFaceMap,
+ const Xfer<labelListList>& constructCellMap,
+ const Xfer<labelListList>& constructPatchMap
)
:
mesh_(mesh),
nOldPoints_(nOldPoints),
nOldFaces_(nOldFaces),
nOldCells_(nOldCells),
- oldPatchSizes_(oldPatchStarts.size()),
+ oldPatchSizes_(oldPatchStarts().size()),
oldPatchStarts_(oldPatchStarts),
oldPatchNMeshPoints_(oldPatchNMeshPoints),
pointMap_(mesh.nPoints(), subPointMap, constructPointMap),
@@ -98,44 +98,6 @@ Foam::mapDistributePolyMesh::mapDistributePolyMesh
}
-//- (optionally destructively) construct from components
-Foam::mapDistributePolyMesh::mapDistributePolyMesh
-(
- const polyMesh& mesh,
- const label nOldPoints,
- const label nOldFaces,
- const label nOldCells,
- labelList& oldPatchStarts,
- labelList& oldPatchNMeshPoints,
-
- labelListList& subPointMap,
- labelListList& subFaceMap,
- labelListList& subCellMap,
- labelListList& subPatchMap,
- labelListList& constructPointMap,
- labelListList& constructFaceMap,
- labelListList& constructCellMap,
- labelListList& constructPatchMap,
- const bool reUse // clone or reuse
-)
-:
- mesh_(mesh),
- nOldPoints_(nOldPoints),
- nOldFaces_(nOldFaces),
- nOldCells_(nOldCells),
- oldPatchSizes_(oldPatchStarts.size()),
- oldPatchStarts_(oldPatchStarts, reUse),
- oldPatchNMeshPoints_(oldPatchNMeshPoints, reUse),
-
- pointMap_(mesh.nPoints(), subPointMap, constructPointMap, reUse),
- faceMap_(mesh.nFaces(), subFaceMap, constructFaceMap, reUse),
- cellMap_(mesh.nCells(), subCellMap, constructCellMap, reUse),
- patchMap_(mesh.boundaryMesh().size(), subPatchMap, constructPatchMap, reUse)
-{
- calcPatchSizes();
-}
-
-
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::mapDistributePolyMesh::distributePointIndices(labelList& lst) const
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMesh.H b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMesh.H
index bf5857170b18f41035b7deb902851078b7301561..8fdfe7e8fb51afcd8bb8e8e88151567ed2628b15 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMesh.H
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributePolyMesh.H
@@ -120,42 +120,20 @@ public:
const label nOldPoints,
const label nOldFaces,
const label nOldCells,
- const labelList& oldPatchStarts,
- const labelList& oldPatchNMeshPoints,
+ const Xfer<labelList>& oldPatchStarts,
+ const Xfer<labelList>& oldPatchNMeshPoints,
// how to subset pieces of mesh to send across
- const labelListList& subPointMap,
- const labelListList& subFaceMap,
- const labelListList& subCellMap,
- const labelListList& subPatchMap,
+ const Xfer<labelListList>& subPointMap,
+ const Xfer<labelListList>& subFaceMap,
+ const Xfer<labelListList>& subCellMap,
+ const Xfer<labelListList>& subPatchMap,
// how to reconstruct received mesh
- const labelListList& constructPointMap,
- const labelListList& constructFaceMap,
- const labelListList& constructCellMap,
- const labelListList& constructPatchMap
- );
-
- //- (optionally destructively) construct from components
- // Note that mesh has to be changed already!
- mapDistributePolyMesh
- (
- const polyMesh& mesh,
- const label nOldPoints,
- const label nOldFaces,
- const label nOldCells,
- labelList& oldPatchStarts,
- labelList& oldPatchNMeshPoints,
-
- labelListList& subPointMap,
- labelListList& subFaceMap,
- labelListList& subCellMap,
- labelListList& subPatchMap,
- labelListList& constructPointMap,
- labelListList& constructFaceMap,
- labelListList& constructCellMap,
- labelListList& constructPatchMap,
- const bool reUse // clone or reuse
+ const Xfer<labelListList>& constructPointMap,
+ const Xfer<labelListList>& constructFaceMap,
+ const Xfer<labelListList>& constructCellMap,
+ const Xfer<labelListList>& constructPatchMap
);
diff --git a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
index 5f5d7a9fcf9793fe4f0c0df04e8c60e87e9c6d4f..da1ed19d1e6365c65fa40794ecec64186a579915 100644
--- a/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
+++ b/src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeTemplates.C
@@ -25,6 +25,7 @@ License
\*---------------------------------------------------------------------------*/
#include "Pstream.H"
+#include "PstreamCombineReduceOps.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
@@ -184,138 +185,269 @@ void Foam::mapDistribute::distribute
{
if (!contiguous<T>())
{
- FatalErrorIn
- (
- "template<class T>\n"
- "void mapDistribute::distribute\n"
- "(\n"
- " const Pstream::commsTypes commsType,\n"
- " const List<labelPair>& schedule,\n"
- " const label constructSize,\n"
- " const labelListList& subMap,\n"
- " const labelListList& constructMap,\n"
- " List<T>& field\n"
- ")\n"
- ) << "Non-blocking only supported for contiguous data."
- << exit(FatalError);
- }
+ // 1. convert to contiguous buffer
+ // 2. send buffer
+ // 3. receive buffer
+ // 4. read from buffer into List<T>
+
+ List<List<char> > sendFields(Pstream::nProcs());
+ labelListList allNTrans(Pstream::nProcs());
+ labelList& nsTransPs = allNTrans[Pstream::myProcNo()];
+ nsTransPs.setSize(Pstream::nProcs(), 0);
+
+ // Stream data into sendField buffers
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
+ {
+ const labelList& map = subMap[domain];
- // Set up sends to neighbours
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ // Put data into send buffer
+ OPstream toDomain(Pstream::nonBlocking, domain);
+ toDomain << UIndirectList<T>(field, map);
- List<List<T > > sendFields(Pstream::nProcs());
+ // Store the size
+ nsTransPs[domain] = toDomain.bufPosition();
- for (label domain = 0; domain < Pstream::nProcs(); domain++)
- {
- const labelList& map = subMap[domain];
+ // Transfer buffer out
+ sendFields[domain].transfer(toDomain.buf());
+ toDomain.bufPosition() = 0;
- if (domain != Pstream::myProcNo() && map.size())
+ }
+ }
+
+ // Send sizes across
+ combineReduce(allNTrans, listEq());
+
+ // Start sending buffers
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
- List<T>& subField = sendFields[domain];
- subField.setSize(map.size());
- forAll(map, i)
+ const labelList& map = subMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
{
- subField[i] = field[map[i]];
+ OPstream::write
+ (
+ Pstream::nonBlocking,
+ domain,
+ reinterpret_cast<const char*>
+ (
+ sendFields[domain].begin()
+ ),
+ nsTransPs[domain]
+ );
}
-
- OPstream::write
- (
- Pstream::nonBlocking,
- domain,
- reinterpret_cast<const char*>(subField.begin()),
- subField.size()*sizeof(T)
- );
}
- }
- // Set up receives from neighbours
+ // Set up receives from neighbours
- List<List<T > > recvFields(Pstream::nProcs());
+ PtrList<IPstream> fromSlave(Pstream::nProcs());
- for (label domain = 0; domain < Pstream::nProcs(); domain++)
- {
- const labelList& map = constructMap[domain];
-
- if (domain != Pstream::myProcNo() && map.size())
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
- recvFields[domain].setSize(map.size());
- IPstream::read
- (
- Pstream::nonBlocking,
- domain,
- reinterpret_cast<char*>(recvFields[domain].begin()),
- recvFields[domain].size()*sizeof(T)
- );
+ const labelList& map = constructMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ // Start receiving
+ fromSlave.set
+ (
+ domain,
+ new IPstream
+ (
+ Pstream::nonBlocking,
+ domain,
+ allNTrans[domain][Pstream::myProcNo()]
+ )
+ );
+ }
}
- }
- // Set up 'send' to myself
+ {
+ // Set up 'send' to myself
+ const labelList& mySubMap = subMap[Pstream::myProcNo()];
+ List<T> mySubField(mySubMap.size());
+ forAll(mySubMap, i)
+ {
+ mySubField[i] = field[mySubMap[i]];
+ }
+ // Combine bits. Note that can reuse field storage
+ field.setSize(constructSize);
+ // Receive sub field from myself
+ {
+ const labelList& map = constructMap[Pstream::myProcNo()];
- {
- const labelList& map = subMap[Pstream::myProcNo()];
+ forAll(map, i)
+ {
+ field[map[i]] = mySubField[i];
+ }
+ }
+ }
- List<T>& subField = sendFields[Pstream::myProcNo()];
- subField.setSize(map.size());
- forAll(map, i)
+
+ // Wait till all finished
+ IPstream::waitRequests();
+ OPstream::waitRequests();
+
+ // Consume
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
- subField[i] = field[map[i]];
+ const labelList& map = constructMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ List<T> recvField(fromSlave[domain]);
+
+ if (recvField.size() != map.size())
+ {
+ FatalErrorIn
+ (
+ "template<class T>\n"
+ "void mapDistribute::distribute\n"
+ "(\n"
+ " const Pstream::commsTypes commsType,\n"
+ " const List<labelPair>& schedule,\n"
+ " const label constructSize,\n"
+ " const labelListList& subMap,\n"
+ " const labelListList& constructMap,\n"
+ " List<T>& field\n"
+ ")\n"
+ ) << "Expected from processor " << domain
+ << " " << map.size() << " but received "
+ << recvField.size() << " elements."
+ << abort(FatalError);
+ }
+
+ forAll(map, i)
+ {
+ field[map[i]] = recvField[i];
+ }
+
+ // Delete receive buffer
+ fromSlave.set(domain, NULL);
+ }
}
}
+ else
+ {
+ // Set up sends to neighbours
+ List<List<T > > sendFields(Pstream::nProcs());
- // Combine bits. Note that can reuse field storage
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
+ {
+ const labelList& map = subMap[domain];
- field.setSize(constructSize);
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ List<T>& subField = sendFields[domain];
+ subField.setSize(map.size());
+ forAll(map, i)
+ {
+ subField[i] = field[map[i]];
+ }
+
+ OPstream::write
+ (
+ Pstream::nonBlocking,
+ domain,
+ reinterpret_cast<const char*>(subField.begin()),
+ subField.byteSize()
+ );
+ }
+ }
+ // Set up receives from neighbours
- // Receive sub field from myself (sendFields[Pstream::myProcNo()])
- {
- const labelList& map = constructMap[Pstream::myProcNo()];
- const List<T>& subField = sendFields[Pstream::myProcNo()];
+ List<List<T > > recvFields(Pstream::nProcs());
- forAll(map, i)
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
- field[map[i]] = subField[i];
+ const labelList& map = constructMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ recvFields[domain].setSize(map.size());
+ IPstream::read
+ (
+ Pstream::nonBlocking,
+ domain,
+ reinterpret_cast<char*>(recvFields[domain].begin()),
+ recvFields[domain].byteSize()
+ );
+ }
}
- }
- // Wait for all to finish
+ // Set up 'send' to myself
- OPstream::waitRequests();
- IPstream::waitRequests();
+ {
+ const labelList& map = subMap[Pstream::myProcNo()];
- // Collect neighbour fields
+ List<T>& subField = sendFields[Pstream::myProcNo()];
+ subField.setSize(map.size());
+ forAll(map, i)
+ {
+ subField[i] = field[map[i]];
+ }
+ }
- for (label domain = 0; domain < Pstream::nProcs(); domain++)
- {
- const labelList& map = constructMap[domain];
- if (domain != Pstream::myProcNo() && map.size())
+ // Combine bits. Note that can reuse field storage
+
+ field.setSize(constructSize);
+
+
+ // Receive sub field from myself (sendFields[Pstream::myProcNo()])
{
- if (recvFields[domain].size() != map.size())
+ const labelList& map = constructMap[Pstream::myProcNo()];
+ const List<T>& subField = sendFields[Pstream::myProcNo()];
+
+ forAll(map, i)
{
- FatalErrorIn
- (
- "template<class T>\n"
- "void mapDistribute::distribute\n"
- "(\n"
- " const Pstream::commsTypes commsType,\n"
- " const List<labelPair>& schedule,\n"
- " const label constructSize,\n"
- " const labelListList& subMap,\n"
- " const labelListList& constructMap,\n"
- " List<T>& field\n"
- ")\n"
- ) << "Expected from processor " << domain
- << " " << map.size() << " but received "
- << recvFields[domain].size() << " elements."
- << abort(FatalError);
+ field[map[i]] = subField[i];
}
+ }
- forAll(map, i)
+
+ // Wait for all to finish
+
+ OPstream::waitRequests();
+ IPstream::waitRequests();
+
+ // Collect neighbour fields
+
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
+ {
+ const labelList& map = constructMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
{
- field[map[i]] = recvFields[domain][i];
+ if (recvFields[domain].size() != map.size())
+ {
+ FatalErrorIn
+ (
+ "template<class T>\n"
+ "void mapDistribute::distribute\n"
+ "(\n"
+ " const Pstream::commsTypes commsType,\n"
+ " const List<labelPair>& schedule,\n"
+ " const label constructSize,\n"
+ " const labelListList& subMap,\n"
+ " const labelListList& constructMap,\n"
+ " List<T>& field\n"
+ ")\n"
+ ) << "Expected from processor " << domain
+ << " " << map.size() << " but received "
+ << recvFields[domain].size() << " elements."
+ << abort(FatalError);
+ }
+
+ forAll(map, i)
+ {
+ field[map[i]] = recvFields[domain][i];
+ }
}
}
}
@@ -488,137 +620,263 @@ void Foam::mapDistribute::distribute
{
if (!contiguous<T>())
{
- FatalErrorIn
- (
- "template<class T>\n"
- "void mapDistribute::distribute\n"
- "(\n"
- " const Pstream::commsTypes commsType,\n"
- " const List<labelPair>& schedule,\n"
- " const label constructSize,\n"
- " const labelListList& subMap,\n"
- " const labelListList& constructMap,\n"
- " List<T>& field\n"
- ")\n"
- ) << "Non-blocking only supported for contiguous data."
- << exit(FatalError);
- }
+ // 1. convert to contiguous buffer
+ // 2. send buffer
+ // 3. receive buffer
+ // 4. read from buffer into List<T>
+
+ List<List<char> > sendFields(Pstream::nProcs());
+ labelListList allNTrans(Pstream::nProcs());
+ labelList& nsTransPs = allNTrans[Pstream::myProcNo()];
+ nsTransPs.setSize(Pstream::nProcs());
+
+ // Stream data into sendField buffers
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
+ {
+ const labelList& map = subMap[domain];
- // Set up sends to neighbours
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ // Put data into send buffer
+ OPstream toDomain(Pstream::nonBlocking, domain);
+ toDomain << UIndirectList<T>(field, map);
- List<List<T > > sendFields(Pstream::nProcs());
+ // Store the size
+ nsTransPs[domain] = toDomain.bufPosition();
- for (label domain = 0; domain < Pstream::nProcs(); domain++)
- {
- const labelList& map = subMap[domain];
+ // Transfer buffer out
+ sendFields[domain].transfer(toDomain.buf());
+ toDomain.bufPosition() = 0;
+ }
+ }
- if (domain != Pstream::myProcNo() && map.size())
+ // Send sizes across
+ combineReduce(allNTrans, listEq());
+
+ // Start sending buffers
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
- List<T>& subField = sendFields[domain];
- subField.setSize(map.size());
- forAll(map, i)
+ const labelList& map = subMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
{
- subField[i] = field[map[i]];
+ OPstream::write
+ (
+ Pstream::nonBlocking,
+ domain,
+ reinterpret_cast<const char*>
+ (
+ sendFields[domain].begin()
+ ),
+ nsTransPs[domain]
+ );
}
-
- OPstream::write
- (
- Pstream::nonBlocking,
- domain,
- reinterpret_cast<const char*>(subField.begin()),
- subField.size()*sizeof(T)
- );
}
- }
- // Set up receives from neighbours
+ // Set up receives from neighbours
- List<List<T > > recvFields(Pstream::nProcs());
+ PtrList<IPstream> fromSlave(Pstream::nProcs());
+
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
+ {
+ const labelList& map = constructMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ // Start receiving
+ fromSlave.set
+ (
+ domain,
+ new IPstream
+ (
+ Pstream::nonBlocking,
+ domain,
+ allNTrans[domain][Pstream::myProcNo()]
+ )
+ );
+ }
+ }
- for (label domain = 0; domain < Pstream::nProcs(); domain++)
- {
- const labelList& map = constructMap[domain];
- if (domain != Pstream::myProcNo() && map.size())
{
- recvFields[domain].setSize(map.size());
- IPstream::read
- (
- Pstream::nonBlocking,
- domain,
- reinterpret_cast<char*>(recvFields[domain].begin()),
- recvFields[domain].size()*sizeof(T)
- );
+ // Set up 'send' to myself
+ List<T> mySubField(field, subMap[Pstream::myProcNo()]);
+ // Combine bits. Note that can reuse field storage
+ field.setSize(constructSize);
+ field = nullValue;
+ // Receive sub field from myself
+ {
+ const labelList& map = constructMap[Pstream::myProcNo()];
+
+ forAll(map, i)
+ {
+ cop(field[map[i]], mySubField[i]);
+ }
+ }
}
- }
- // Set up 'send' to myself
- {
- const labelList& map = subMap[Pstream::myProcNo()];
+ // Wait till all finished
+ IPstream::waitRequests();
+ OPstream::waitRequests();
- List<T>& subField = sendFields[Pstream::myProcNo()];
- subField.setSize(map.size());
- forAll(map, i)
+ // Consume
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
- subField[i] = field[map[i]];
+ const labelList& map = constructMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ List<T> recvField(fromSlave[domain]);
+
+ if (recvField.size() != map.size())
+ {
+ FatalErrorIn
+ (
+ "template<class T>\n"
+ "void mapDistribute::distribute\n"
+ "(\n"
+ " const Pstream::commsTypes commsType,\n"
+ " const List<labelPair>& schedule,\n"
+ " const label constructSize,\n"
+ " const labelListList& subMap,\n"
+ " const labelListList& constructMap,\n"
+ " List<T>& field\n"
+ ")\n"
+ ) << "Expected from processor " << domain
+ << " " << map.size() << " but received "
+ << recvField.size() << " elements."
+ << abort(FatalError);
+ }
+
+ forAll(map, i)
+ {
+ cop(field[map[i]], recvField[i]);
+ }
+
+ // Delete receive buffer
+ fromSlave.set(domain, NULL);
+ }
}
}
+ else
+ {
+ // Set up sends to neighbours
+ List<List<T > > sendFields(Pstream::nProcs());
- // Combine bits. Note that can reuse field storage
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
+ {
+ const labelList& map = subMap[domain];
- field.setSize(constructSize);
- field = nullValue;
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ List<T>& subField = sendFields[domain];
+ subField.setSize(map.size());
+ forAll(map, i)
+ {
+ subField[i] = field[map[i]];
+ }
+
+ OPstream::write
+ (
+ Pstream::nonBlocking,
+ domain,
+ reinterpret_cast<const char*>(subField.begin()),
+ subField.size()*sizeof(T)
+ );
+ }
+ }
- // Receive sub field from myself (subField)
- {
- const labelList& map = constructMap[Pstream::myProcNo()];
- const List<T>& subField = sendFields[Pstream::myProcNo()];
+ // Set up receives from neighbours
+
+ List<List<T > > recvFields(Pstream::nProcs());
- forAll(map, i)
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
{
- cop(field[map[i]], subField[i]);
+ const labelList& map = constructMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
+ {
+ recvFields[domain].setSize(map.size());
+ IPstream::read
+ (
+ Pstream::nonBlocking,
+ domain,
+ reinterpret_cast<char*>(recvFields[domain].begin()),
+ recvFields[domain].size()*sizeof(T)
+ );
+ }
}
- }
+ // Set up 'send' to myself
- // Wait for all to finish
+ {
+ const labelList& map = subMap[Pstream::myProcNo()];
- OPstream::waitRequests();
- IPstream::waitRequests();
+ List<T>& subField = sendFields[Pstream::myProcNo()];
+ subField.setSize(map.size());
+ forAll(map, i)
+ {
+ subField[i] = field[map[i]];
+ }
+ }
- // Collect neighbour fields
- for (label domain = 0; domain < Pstream::nProcs(); domain++)
- {
- const labelList& map = constructMap[domain];
+ // Combine bits. Note that can reuse field storage
- if (domain != Pstream::myProcNo() && map.size())
+ field.setSize(constructSize);
+ field = nullValue;
+
+ // Receive sub field from myself (subField)
{
- if (recvFields[domain].size() != map.size())
+ const labelList& map = constructMap[Pstream::myProcNo()];
+ const List<T>& subField = sendFields[Pstream::myProcNo()];
+
+ forAll(map, i)
{
- FatalErrorIn
- (
- "template<class T>\n"
- "void mapDistribute::distribute\n"
- "(\n"
- " const Pstream::commsTypes commsType,\n"
- " const List<labelPair>& schedule,\n"
- " const label constructSize,\n"
- " const labelListList& subMap,\n"
- " const labelListList& constructMap,\n"
- " List<T>& field\n"
- ")\n"
- ) << "Expected from processor " << domain
- << " " << map.size() << " but received "
- << recvFields[domain].size() << " elements."
- << abort(FatalError);
+ cop(field[map[i]], subField[i]);
}
+ }
- forAll(map, i)
+
+ // Wait for all to finish
+
+ OPstream::waitRequests();
+ IPstream::waitRequests();
+
+ // Collect neighbour fields
+
+ for (label domain = 0; domain < Pstream::nProcs(); domain++)
+ {
+ const labelList& map = constructMap[domain];
+
+ if (domain != Pstream::myProcNo() && map.size())
{
- cop(field[map[i]], recvFields[domain][i]);
+ if (recvFields[domain].size() != map.size())
+ {
+ FatalErrorIn
+ (
+ "template<class T>\n"
+ "void mapDistribute::distribute\n"
+ "(\n"
+ " const Pstream::commsTypes commsType,\n"
+ " const List<labelPair>& schedule,\n"
+ " const label constructSize,\n"
+ " const labelListList& subMap,\n"
+ " const labelListList& constructMap,\n"
+ " List<T>& field\n"
+ ")\n"
+ ) << "Expected from processor " << domain
+ << " " << map.size() << " but received "
+ << recvFields[domain].size() << " elements."
+ << abort(FatalError);
+ }
+
+ forAll(map, i)
+ {
+ cop(field[map[i]], recvFields[domain][i]);
+ }
}
}
}
diff --git a/src/autoMesh/autoHexMesh/autoHexMeshDriver/autoHexMeshDriver.C b/src/autoMesh/autoHexMesh/autoHexMeshDriver/autoHexMeshDriver.C
index 1612a23deac026c60ab56110d5061e49a5518e48..cc110c1d406acb0eaf63e2eab720a815edb656d6 100644
--- a/src/autoMesh/autoHexMesh/autoHexMeshDriver/autoHexMeshDriver.C
+++ b/src/autoMesh/autoHexMesh/autoHexMeshDriver/autoHexMeshDriver.C
@@ -28,12 +28,9 @@ License
#include "fvMesh.H"
#include "Time.H"
#include "boundBox.H"
-#include "globalIndex.H"
#include "wallPolyPatch.H"
-#include "mapDistributePolyMesh.H"
#include "cellSet.H"
#include "syncTools.H"
-#include "motionSmoother.H"
#include "refinementParameters.H"
#include "snapParameters.H"
#include "layerParameters.H"
diff --git a/src/autoMesh/autoHexMesh/autoHexMeshDriver/autoRefineDriver.C b/src/autoMesh/autoHexMesh/autoHexMeshDriver/autoRefineDriver.C
index a9b31b9bd1160800255fa6311b991083a56fcf6d..68d4b3b1e6a68e1f67d29e06f94e54f5db2378ca 100644
--- a/src/autoMesh/autoHexMesh/autoHexMeshDriver/autoRefineDriver.C
+++ b/src/autoMesh/autoHexMesh/autoHexMeshDriver/autoRefineDriver.C
@@ -28,8 +28,6 @@ License
#include "meshRefinement.H"
#include "fvMesh.H"
#include "Time.H"
-#include "boundBox.H"
-#include "mapDistributePolyMesh.H"
#include "cellSet.H"
#include "syncTools.H"
#include "refinementParameters.H"
diff --git a/src/dynamicMesh/fvMeshDistribute/fvMeshDistribute.C b/src/dynamicMesh/fvMeshDistribute/fvMeshDistribute.C
index 633c39f4782b13cd273ae0ccd2eaeeb76110f326..f815b46d718c988834f0111b20eee3ecd88a1cec 100644
--- a/src/dynamicMesh/fvMeshDistribute/fvMeshDistribute.C
+++ b/src/dynamicMesh/fvMeshDistribute/fvMeshDistribute.C
@@ -1418,18 +1418,18 @@ Foam::autoPtr<Foam::mapDistributePolyMesh> Foam::fvMeshDistribute::distribute
nOldPoints,
nOldFaces,
nOldCells,
- oldPatchStarts,
- oldPatchNMeshPoints,
-
- labelListList(1, identity(mesh_.nPoints())),//subPointMap
- labelListList(1, identity(mesh_.nFaces())), //subFaceMap
- labelListList(1, identity(mesh_.nCells())), //subCellMap
- labelListList(1, identity(patches.size())), //subPatchMap
-
- labelListList(1, identity(mesh_.nPoints())),//constructPointMap
- labelListList(1, identity(mesh_.nFaces())), //constructFaceMap
- labelListList(1, identity(mesh_.nCells())), //constructCellMap
- labelListList(1, identity(patches.size())) //constructPatchMap
+ oldPatchStarts.xfer(),
+ oldPatchNMeshPoints.xfer(),
+
+ labelListList(1, identity(mesh_.nPoints())).xfer(),//subPointMap
+ labelListList(1, identity(mesh_.nFaces())).xfer(), //subFaceMap
+ labelListList(1, identity(mesh_.nCells())).xfer(), //subCellMap
+ labelListList(1, identity(patches.size())).xfer(), //subPatchMap
+
+ labelListList(1, identity(mesh_.nPoints())).xfer(),//pointMap
+ labelListList(1, identity(mesh_.nFaces())).xfer(), //faceMap
+ labelListList(1, identity(mesh_.nCells())).xfer(), //cellMap
+ labelListList(1, identity(patches.size())).xfer() //patchMap
)
);
}
@@ -2207,19 +2207,18 @@ Foam::autoPtr<Foam::mapDistributePolyMesh> Foam::fvMeshDistribute::distribute
nOldPoints,
nOldFaces,
nOldCells,
- oldPatchStarts,
- oldPatchNMeshPoints,
-
- subPointMap,
- subFaceMap,
- subCellMap,
- subPatchMap,
-
- constructPointMap,
- constructFaceMap,
- constructCellMap,
- constructPatchMap,
- true // reuse storage
+ oldPatchStarts.xfer(),
+ oldPatchNMeshPoints.xfer(),
+
+ subPointMap.xfer(),
+ subFaceMap.xfer(),
+ subCellMap.xfer(),
+ subPatchMap.xfer(),
+
+ constructPointMap.xfer(),
+ constructFaceMap.xfer(),
+ constructCellMap.xfer(),
+ constructPatchMap.xfer()
)
);
}
diff --git a/src/finiteVolume/fvMesh/extendedStencil/cellToFace/extendedCellToFaceStencil.C b/src/finiteVolume/fvMesh/extendedStencil/cellToFace/extendedCellToFaceStencil.C
index 5b1cb12e342a15f38eaa1caa5f9b3da4787197c9..8a7224288ce94f4b0fd49b515340855c31a9f7cb 100644
--- a/src/finiteVolume/fvMesh/extendedStencil/cellToFace/extendedCellToFaceStencil.C
+++ b/src/finiteVolume/fvMesh/extendedStencil/cellToFace/extendedCellToFaceStencil.C
@@ -280,9 +280,8 @@ Foam::extendedCellToFaceStencil::calcDistributeMap
new mapDistribute
(
nCompact,
- sendCompact,
- recvCompact,
- true // reuse send/recv maps.
+ sendCompact.xfer(),
+ recvCompact.xfer()
)
);
diff --git a/src/meshTools/searchableSurface/distributedTriSurfaceMesh.C b/src/meshTools/searchableSurface/distributedTriSurfaceMesh.C
index 7f86c0f6581fcd94504b9cb7d39fdc67284a4506..33b6b6f1f55e71a87e4a213db2253556f3f740a2 100644
--- a/src/meshTools/searchableSurface/distributedTriSurfaceMesh.C
+++ b/src/meshTools/searchableSurface/distributedTriSurfaceMesh.C
@@ -340,9 +340,8 @@ Foam::distributedTriSurfaceMesh::distributeSegments
new mapDistribute
(
segmentI, // size after construction
- sendMap,
- constructMap,
- true // reuse storage
+ sendMap.xfer(),
+ constructMap.xfer()
)
);
}
@@ -642,9 +641,8 @@ Foam::distributedTriSurfaceMesh::calcLocalQueries
new mapDistribute
(
segmentI, // size after construction
- sendMap,
- constructMap,
- true // reuse storage
+ sendMap.xfer(),
+ constructMap.xfer()
)
);
const mapDistribute& map = mapPtr();
@@ -806,9 +804,8 @@ Foam::distributedTriSurfaceMesh::calcLocalQueries
new mapDistribute
(
segmentI, // size after construction
- sendMap,
- constructMap,
- true // reuse storage
+ sendMap.xfer(),
+ constructMap.xfer()
)
);
return mapPtr;
@@ -2399,9 +2396,8 @@ void Foam::distributedTriSurfaceMesh::distribute
new mapDistribute
(
allTris.size(),
- faceSendMap,
- faceConstructMap,
- true
+ faceSendMap.xfer(),
+ faceConstructMap.xfer()
)
);
pointMap.reset
@@ -2409,9 +2405,8 @@ void Foam::distributedTriSurfaceMesh::distribute
new mapDistribute
(
allPoints.size(),
- pointSendMap,
- pointConstructMap,
- true
+ pointSendMap.xfer(),
+ pointConstructMap.xfer()
)
);