diff --git a/src/OpenFOAM/Make/files b/src/OpenFOAM/Make/files
index 3dddd831904ddd57b2a71088ea8d009b9d40fad0..6d5f79331649cc12231616b31095d801db13448c 100644
--- a/src/OpenFOAM/Make/files
+++ b/src/OpenFOAM/Make/files
@@ -592,8 +592,6 @@ polyBoundaryMesh = $(polyMesh)/polyBoundaryMesh
$(polyBoundaryMesh)/polyBoundaryMesh.C
$(polyBoundaryMesh)/polyBoundaryMeshEntries.C
-meshes/ProcessorTopology/commSchedule.C
-
globalMeshData = $(polyMesh)/globalMeshData
$(globalMeshData)/globalMeshData.C
$(globalMeshData)/globalPoints.C
@@ -812,6 +810,7 @@ algorithms/indexedOctree/volumeType.C
algorithms/dynamicIndexedOctree/dynamicIndexedOctreeName.C
algorithms/dynamicIndexedOctree/dynamicTreeDataPoint.C
+parallel/commSchedule/commSchedule.C
parallel/globalIndex/globalIndex.C
meshes/data/data.C
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C
index 07e7eccc49ec21e9eb0bde115313b24449293d09..d7e4c3d87ed4c606761b59a41d228c7674a1db0a 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C
@@ -27,6 +27,7 @@ License
\*---------------------------------------------------------------------------*/
#include "PstreamBuffers.H"
+#include "bitSet.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
@@ -306,4 +307,72 @@ void Foam::PstreamBuffers::finishedSends
}
+bool Foam::PstreamBuffers::finishedSends
+(
+ bitSet& sendConnections,
+ DynamicList<label>& sendProcs,
+ DynamicList<label>& recvProcs,
+ const bool wait
+)
+{
+ bool changed = (sendConnections.size() != nProcs());
+
+ if (changed)
+ {
+ sendConnections.resize(nProcs());
+ }
+
+ // Update send connections
+ // - reasonable to assume there are no self-sends on UPstream::myProcNo
+ forAll(sendBuf_, proci)
+ {
+ // ie, hasSendData(proci)
+ if (sendConnections.set(proci, !sendBuf_[proci].empty()))
+ {
+ // The state changed
+ changed = true;
+ }
+ }
+
+ reduce(changed, orOp<bool>());
+
+ if (changed)
+ {
+ // Create send/recv topology
+
+ // The send ranks
+ sendProcs.clear();
+ forAll(sendBuf_, proci)
+ {
+ // ie, hasSendData(proci)
+ if (!sendBuf_[proci].empty())
+ {
+ sendProcs.append(proci);
+ }
+ }
+
+ finishedSends(wait); // All-to-all
+
+ // The recv ranks
+ recvProcs.clear();
+ forAll(recvBuf_, proci)
+ {
+ // ie, hasRecvData(proci)
+ if (!recvBuf_[proci].empty())
+ {
+ recvProcs.append(proci);
+ }
+ }
+ }
+ else
+ {
+ // Use existing send/recv ranks
+
+ finishedSends(sendProcs, recvProcs, wait);
+ }
+
+ return changed;
+}
+
+
// ************************************************************************* //
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H
index 2fec28c518f0970911707801f1c75f910d852d83..81666658e6a8985952f3c8f578235c6ff1bb8bcc 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H
@@ -82,6 +82,9 @@ SourceFiles
namespace Foam
{
+// Forward Declarations
+class bitSet;
+
/*---------------------------------------------------------------------------*\
Class PstreamBuffers Declaration
\*---------------------------------------------------------------------------*/
@@ -300,6 +303,25 @@ public:
const bool wait = true
);
+ //- A caching version that uses a limited send/recv connectivity.
+ //
+ // Non-blocking mode: populates receive buffers.
+ // \param sendConnections on/off for sending ranks
+ // \param sendProcs ranks used for sends
+ // \param recvProcs ranks used for recvs
+ // \param wait wait for requests to complete (in nonBlocking mode)
+ //
+ // \return True if the send/recv connectivity changed
+ //
+ // \warning currently only valid for nonBlocking comms.
+ bool finishedSends
+ (
+ bitSet& sendConnections,
+ DynamicList<label>& sendProcs,
+ DynamicList<label>& recvProcs,
+ const bool wait = true
+ );
+
//- Mark sends as done using subset of send/recv ranks
//- and recover the sizes (bytes) received.
//
diff --git a/src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.C b/src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.C
index 2a5877d51423b8ed07793e9c7145334183fb62b9..1343bd05d4122e1dc3392ea0a30212b2293b5ccd 100644
--- a/src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.C
+++ b/src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.C
@@ -29,7 +29,6 @@ License
#include "ListOps.H"
#include "Pstream.H"
#include "commSchedule.H"
-#include "boolList.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
@@ -50,22 +49,16 @@ Foam::labelList Foam::ProcessorTopology<Container, ProcPatch>::procNeighbours
forAll(patches, patchi)
{
- const typename Container::const_reference patch = patches[patchi];
-
- if (isA<ProcPatch>(patch))
+ const auto* cpp = isA<ProcPatch>(patches[patchi]);
+ if (cpp)
{
- const ProcPatch& procPatch =
- refCast<const ProcPatch>(patch);
-
- label pNeighbProcNo = procPatch.neighbProcNo();
+ const label nbrProci = cpp->neighbProcNo();
- if (!isNeighbourProc[pNeighbProcNo])
+ if (!isNeighbourProc[nbrProci])
{
- nNeighbours++;
-
- maxNb = max(maxNb, procPatch.neighbProcNo());
-
- isNeighbourProc[pNeighbProcNo] = true;
+ isNeighbourProc[nbrProci] = true;
+ maxNb = max(maxNb, nbrProci);
+ ++nNeighbours;
}
}
}
@@ -87,15 +80,13 @@ Foam::labelList Foam::ProcessorTopology<Container, ProcPatch>::procNeighbours
forAll(patches, patchi)
{
- const typename Container::const_reference patch = patches[patchi];
-
- if (isA<ProcPatch>(patch))
+ const auto* cpp = isA<ProcPatch>(patches[patchi]);
+ if (cpp)
{
- const ProcPatch& procPatch =
- refCast<const ProcPatch>(patch);
+ const label nbrProci = cpp->neighbProcNo();
- // Construct reverse map
- procPatchMap_[procPatch.neighbProcNo()] = patchi;
+ // Reverse map
+ procPatchMap_[nbrProci] = patchi;
}
}
@@ -113,7 +104,7 @@ Foam::ProcessorTopology<Container, ProcPatch>::ProcessorTopology
)
:
labelListList(Pstream::nProcs(comm)),
- patchSchedule_(2*patches.size())
+ patchSchedule_()
{
if (Pstream::parRun())
{
@@ -132,6 +123,8 @@ Foam::ProcessorTopology<Container, ProcPatch>::ProcessorTopology
&& Pstream::defaultCommsType == Pstream::commsTypes::scheduled
)
{
+ patchSchedule_.resize(2*patches.size());
+
label patchEvali = 0;
// 1. All non-processor patches
@@ -155,21 +148,21 @@ Foam::ProcessorTopology<Container, ProcPatch>::ProcessorTopology
// to determine the schedule. Each processor pair stands for both
// send and receive.
label nComms = 0;
- forAll(*this, proci)
+ for (const labelList& neighbours : *this)
{
- nComms += operator[](proci).size();
+ nComms += neighbours.size();
}
DynamicList<labelPair> comms(nComms);
forAll(*this, proci)
{
- const labelList& nbrs = operator[](proci);
+ const labelList& neighbours = operator[](proci);
- forAll(nbrs, i)
+ for (const label nbrProci : neighbours)
{
- if (proci < nbrs[i])
+ if (proci < nbrProci)
{
- comms.append(labelPair(proci, nbrs[i]));
+ comms.append(labelPair(proci, nbrProci));
}
}
}
@@ -185,10 +178,8 @@ Foam::ProcessorTopology<Container, ProcPatch>::ProcessorTopology
).procSchedule()[Pstream::myProcNo(comm)]
);
- forAll(mySchedule, iter)
+ for (const label commI : mySchedule)
{
- label commI = mySchedule[iter];
-
// Get the other processor
label nb = comms[commI][0];
if (nb == Pstream::myProcNo(comm))
diff --git a/src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.H b/src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.H
index 976d5dc8735309e8b10ed01d216feaf7f2917b81..3dbcff1777c883c1d697762b5e59549f7415a49c 100644
--- a/src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.H
+++ b/src/OpenFOAM/meshes/ProcessorTopology/ProcessorTopology.H
@@ -40,8 +40,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/
-#ifndef ProcessorTopology_H
-#define ProcessorTopology_H
+#ifndef Foam_ProcessorTopology_H
+#define Foam_ProcessorTopology_H
#include "labelList.H"
#include "lduSchedule.H"
@@ -60,10 +60,7 @@ class ProcessorTopology
:
public labelListList
{
-
-private:
-
- // Private data
+ // Private Data
//- Local map from neighbour proc to patchi. Different per processor!
// -1 or patchi for connection to procID
@@ -76,8 +73,8 @@ private:
// Private Member Functions
- //- Return all neighbouring processors of this processor. Set
- // procPatchMap_.
+ //- Return all neighbouring processors of this processor.
+ // Sets procPatchMap_.
labelList procNeighbours(const label nProcs, const Container&);
public:
@@ -88,25 +85,27 @@ public:
ProcessorTopology(const Container& patches, const label comm);
+ // Static Functions
+
+ //- Calculate non-blocking (i.e. unscheduled) schedule
+ static lduSchedule nonBlockingSchedule(const Container& patches);
+
+
// Member Functions
- //- From neighbour processor to index in boundaryMesh. Local information
- // (so not same over all processors)
- const labelList& procPatchMap() const
+ //- From neighbour processor to index in boundaryMesh.
+ // Local information (so not same over all processors)
+ const labelList& procPatchMap() const noexcept
{
return procPatchMap_;
}
//- Order in which the patches should be initialised/evaluated
- // corresponding to the schedule
- const lduSchedule& patchSchedule() const
+ //- corresponding to the schedule
+ const lduSchedule& patchSchedule() const noexcept
{
return patchSchedule_;
}
-
- //- Calculate non-blocking (i.e. unscheduled) schedule
- static lduSchedule nonBlockingSchedule(const Container& patches);
-
};
diff --git a/src/OpenFOAM/meshes/ProcessorTopology/commSchedule.C b/src/OpenFOAM/parallel/commSchedule/commSchedule.C
similarity index 97%
rename from src/OpenFOAM/meshes/ProcessorTopology/commSchedule.C
rename to src/OpenFOAM/parallel/commSchedule/commSchedule.C
index 0f2bb2e65d5ab2199f75bf9ae4ef79e901a76c8e..6bb6d5949bf7cc1364bd19d17bab0542f7ead890 100644
--- a/src/OpenFOAM/meshes/ProcessorTopology/commSchedule.C
+++ b/src/OpenFOAM/parallel/commSchedule/commSchedule.C
@@ -45,7 +45,6 @@ namespace Foam
namespace Foam
{
-// Private Member Functions
// Count the number of outstanding communications for a single processor
static label outstandingComms
@@ -99,7 +98,7 @@ Foam::commSchedule::commSchedule
}
// Note: no need to shrink procToComms. Are small.
- if (debug && Pstream::master())
+ if (debug && UPstream::master())
{
Pout<< "commSchedule : Wanted communication:" << endl;
@@ -198,7 +197,7 @@ Foam::commSchedule::commSchedule
busy[comms[maxCommI][1]] = true;
}
- if (debug && Pstream::master())
+ if (debug && UPstream::master())
{
label nIterComms = nScheduled-oldNScheduled;
@@ -240,7 +239,7 @@ Foam::commSchedule::commSchedule
iter++;
}
- if (debug && Pstream::master())
+ if (debug && UPstream::master())
{
Pout<< endl;
}
@@ -282,7 +281,7 @@ Foam::commSchedule::commSchedule
procSchedule_[proc1][nProcScheduled[proc1]++] = commI;
}
- if (debug && Pstream::master())
+ if (debug && UPstream::master())
{
Pout<< "commSchedule::commSchedule : Per processor:" << endl;
diff --git a/src/OpenFOAM/meshes/ProcessorTopology/commSchedule.H b/src/OpenFOAM/parallel/commSchedule/commSchedule.H
similarity index 100%
rename from src/OpenFOAM/meshes/ProcessorTopology/commSchedule.H
rename to src/OpenFOAM/parallel/commSchedule/commSchedule.H
diff --git a/src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.C b/src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.C
index bd600b1fb5ba99f96df80521f883c567eb08f8ce..2720b0204847fd7ffc52e5253d32169b6c81a4c8 100644
--- a/src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.C
+++ b/src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.C
@@ -27,13 +27,6 @@ License
\*---------------------------------------------------------------------------*/
#include "zoneDistribute.H"
-#include "dummyTransform.H"
-#include "emptyPolyPatch.H"
-#include "processorPolyPatch.H"
-#include "syncTools.H"
-#include "wedgePolyPatch.H"
-
-#include "globalPoints.H"
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@@ -43,61 +36,14 @@ namespace Foam
}
-// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
-
-Foam::autoPtr<Foam::indirectPrimitivePatch>
-Foam::zoneDistribute::coupledFacesPatch() const
-{
- const polyBoundaryMesh& patches = mesh_.boundaryMesh();
-
- label nCoupled = 0;
-
- for (const polyPatch& pp : patches)
- {
- if (isA<processorPolyPatch>(pp))
- {
- nCoupled += pp.size();
- }
- }
- labelList coupledFaces(nCoupled);
- nCoupled = 0;
-
- for (const polyPatch& pp : patches)
- {
- if (isA<processorPolyPatch>(pp))
- {
- label facei = pp.start();
-
- forAll(pp, i)
- {
- coupledFaces[nCoupled++] = facei++;
- }
- }
- }
-
- return autoPtr<indirectPrimitivePatch>::New
- (
- IndirectList<face>
- (
- mesh_.faces(),
- coupledFaces
- ),
- mesh_.points()
- );
-}
-
-
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::zoneDistribute::zoneDistribute(const fvMesh& mesh)
:
MeshObject<fvMesh, Foam::TopologicalMeshObject, zoneDistribute>(mesh),
- coupledBoundaryPoints_(coupledFacesPatch()().meshPoints()),
- send_(UPstream::nProcs()),
stencil_(zoneCPCStencil::New(mesh)),
globalNumbering_(stencil_.globalNumbering()),
- sendTo_(), // Initial zero-sized
- recvFrom_() // Initial zero-sized
+ send_(UPstream::nProcs())
{}
@@ -140,20 +86,10 @@ void Foam::zoneDistribute::setUpCommforZone
if (UPstream::parRun())
{
- if (sendTo_.empty())
- {
- // First time
- sendTo_.resize(UPstream::nProcs());
- recvFrom_.resize(UPstream::nProcs());
- sendTo_ = false;
- recvFrom_ = false;
- }
-
- const labelHashSet& comms = stencil.needsComm();
-
List<labelHashSet> needed(UPstream::nProcs());
- for (const label celli : comms)
+ // Bin according to originating (sending) processor
+ for (const label celli : stencil.needsComm())
{
if (zone[celli])
{
@@ -169,73 +105,24 @@ void Foam::zoneDistribute::setUpCommforZone
}
}
+ // Stream the send data into PstreamBuffers,
+ // which we also use to track the current topology.
PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking);
- // Stream data into buffer
for (const int proci : UPstream::allProcs())
{
if (proci != UPstream::myProcNo() && !needed[proci].empty())
{
- // Put data into send buffer
+ // Serialize as List
UOPstream toProc(proci, pBufs);
-
toProc << needed[proci].sortedToc();
}
}
+ pBufs.finishedSends(sendConnections_, sendProcs_, recvProcs_);
- // Need update, or use existing partial communication info?
- bool fullUpdate = false;
- for (const int proci : UPstream::allProcs())
- {
- if
- (
- proci != UPstream::myProcNo()
- && (!sendTo_[proci] && !needed[proci].empty())
- )
- {
- // Changed state sendTo_ from false -> true
- sendTo_[proci] = true;
- fullUpdate = true;
- }
- }
-
-
- if (returnReduce(fullUpdate, orOp<bool>()))
- {
- pBufs.finishedSends();
-
- // Update which ones receive
- for (const int proci : UPstream::allProcs())
- {
- recvFrom_[proci] = pBufs.hasRecvData(proci);
- }
- }
- else
- {
- // No change in senders...
- // - can communicate with a subset of processors
- DynamicList<label> sendProcs;
- DynamicList<label> recvProcs;
-
- for (const int proci : UPstream::allProcs())
- {
- if (sendTo_[proci])
- {
- sendProcs.append(proci);
- }
- if (recvFrom_[proci])
- {
- recvProcs.append(proci);
- }
- }
-
- // Wait until everything is written
- pBufs.finishedSends(sendProcs, recvProcs);
- }
-
- for (const int proci : UPstream::allProcs())
+ for (const int proci : pBufs.allProcs())
{
send_[proci].clear();
diff --git a/src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.H b/src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.H
index 0dbb7b2f3b327acf62b4bad3e40d382e857bf957..4a58c8445f677514a5a4d27db877fa4f2d3bd562 100644
--- a/src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.H
+++ b/src/finiteVolume/fvMesh/zoneDistribute/zoneDistribute.H
@@ -31,12 +31,28 @@ Description
Class for parallel communication in a narrow band. It either provides a Map
with the neighbouring values of the selected region or returns a Map of the
required values in global addressing. Also holds a reference to the stencil
- Before the data transfer the communation has to be set up:
+ Before the data transfer the communication has to be set up:
exchangeFields_.setUpCommforZone(interfaceCell_);
Is used in the plicRDF
Original code supplied by Henning Scheufler, DLR (2019)
+ Additional optimization of processor communication
+ provided by Tetsuo AOYAGI, RIST (2022), to use a more compact
+ exchange of sizes with an updated version of PstreamBuffers.
+ This optimization uses additional sendTo/recvFrom member data
+ to track the topological connectivity, acting like an on-the-fly
+ sub-communicator, and respects corner connectivity.
+
+ -# Initially topological connections are empty (or all false).
+ -# Scan the stencil global cellIds (active zones only) and split
+ into sub-lists according the originating processor (the sender).
+ -# If an originating processor appears/disappears, need to update
+ the connectivity information (requires an all-to-all).
+ -# When possible, the topological send/recv is used in PstreamBuffers
+ finishedSends (minimizes communication).
+ .
+
SourceFiles
zoneDistributeI.H
zoneDistribute.C
@@ -51,7 +67,6 @@ SourceFiles
#include "volFields.H"
#include "zoneCPCStencil.H"
-#include "IOobject.H"
#include "MeshObject.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@@ -69,47 +84,41 @@ class zoneDistribute
{
// Private Data
- //- labels of the points on coupled patches
- labelList coupledBoundaryPoints_;
-
- //- Addressing for processor-to-processor comms
- List<labelList> send_;
-
- //- Return patch of all coupled faces.
- autoPtr<indirectPrimitivePatch> coupledFacesPatch() const;
-
+ //- Reference to the zone stencil
zoneCPCStencil& stencil_;
- //- Global number into cells and faces
+ //- Global number into index of cells/faces
const globalIndex& globalNumbering_;
- //- Parallel: send data to processors (true/false)
- // updated in setUpCommforZone
- boolList sendTo_;
+ //- Global cell/face index to send for processor-to-processor comms
+ List<labelList> send_;
- //- Parallel: receive data from processors (true/false)
- // updated in setUpCommforZone
- boolList recvFrom_;
+ //- Parallel [cache]: send connectivity (true/false)
+ bitSet sendConnections_;
+
+ //- Parallel [cache]: send data to these ranks
+ DynamicList<label> sendProcs_;
+
+ //- Parallel [cache]: recv data from these ranks
+ DynamicList<label> recvProcs_;
// Private Member Functions
- //- Gives patchNumber and patchFaceNumber for a given
- //- Geometric volume field
+ //- Return local volField value at (cell or face) index
template<typename Type>
Type getLocalValue
(
- const GeometricField<Type, fvPatchField, volMesh>& phi,
+ const VolumeField<Type>& phi,
const label localIdx
) const;
-
//- Gives patchNumber and patchFaceNumber for a given
//- Geometric volume field
template<typename Type>
Type faceValue
(
- const GeometricField<Type, fvPatchField, volMesh>& phi,
+ const VolumeField<Type>& phi,
const label localIdx
) const;
@@ -158,7 +167,7 @@ public:
template<typename Type>
Type getValue
(
- const GeometricField<Type, fvPatchField, volMesh>& phi,
+ const VolumeField<Type>& phi,
const Map<Type>& valuesFromOtherProc,
const label gblIdx
) const;
@@ -168,7 +177,7 @@ public:
Map<Field<Type>> getFields
(
const boolList& zone,
- const GeometricField<Type, fvPatchField, volMesh>& phi
+ const VolumeField<Type>& phi
);
//- Returns stencil and provides a Map with globalNumbering
@@ -176,7 +185,7 @@ public:
Map<Type> getDatafromOtherProc
(
const boolList& zone,
- const GeometricField<Type, fvPatchField, volMesh>& phi
+ const VolumeField<Type>& phi
);
};
diff --git a/src/finiteVolume/fvMesh/zoneDistribute/zoneDistributeI.H b/src/finiteVolume/fvMesh/zoneDistribute/zoneDistributeI.H
index e709dde4bb28906242db6fab4d5a17cc8e43ed48..91c70f31cf74895c525813495ebcc5a3ed76a730 100644
--- a/src/finiteVolume/fvMesh/zoneDistribute/zoneDistributeI.H
+++ b/src/finiteVolume/fvMesh/zoneDistribute/zoneDistributeI.H
@@ -26,16 +26,14 @@ License
\*---------------------------------------------------------------------------*/
-#include "zoneDistribute.H"
#include "DynamicField.H"
-#include "syncTools.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
template<typename Type>
Type Foam::zoneDistribute::getLocalValue
(
- const GeometricField<Type, fvPatchField, volMesh>& phi,
+ const VolumeField<Type>& phi,
const label localIdx
) const
{
@@ -51,7 +49,7 @@ Type Foam::zoneDistribute::getLocalValue
template<typename Type>
Type Foam::zoneDistribute::faceValue
(
- const GeometricField<Type, fvPatchField, volMesh>& phi,
+ const VolumeField<Type>& phi,
const label localIdx
) const
{
@@ -80,7 +78,7 @@ Type Foam::zoneDistribute::faceValue
template<typename Type>
Type Foam::zoneDistribute::getValue
(
- const GeometricField<Type, fvPatchField, volMesh>& phi,
+ const VolumeField<Type>& phi,
const Map<Type>& valuesFromOtherProc,
const label gblIdx
) const
@@ -90,8 +88,9 @@ Type Foam::zoneDistribute::getValue
const label localIdx = globalNumbering_.toLocal(gblIdx);
return getLocalValue(phi,localIdx);
}
- else // from other proc
+ else
{
+ // From other proc
return valuesFromOtherProc[gblIdx];
}
}
@@ -101,7 +100,7 @@ template<typename Type>
Foam::Map<Foam::Field<Type>> Foam::zoneDistribute::getFields
(
const boolList& zone,
- const GeometricField<Type, fvPatchField, volMesh>& phi
+ const VolumeField<Type>& phi
)
{
if (zone.size() != phi.size())
@@ -144,7 +143,7 @@ template<typename Type>
Foam::Map<Type> Foam::zoneDistribute::getDatafromOtherProc
(
const boolList& zone,
- const GeometricField<Type, fvPatchField, volMesh>& phi
+ const VolumeField<Type>& phi
)
{
if (zone.size() != phi.size())
@@ -162,86 +161,51 @@ Foam::Map<Type> Foam::zoneDistribute::getDatafromOtherProc
if (UPstream::parRun())
{
- const bool commsPrepared = (sendTo_.size() == UPstream::nProcs());
-
- if (!commsPrepared)
+ if (sendConnections_.empty())
{
- FatalErrorInFunction
- << "The send/recv information not initialized - "
+ WarningInFunction
+ << "The send/recv connections not initialized - "
<< "likely that setUpCommforZone() was not called"
<< endl;
-
// But don't exit/abort for now
}
-
- List<Map<Type>> sendValues(UPstream::nProcs());
-
- forAll(send_, proci)
- {
- for (const label sendIdx : send_[proci])
- {
- sendValues[proci].insert
- (
- sendIdx,
- getLocalValue(phi, globalNumbering_.toLocal(sendIdx))
- );
- }
- }
-
+ // Stream the send data into PstreamBuffers,
+ // which we also use to track the current topology.
PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking);
- // Stream data into buffer
for (const int proci : UPstream::allProcs())
{
- if (proci != UPstream::myProcNo() && !sendValues[proci].empty())
+ if (proci != UPstream::myProcNo() && !send_[proci].empty())
{
- // Put data into send buffer
- UOPstream toProc(proci, pBufs);
- toProc << sendValues[proci];
- }
- }
-
+ // Serialize as Map
+ Map<Type> sendValues(2*send_[proci].size());
- if (commsPrepared)
- {
- DynamicList<label> sendProcs;
- DynamicList<label> recvProcs;
-
- for (const int proci : UPstream::allProcs())
- {
- if (sendTo_[proci])
+ for (const label sendIdx : send_[proci])
{
- sendProcs.append(proci);
+ sendValues.insert
+ (
+ sendIdx,
+ getLocalValue(phi, globalNumbering_.toLocal(sendIdx))
+ );
}
- if (recvFrom_[proci])
- {
- recvProcs.append(proci);
- }
- }
- pBufs.finishedSends(sendProcs, recvProcs);
- }
- else
- {
- // Fallback to all-to-all communication
-
- pBufs.finishedSends();
+ UOPstream toProc(proci, pBufs);
+ toProc << sendValues;
+ }
}
+ pBufs.finishedSends(sendConnections_, sendProcs_, recvProcs_);
- for (const int proci : UPstream::allProcs())
+ for (const int proci : pBufs.allProcs())
{
- // Do not use recvFrom_[proci] here
- // - could be incorrect if comms are not prepared!
-
if (proci != UPstream::myProcNo() && pBufs.hasRecvData(proci))
{
UIPstream fromProc(proci, pBufs);
- Map<Type> tmpValue(fromProc);
+ Map<Type> tmpValues(fromProc);
- neiValues += tmpValue;
+ neiValues += tmpValues;
}
}
}
diff --git a/src/transportModels/geometricVoF/reconstructedDistanceFunction/reconstructedDistanceFunction.C b/src/transportModels/geometricVoF/reconstructedDistanceFunction/reconstructedDistanceFunction.C
index fedd64a9090a9355dd13fdfc6add31ca71119a7a..da0c0a0b4736d06ef7942b980186c93d428d1ba2 100644
--- a/src/transportModels/geometricVoF/reconstructedDistanceFunction/reconstructedDistanceFunction.C
+++ b/src/transportModels/geometricVoF/reconstructedDistanceFunction/reconstructedDistanceFunction.C
@@ -67,11 +67,7 @@ Foam::reconstructedDistanceFunction::coupledFacesPatch() const
return autoPtr<indirectPrimitivePatch>::New
(
- IndirectList<face>
- (
- mesh_.faces(),
- nCoupledFaces
- ),
+ IndirectList<face>(mesh_.faces(), std::move(nCoupledFaces)),
mesh_.points()
);
}
diff --git a/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/gradAlpha/gradAlpha.C b/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/gradAlpha/gradAlpha.C
index 14d899b4b31739be772088525173511917f0151f..7b9e7fe11057a40926c5f538a12645c318749abc 100644
--- a/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/gradAlpha/gradAlpha.C
+++ b/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/gradAlpha/gradAlpha.C
@@ -28,8 +28,10 @@ License
#include "gradAlpha.H"
#include "fvc.H"
#include "leastSquareGrad.H"
+#include "zoneDistribute.H"
#include "addToRunTimeSelectionTable.H"
#include "profiling.H"
+
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
namespace Foam
@@ -50,8 +52,7 @@ void Foam::reconstruction::gradAlpha::gradSurf(const volScalarField& phi)
leastSquareGrad<scalar> lsGrad("polyDegree1",mesh_.geometricD());
zoneDistribute& exchangeFields = zoneDistribute::New(mesh_);
-
- exchangeFields.setUpCommforZone(interfaceCell_,true);
+ exchangeFields.setUpCommforZone(interfaceCell_, true);
Map<vector> mapCC
(
@@ -135,10 +136,7 @@ void Foam::reconstruction::gradAlpha::reconstruct(bool forceUpdate)
if (mesh_.topoChanging())
{
// Introduced resizing to cope with changing meshes
- if (interfaceCell_.size() != mesh_.nCells())
- {
- interfaceCell_.resize(mesh_.nCells());
- }
+ interfaceCell_.resize_nocopy(mesh_.nCells());
}
interfaceCell_ = false;
diff --git a/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/gradAlpha/gradAlpha.H b/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/gradAlpha/gradAlpha.H
index f895ff625c7ae29bf227233f0066992c103eec3e..7666d348fe12106431ec718d8651d5903dee57e1 100644
--- a/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/gradAlpha/gradAlpha.H
+++ b/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/gradAlpha/gradAlpha.H
@@ -45,10 +45,9 @@ SourceFiles
#include "typeInfo.H"
#include "reconstructionSchemes.H"
#include "volFields.H"
+#include "DynamicField.H"
#include "dimensionedScalar.H"
-#include "autoPtr.H"
#include "surfaceIteratorPLIC.H"
-#include "zoneDistribute.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@@ -86,6 +85,7 @@ class gradAlpha
//- SurfaceIterator finds the plane centre for specified VOF value
surfaceIteratorPLIC sIterPLIC_;
+
// Private Member Functions
//- Compute gradient at the surfaces
diff --git a/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/plicRDF/plicRDF.C b/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/plicRDF/plicRDF.C
index dbdcb9b21a6322e6efc9a1ac54f20138a9cf7f46..10fe2c055a84d22230e5550c2420d6a5931f1b52 100644
--- a/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/plicRDF/plicRDF.C
+++ b/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/plicRDF/plicRDF.C
@@ -29,6 +29,7 @@ License
#include "interpolationCellPoint.H"
#include "fvc.H"
#include "leastSquareGrad.H"
+#include "zoneDistribute.H"
#include "addToRunTimeSelectionTable.H"
#include "profiling.H"
@@ -50,11 +51,11 @@ void Foam::reconstruction::plicRDF::interpolateNormal()
{
addProfilingInFunction(geometricVoF);
scalar dt = mesh_.time().deltaTValue();
- zoneDistribute& exchangeFields = zoneDistribute::New(mesh_);
leastSquareGrad<scalar> lsGrad("polyDegree1",mesh_.geometricD());
- exchangeFields.setUpCommforZone(interfaceCell_,false);
+ zoneDistribute& exchangeFields = zoneDistribute::New(mesh_);
+ exchangeFields.setUpCommforZone(interfaceCell_, false);
Map<vector> mapCentre
(
@@ -74,8 +75,8 @@ void Foam::reconstruction::plicRDF::interpolateNormal()
exchangeFields.getDatafromOtherProc(interfaceCell_, alpha1_)
);
- DynamicField<vector > cellCentre(100);
- DynamicField<scalar > alphaValues(100);
+ DynamicField<vector> cellCentre(100);
+ DynamicField<scalar> alphaValues(100);
DynamicList<vector> foundNormals(30);
diff --git a/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/plicRDF/plicRDF.H b/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/plicRDF/plicRDF.H
index 02a1bdf139d8886ffcc545d9116b7da5864625db..146b35ccbd08d1bef3258fc6a5004284de03232d 100644
--- a/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/plicRDF/plicRDF.H
+++ b/src/transportModels/geometricVoF/reconstructionSchemes/plicSchemes/plicRDF/plicRDF.H
@@ -59,7 +59,6 @@ SourceFiles
#include "autoPtr.H"
#include "surfaceIteratorPLIC.H"
#include "reconstructedDistanceFunction.H"
-#include "zoneDistribute.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
diff --git a/tutorials/multiphase/compressibleInterIsoFoam/laminar/depthCharge2D/Allrun-parallel b/tutorials/multiphase/compressibleInterIsoFoam/laminar/depthCharge2D/Allrun-parallel
new file mode 100755
index 0000000000000000000000000000000000000000..921e2b8fa24dfb2519e4bacc9a72bedd7c35e45e
--- /dev/null
+++ b/tutorials/multiphase/compressibleInterIsoFoam/laminar/depthCharge2D/Allrun-parallel
@@ -0,0 +1,18 @@
+#!/bin/sh
+cd "${0%/*}" || exit # Run from this directory
+. ${WM_PROJECT_DIR:?}/bin/tools/RunFunctions # Tutorial run functions
+#------------------------------------------------------------------------------
+
+runApplication blockMesh
+
+restore0Dir
+
+runApplication setFields
+
+runApplication setAlphaField
+
+decomposePar
+
+runParallel $(getApplication)
+
+#------------------------------------------------------------------------------
diff --git a/tutorials/multiphase/compressibleInterIsoFoam/laminar/depthCharge2D/system/decomposeParDict b/tutorials/multiphase/compressibleInterIsoFoam/laminar/depthCharge2D/system/decomposeParDict
new file mode 100644
index 0000000000000000000000000000000000000000..d283b764dc599e0b04d9fb2d1cd0f594e171c67b
--- /dev/null
+++ b/tutorials/multiphase/compressibleInterIsoFoam/laminar/depthCharge2D/system/decomposeParDict
@@ -0,0 +1,27 @@
+/*--------------------------------*- C++ -*----------------------------------*\
+| ========= | |
+| \\ / F ield | OpenFOAM: The Open Source CFD Toolbox |
+| \\ / O peration | Version: v2112 |
+| \\ / A nd | Website: www.openfoam.com |
+| \\/ M anipulation | |
+\*---------------------------------------------------------------------------*/
+FoamFile
+{
+ version 2.0;
+ format ascii;
+ class dictionary;
+ object decomposeParDict;
+}
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+numberOfSubdomains 8;
+
+method hierarchical;
+
+coeffs
+{
+ n (4 2 1);
+}
+
+
+// ************************************************************************* //