From 1ab9dca2abdd3b26af82d2713f0213ccdee98881 Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Fri, 10 Feb 2023 13:59:18 +0100
Subject: [PATCH] ENH: reduce all-to-all communication in mesh wave algorithms
- use persistent PstreamBuffers between iterations, restrict size
information exchange to the processor neighbours (which is what the
algorithm is handling there anyhow).
---
.../polyMesh/globalMeshData/globalPoints.C | 39 ++++++++++---------
.../polyTopoChange/addPatchCellLayer.C | 25 ------------
.../polyTopoChange/addPatchCellLayer.H | 3 --
.../algorithms/MeshWave/FaceCellWave.C | 18 +++++----
.../algorithms/MeshWave/FaceCellWave.H | 10 +++--
.../algorithms/MeshWave/FaceCellWaveBase.C | 3 +-
.../algorithms/PointEdgeWave/PointEdgeWave.C | 28 ++++++++-----
.../algorithms/PointEdgeWave/PointEdgeWave.H | 8 +++-
.../PointEdgeWave/PointEdgeWaveBase.C | 3 +-
9 files changed, 65 insertions(+), 72 deletions(-)
diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalPoints.C b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalPoints.C
index f34b7c542d7..5466f411304 100644
--- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalPoints.C
+++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalPoints.C
@@ -891,19 +891,22 @@ void Foam::globalPoints::calculateSharedPoints
// a point or edge.
initOwnPoints(meshToPatchPoint, true, changedPoints);
+ // Note: to use 'scheduled' would have to intersperse send and receive.
+ // So for now just use nonBlocking. Also globalPoints itself gets
+ // constructed by mesh.globalData().patchSchedule() so creates a loop.
+ PstreamBuffers pBufs
+ (
+ (
+ Pstream::defaultCommsType == Pstream::commsTypes::scheduled
+ ? Pstream::commsTypes::nonBlocking
+ : Pstream::defaultCommsType
+ )
+ );
+
// Do one exchange iteration to get neighbour points.
{
- // Note: to use 'scheduled' would have to intersperse send and receive.
- // So for now just use nonBlocking. Also globalPoints itself gets
- // constructed by mesh.globalData().patchSchedule() so creates a loop.
- PstreamBuffers pBufs
- (
- (
- Pstream::defaultCommsType == Pstream::commsTypes::scheduled
- ? Pstream::commsTypes::nonBlocking
- : Pstream::defaultCommsType
- )
- );
+ pBufs.clear();
+
sendPatchPoints
(
mergeSeparated,
@@ -911,7 +914,9 @@ void Foam::globalPoints::calculateSharedPoints
pBufs,
changedPoints
);
+
pBufs.finishedSends();
+
receivePatchPoints
(
mergeSeparated,
@@ -933,14 +938,8 @@ void Foam::globalPoints::calculateSharedPoints
do
{
- PstreamBuffers pBufs
- (
- (
- Pstream::defaultCommsType == Pstream::commsTypes::scheduled
- ? Pstream::commsTypes::nonBlocking
- : Pstream::defaultCommsType
- )
- );
+ pBufs.clear();
+
sendPatchPoints
(
mergeSeparated,
@@ -948,7 +947,9 @@ void Foam::globalPoints::calculateSharedPoints
pBufs,
changedPoints
);
+
pBufs.finishedSends();
+
receivePatchPoints
(
mergeSeparated,
diff --git a/src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.C b/src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.C
index 8802246e926..f0441afd67e 100644
--- a/src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.C
+++ b/src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.C
@@ -452,31 +452,6 @@ Foam::label Foam::addPatchCellLayer::addSideFace
}
-Foam::label Foam::addPatchCellLayer::findProcPatch
-(
- const polyMesh& mesh,
- const label nbrProcID
-)
-{
- const polyBoundaryMesh& patches = mesh.boundaryMesh();
-
- forAll(mesh.globalData().processorPatches(), i)
- {
- label patchi = mesh.globalData().processorPatches()[i];
-
- if
- (
- refCast<const processorPolyPatch>(patches[patchi]).neighbProcNo()
- == nbrProcID
- )
- {
- return patchi;
- }
- }
- return -1;
-}
-
-
void Foam::addPatchCellLayer::setFaceProps
(
const polyMesh& mesh,
diff --git a/src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.H b/src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.H
index b0d12af7ed6..fe59a8f6080 100644
--- a/src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.H
+++ b/src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.H
@@ -202,9 +202,6 @@ class addPatchCellLayer
polyTopoChange&
) const;
- //- Find patch to neighbouring processor
- static label findProcPatch(const polyMesh&, const label nbrProcID);
-
//- Extract properties from mesh face
static void setFaceProps
(
diff --git a/src/meshTools/algorithms/MeshWave/FaceCellWave.C b/src/meshTools/algorithms/MeshWave/FaceCellWave.C
index aeb89303248..325f93ba944 100644
--- a/src/meshTools/algorithms/MeshWave/FaceCellWave.C
+++ b/src/meshTools/algorithms/MeshWave/FaceCellWave.C
@@ -526,13 +526,15 @@ void Foam::FaceCellWave<Type, TrackingData>::handleProcPatches()
// Which patches are processor patches
const labelList& procPatches = pData.processorPatches();
- // Send all
+ // Which processors this processor is connected to
+ const labelList& neighbourProcs = pData.topology().procNeighbours();
- PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+ // Reset buffers
+ pBufs_.clear();
for (const label patchi : procPatches)
{
- const processorPolyPatch& procPatch =
+ const auto& procPatch =
refCast<const processorPolyPatch>(mesh_.boundaryMesh()[patchi]);
// Allocate buffers
@@ -567,20 +569,20 @@ void Foam::FaceCellWave<Type, TrackingData>::handleProcPatches()
<< endl;
}
- UOPstream toNeighbour(procPatch.neighbProcNo(), pBufs);
+ UOPstream toNeighbour(procPatch.neighbProcNo(), pBufs_);
//writeFaces(nSendFaces, sendFaces, sendFacesInfo, toNeighbour);
toNeighbour
<< SubList<label>(sendFaces, nSendFaces)
<< SubList<Type>(sendFacesInfo, nSendFaces);
}
- pBufs.finishedSends();
+ // Finished sends
+ pBufs_.finishedNeighbourSends(neighbourProcs);
- // Receive all
for (const label patchi : procPatches)
{
- const processorPolyPatch& procPatch =
+ const auto& procPatch =
refCast<const processorPolyPatch>(mesh_.boundaryMesh()[patchi]);
// Allocate buffers
@@ -588,7 +590,7 @@ void Foam::FaceCellWave<Type, TrackingData>::handleProcPatches()
List<Type> receiveFacesInfo;
{
- UIPstream fromNeighbour(procPatch.neighbProcNo(), pBufs);
+ UIPstream fromNeighbour(procPatch.neighbProcNo(), pBufs_);
fromNeighbour >> receiveFaces >> receiveFacesInfo;
}
diff --git a/src/meshTools/algorithms/MeshWave/FaceCellWave.H b/src/meshTools/algorithms/MeshWave/FaceCellWave.H
index e98f41d4131..e2dd118bd42 100644
--- a/src/meshTools/algorithms/MeshWave/FaceCellWave.H
+++ b/src/meshTools/algorithms/MeshWave/FaceCellWave.H
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
- Copyright (C) 2018-2022 OpenCFD Ltd.
+ Copyright (C) 2018-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -51,9 +51,10 @@ SourceFiles
#define Foam_FaceCellWave_H
#include "bitSet.H"
+#include "labelPair.H"
#include "DynamicList.H"
#include "primitiveFieldsFwd.H"
-#include "labelPair.H"
+#include "PstreamBuffers.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@@ -83,6 +84,9 @@ protected:
//- Reference to mesh
const polyMesh& mesh_;
+ //- Buffers when updating processor patches
+ PstreamBuffers pBufs_;
+
//- Track if face has changed
bitSet changedFace_;
@@ -115,7 +119,7 @@ public:
// Constructors
//- Construct with mesh reference and set initial sizes
- FaceCellWaveBase(const polyMesh& mesh);
+ explicit FaceCellWaveBase(const polyMesh& mesh);
// Static Functions
diff --git a/src/meshTools/algorithms/MeshWave/FaceCellWaveBase.C b/src/meshTools/algorithms/MeshWave/FaceCellWaveBase.C
index 0666052af5c..9cbeb1e675d 100644
--- a/src/meshTools/algorithms/MeshWave/FaceCellWaveBase.C
+++ b/src/meshTools/algorithms/MeshWave/FaceCellWaveBase.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2012 OpenFOAM Foundation
- Copyright (C) 2022 OpenCFD Ltd.
+ Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -52,6 +52,7 @@ Foam::FaceCellWaveBase::FaceCellWaveBase
)
:
mesh_(mesh),
+ pBufs_(UPstream::commsTypes::nonBlocking),
changedFace_(mesh_.nFaces()),
changedCell_(mesh_.nCells()),
changedFaces_(mesh_.nFaces()),
diff --git a/src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.C b/src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.C
index 9ccf3e3762e..215be1c4674 100644
--- a/src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.C
+++ b/src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.C
@@ -305,16 +305,24 @@ void Foam::PointEdgeWave<Type, TrackingData>::handleProcPatches()
{
// 1. Send all point info on processor patches.
- PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+ const globalMeshData& pData = mesh_.globalData();
+
+ // Which patches are processor patches
+ const labelList& procPatches = pData.processorPatches();
+
+ // Which processors this processor is connected to
+ const labelList& neighbourProcs = pData.topology().procNeighbours();
+
+ // Reset buffers
+ pBufs_.clear();
DynamicList<Type> patchInfo;
DynamicList<label> thisPoints;
DynamicList<label> nbrPoints;
- forAll(mesh_.globalData().processorPatches(), i)
+ for (const label patchi : procPatches)
{
- label patchi = mesh_.globalData().processorPatches()[i];
- const processorPolyPatch& procPatch =
+ const auto& procPatch =
refCast<const processorPolyPatch>(mesh_.boundaryMesh()[patchi]);
patchInfo.clear();
@@ -347,28 +355,28 @@ void Foam::PointEdgeWave<Type, TrackingData>::handleProcPatches()
// << " Sending:" << patchInfo.size() << endl;
//}
- UOPstream toNeighbour(procPatch.neighbProcNo(), pBufs);
+ UOPstream toNeighbour(procPatch.neighbProcNo(), pBufs_);
toNeighbour << nbrPoints << patchInfo;
}
- pBufs.finishedSends();
+ // Finished sends
+ pBufs_.finishedNeighbourSends(neighbourProcs);
//
// 2. Receive all point info on processor patches.
//
- forAll(mesh_.globalData().processorPatches(), i)
+ for (const label patchi : procPatches)
{
- label patchi = mesh_.globalData().processorPatches()[i];
- const processorPolyPatch& procPatch =
+ const auto& procPatch =
refCast<const processorPolyPatch>(mesh_.boundaryMesh()[patchi]);
List<Type> patchInfo;
labelList patchPoints;
{
- UIPstream fromNeighbour(procPatch.neighbProcNo(), pBufs);
+ UIPstream fromNeighbour(procPatch.neighbProcNo(), pBufs_);
fromNeighbour >> patchPoints >> patchInfo;
}
diff --git a/src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.H b/src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.H
index 453d6e4b17d..10af737920c 100644
--- a/src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.H
+++ b/src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.H
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
- Copyright (C) 2020-2022 OpenCFD Ltd.
+ Copyright (C) 2020-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -64,6 +64,7 @@ SourceFiles
#include "bitSet.H"
#include "scalarField.H"
#include "tensorField.H"
+#include "PstreamBuffers.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@@ -95,6 +96,9 @@ protected:
//- Reference to mesh
const polyMesh& mesh_;
+ //- Buffers when updating processor patches
+ PstreamBuffers pBufs_;
+
//- Track if point has changed
bitSet changedPoint_;
@@ -127,7 +131,7 @@ public:
// Constructors
//- Construct with mesh reference and set initial sizes
- PointEdgeWaveBase(const polyMesh& mesh);
+ explicit PointEdgeWaveBase(const polyMesh& mesh);
// Static Functions
diff --git a/src/meshTools/algorithms/PointEdgeWave/PointEdgeWaveBase.C b/src/meshTools/algorithms/PointEdgeWave/PointEdgeWaveBase.C
index 2af77fb4f27..2b677989744 100644
--- a/src/meshTools/algorithms/PointEdgeWave/PointEdgeWaveBase.C
+++ b/src/meshTools/algorithms/PointEdgeWave/PointEdgeWaveBase.C
@@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2012 OpenFOAM Foundation
- Copyright (C) 2022 OpenCFD Ltd.
+ Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@@ -50,6 +50,7 @@ Foam::PointEdgeWaveBase::PointEdgeWaveBase
)
:
mesh_(mesh),
+ pBufs_(UPstream::commsTypes::nonBlocking),
changedPoint_(mesh_.nPoints()),
changedEdge_(mesh_.nEdges()),
changedPoints_(mesh_.nPoints()),
--
GitLab