diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C index 37459b1abdaa3a3572073cbf87f7e252f4a89f11..391c094beace15fa281aef2584b4234df1a44691 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C @@ -45,6 +45,11 @@ registerOptSwitch Foam::PstreamBuffers::algorithm ); +namespace Foam +{ + defineTypeNameAndDebug(PstreamBuffers, 0); +} + // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // @@ -231,6 +236,12 @@ void Foam::PstreamBuffers::finalExchange labelList& recvSizes ) { + DebugPoutInFunction + << "tag:" << tag_ + << " comm:" << comm_ + << " nProcs:" << nProcs_ + << endl; + initFinalExchange(); if (commsType_ == UPstream::commsTypes::nonBlocking) @@ -303,13 +314,25 @@ Foam::PstreamBuffers::PstreamBuffers sendBuffers_(nProcs_), recvBuffers_(nProcs_), recvPositions_(nProcs_, Zero) -{} +{ + DebugPoutInFunction + << "tag:" << tag_ + << " comm:" << comm_ + << " nProcs:" << nProcs_ + << endl; +} // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // Foam::PstreamBuffers::~PstreamBuffers() { + DebugPoutInFunction + << "tag:" << tag_ + << " comm:" << comm_ + << " nProcs:" << nProcs_ + << endl; + // Check that all data has been consumed. forAll(recvBuffers_, proci) { @@ -321,6 +344,7 @@ Foam::PstreamBuffers::~PstreamBuffers() FatalErrorInFunction << "Message from processor " << proci << " Only consumed " << pos << " of " << len << " bytes" << nl + << " comm " << comm_ << " tag " << tag_ << nl << Foam::abort(FatalError); } } @@ -623,6 +647,12 @@ Foam::PstreamBuffers::peekRecvData(const label proci) const void Foam::PstreamBuffers::finishedSends(const bool wait) { + DebugPoutInFunction + << "tag:" << tag_ + << " comm:" << comm_ + << " nProcs:" << nProcs_ + << endl; + labelList recvSizes; finalExchange(modeOption::DEFAULT, wait, recvSizes); } @@ -630,6 +660,12 @@ void Foam::PstreamBuffers::finishedSends(const bool wait) void Foam::PstreamBuffers::finishedSendsNBX(const bool wait) { + DebugPoutInFunction + << "tag:" << tag_ + << " comm:" << comm_ + << " nProcs:" << nProcs_ + << endl; + labelList recvSizes; finalExchange(modeOption::NBX_PEX, wait, recvSizes); } @@ -641,6 +677,12 @@ void Foam::PstreamBuffers::finishedSends const bool wait ) { + DebugPoutInFunction + << "tag:" << tag_ + << " comm:" << comm_ + << " nProcs:" << nProcs_ + << endl; + // Resize for copying back recvSizes.resize_nocopy(sendBuffers_.size()); diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H index ed40029c9329aef3002343c72f62477bc3adc218..97cb68fe9acb624bb1d0814520c64f39effc8a03 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H @@ -248,6 +248,10 @@ class PstreamBuffers public: + // Declare name of the class and its debug switch + ClassName("PstreamBuffers"); + + // Static Data //- Preferred exchange algorithm (may change or be removed in future) diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C index c46241ecebaf2cf703b33d711f962f3dda13b375..55ac9b98fc440df45d1fbc6259adb5ef7bee8499 100644 --- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C +++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C @@ -88,6 +88,9 @@ void Foam::globalMeshData::initProcAddr() if (UPstream::parRun()) { + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + PstreamBuffers pBufs(mesh_.comm()); // Send indices of my processor patches to my neighbours @@ -120,6 +123,9 @@ void Foam::globalMeshData::initProcAddr() fromNeighbour >> processorPatchNeighbours_[patchi]; } + + // Reset tag + UPstream::msgType(oldTag); } } @@ -137,6 +143,9 @@ void Foam::globalMeshData::calcSharedPoints() const << "Shared point addressing already done" << abort(FatalError); } + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + // Calculate all shared points (exclude points that are only // on two coupled patches). This does all the hard work. const globalPoints parallelPoints(mesh_, false, true); @@ -229,6 +238,9 @@ void Foam::globalMeshData::calcSharedPoints() const } } + // Reset tag + UPstream::msgType(oldTag); + if (debug) { Pout<< "globalMeshData : nGlobalPoints_:" << nGlobalPoints_ << nl @@ -356,6 +368,9 @@ void Foam::globalMeshData::calcSharedEdges() const EdgeMap<label> globalShared(2*nGlobalPoints()); + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + if (UPstream::master()) { label sharedEdgeI = 0; @@ -466,6 +481,9 @@ void Foam::globalMeshData::calcSharedEdges() const new labelList(std::move(dynSharedEdgeAddr)) ); + // Reset tag + UPstream::msgType(oldTag); + if (debug) { Pout<< "globalMeshData : nGlobalEdges_:" << nGlobalEdges_ << nl @@ -486,6 +504,9 @@ void Foam::globalMeshData::calcGlobalPointSlaves() const << endl; } + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + // Calculate connected points for master points. globalPoints globalData(mesh_, coupledPatch(), true, true); @@ -511,6 +532,9 @@ void Foam::globalMeshData::calcGlobalPointSlaves() const std::move(globalData.map()) ) ); + + // Reset tag + UPstream::msgType(oldTag); } @@ -522,10 +546,14 @@ void Foam::globalMeshData::calcPointConnectivity const globalIndexAndTransform& transforms = globalTransforms(); const labelListList& slaves = globalPointSlaves(); const labelListList& transformedSlaves = globalPointTransformedSlaves(); + const auto& slavesMap = globalPointSlavesMap(); + + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); // Create field with my local data - labelPairList myData(globalPointSlavesMap().constructSize()); + labelPairList myData(slavesMap.constructSize()); forAll(slaves, pointi) { myData[pointi] = transforms.encode @@ -536,11 +564,11 @@ void Foam::globalMeshData::calcPointConnectivity ); } // Send to master - globalPointSlavesMap().distribute(myData); + slavesMap.distribute(myData, true, UPstream::msgType()); // String of connected points with their transform - allPointConnectivity.setSize(globalPointSlavesMap().constructSize()); + allPointConnectivity.setSize(slavesMap.constructSize()); allPointConnectivity = labelPairList(0); // Pass1: do the master points since these also update local slaves @@ -569,7 +597,7 @@ void Foam::globalMeshData::calcPointConnectivity forAll(pTransformSlaves, i) { // Get transform from index - label transformI = globalPointSlavesMap().whichTransform + label transformI = slavesMap.whichTransform ( pTransformSlaves[i] ); @@ -610,11 +638,16 @@ void Foam::globalMeshData::calcPointConnectivity } - globalPointSlavesMap().reverseDistribute + slavesMap.reverseDistribute ( slaves.size(), - allPointConnectivity + allPointConnectivity, + true, + UPstream::msgType() ); + + // Reset tag + UPstream::msgType(oldTag); } @@ -655,8 +688,25 @@ void Foam::globalMeshData::calcGlobalPointEdges } // Pull slave data to master. Dummy transform. - globalPointSlavesMap().distribute(globalPointEdges); - globalPointSlavesMap().distribute(globalPointPoints); + + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + + globalPointSlavesMap().distribute + ( + globalPointEdges, + true, + UPstream::msgType() + ); + // Make sure second send uses 'far' away tags in case of NBX deciding on + // multi-pass spraying of messages with consecutive tags + globalPointSlavesMap().distribute + ( + globalPointPoints, + true, + UPstream::msgType()+23456 // Unique, far enough away tag + ); + // Add all pointEdges forAll(slaves, pointi) { @@ -764,14 +814,21 @@ void Foam::globalMeshData::calcGlobalPointEdges globalPointSlavesMap().reverseDistribute ( slaves.size(), - globalPointEdges + globalPointEdges, + true, + UPstream::msgType() ); // Push back globalPointSlavesMap().reverseDistribute ( slaves.size(), - globalPointPoints + globalPointPoints, + true, + UPstream::msgType()+65432 // Unique, far enough away tag ); + + // Reset tag + UPstream::msgType(oldTag); } @@ -852,6 +909,8 @@ void Foam::globalMeshData::calcGlobalEdgeSlaves() const // need to make sure that all points know about connectivity and // the transformations. + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); // 1. collect point connectivity - basically recreating globalPoints output. // All points will now have a string of coupled points. The transforms are @@ -1037,10 +1096,13 @@ void Foam::globalMeshData::calcGlobalEdgeSlaves() const transformedEdges, globalEdgeTransformedSlavesPtr_(), - compactMap + compactMap, + UPstream::msgType() ) ); + // Reset tag + UPstream::msgType(oldTag); if (debug) { @@ -1061,6 +1123,9 @@ void Foam::globalMeshData::calcGlobalEdgeOrientation() const << " calculating edge orientation w.r.t. master edge." << endl; } + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + const globalIndex& globalPoints = globalPointNumbering(); // 1. Determine master point @@ -1172,6 +1237,9 @@ void Foam::globalMeshData::calcGlobalEdgeOrientation() const } } + // Reset tag + UPstream::msgType(oldTag); + if (debug) { Pout<< "globalMeshData::calcGlobalEdgeOrientation() :" @@ -1262,6 +1330,9 @@ void Foam::globalMeshData::calcGlobalPointBoundaryFaces() const const label myProci = UPstream::myProcNo(); + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + // Construct local point to (uncoupled)boundaryfaces. labelListList pointBoundaryFaces; calcPointBoundaryFaces(pointBoundaryFaces); @@ -1429,6 +1500,9 @@ void Foam::globalMeshData::calcGlobalPointBoundaryFaces() const globalPointBoundaryFaces.setSize(coupledPatch().nPoints()); globalPointTransformedBoundaryFacesPtr_().setSize(coupledPatch().nPoints()); + // Reset tag + UPstream::msgType(oldTag); + if (debug) { Pout<< "globalMeshData::calcGlobalPointBoundaryFaces() :" @@ -1498,6 +1572,10 @@ void Foam::globalMeshData::calcGlobalPointBoundaryCells() const // Convert point-cells to global (boundary)cell numbers // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + globalBoundaryCellNumberingPtr_.reset ( new globalIndex(boundaryCells.size()) @@ -1654,6 +1732,9 @@ void Foam::globalMeshData::calcGlobalPointBoundaryCells() const globalPointBoundaryCells.setSize(coupledPatch().nPoints()); globalPointTransformedBoundaryCellsPtr_().setSize(coupledPatch().nPoints()); + // Reset tag + UPstream::msgType(oldTag); + if (debug) { Pout<< "globalMeshData::calcGlobalPointBoundaryCells() :" @@ -1676,6 +1757,9 @@ void Foam::globalMeshData::calcGlobalCoPointSlaves() const << " slave point addressing." << endl; } + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + // Calculate connected points for master points. globalPoints globalData(mesh_, coupledPatch(), true, false); @@ -1694,6 +1778,8 @@ void Foam::globalMeshData::calcGlobalCoPointSlaves() const ) ); + UPstream::msgType(oldTag); + if (debug) { Pout<< "globalMeshData::calcGlobalCoPointSlaves() :" @@ -1850,6 +1936,9 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const // Get all processors to send their shared points to master. // (not very efficient) + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + pointField sharedPoints(nGlobalPoints()); const labelList& pointAddr = sharedPointAddr(); const labelList& pointLabels = sharedPointLabels(); @@ -1901,6 +1990,8 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const // Broadcast: sharedPoints to all Pstream::broadcast(sharedPoints); // == worldComm + // Reset tag + UPstream::msgType(oldTag); return sharedPoints; } @@ -1908,6 +1999,9 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const Foam::pointField Foam::globalMeshData::geometricSharedPoints() const { + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + // Get coords of my shared points pointField sharedPoints(mesh_.points(), sharedPointLabels()); @@ -1926,6 +2020,9 @@ Foam::pointField Foam::globalMeshData::geometricSharedPoints() const pointMap ); + // Reset tag + UPstream::msgType(oldTag); + return sharedPoints; } @@ -2091,10 +2188,16 @@ const Foam::globalIndex& Foam::globalMeshData::globalPointNumbering() const { if (!globalPointNumberingPtr_) { + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + globalPointNumberingPtr_.reset ( new globalIndex(coupledPatch().nPoints()) ); + + // Reset tag + UPstream::msgType(oldTag); } return *globalPointNumberingPtr_; } @@ -2105,7 +2208,13 @@ Foam::globalMeshData::globalTransforms() const { if (!globalTransformsPtr_) { + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + globalTransformsPtr_.reset(new globalIndexAndTransform(mesh_)); + + // Reset tag + UPstream::msgType(oldTag); } return *globalTransformsPtr_; } @@ -2146,10 +2255,16 @@ const Foam::globalIndex& Foam::globalMeshData::globalEdgeNumbering() const { if (!globalEdgeNumberingPtr_) { + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + globalEdgeNumberingPtr_.reset ( new globalIndex(coupledPatch().nEdges()) ); + + // Reset tag + UPstream::msgType(oldTag); } return *globalEdgeNumberingPtr_; } @@ -2326,6 +2441,8 @@ Foam::autoPtr<Foam::globalIndex> Foam::globalMeshData::mergePoints const labelListList& pointSlaves = globalCoPointSlaves(); const mapDistribute& pointSlavesMap = globalCoPointSlavesMap(); + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); // Points are either // - master with slaves @@ -2444,6 +2561,9 @@ Foam::autoPtr<Foam::globalIndex> Foam::globalMeshData::mergePoints } } + // Restore tag + UPstream::msgType(oldTag); + return globalPointsPtr; } @@ -2471,6 +2591,9 @@ Foam::autoPtr<Foam::globalIndex> Foam::globalMeshData::mergePoints // patch point via coupled patches). + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + // Determine mapping: // - from patch point to coupled point (or -1) // - from coupled point to global patch point @@ -2659,6 +2782,9 @@ Foam::autoPtr<Foam::globalIndex> Foam::globalMeshData::mergePoints } } + // Restore tag + UPstream::msgType(oldTag); + return globalPointsPtr; } diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshDataTemplates.C b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshDataTemplates.C index fff043fdb0ba90142724838fdfabaa85531c79ea..2e6fc1460a39f1d4490b075a6acd44f59299b78f 100644 --- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshDataTemplates.C +++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshDataTemplates.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2015 OpenFOAM Foundation - Copyright (C) 2018-2019 OpenCFD Ltd. + Copyright (C) 2018-2024 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -44,6 +44,9 @@ void Foam::globalMeshData::syncData const TransformOp& top ) { + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + // Pull slave data onto master slavesMap.distribute(transforms, elems, top); @@ -94,6 +97,9 @@ void Foam::globalMeshData::syncData elems, top ); + + // Reset tag + UPstream::msgType(oldTag); } @@ -107,6 +113,9 @@ void Foam::globalMeshData::syncData const CombineOp& cop ) { + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + // Pull slave data onto master slavesMap.distribute(elems); @@ -151,6 +160,9 @@ void Foam::globalMeshData::syncData // Push slave-slot data back to slaves slavesMap.reverseDistribute(elems.size(), elems); + + // Reset tag + UPstream::msgType(oldTag); } diff --git a/src/OpenFOAM/meshes/polyMesh/polyBoundaryMesh/polyBoundaryMesh.C b/src/OpenFOAM/meshes/polyMesh/polyBoundaryMesh/polyBoundaryMesh.C index 175684dbf2428912e573ee480ab215128d438064..3ece63c5e75cc346059f056108ce55dd8514e397 100644 --- a/src/OpenFOAM/meshes/polyMesh/polyBoundaryMesh/polyBoundaryMesh.C +++ b/src/OpenFOAM/meshes/polyMesh/polyBoundaryMesh/polyBoundaryMesh.C @@ -300,6 +300,9 @@ void Foam::polyBoundaryMesh::clearAddressing() void Foam::polyBoundaryMesh::calcGeometry() { + // Make sure messages don't interact by having unique tag + const int oldTag = UPstream::incrMsgType(); + PstreamBuffers pBufs(Pstream::defaultCommsType); if @@ -341,6 +344,9 @@ void Foam::polyBoundaryMesh::calcGeometry() } } } + + // Reset tag + UPstream::msgType(oldTag); } diff --git a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C index b23781865970512b0ebd79cc97a82e1996eb6fd1..b08250eade3687b31913ab9f7923ace8e67d5254 100644 --- a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C +++ b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C @@ -92,6 +92,9 @@ void Foam::syncTools::syncPointMap { const polyBoundaryMesh& patches = mesh.boundaryMesh(); + // Make sure we use unique message tag + const int oldTag = UPstream::incrMsgType(); + // Synchronize multiple shared points. const globalMeshData& pd = mesh.globalData(); @@ -369,6 +372,9 @@ void Foam::syncTools::syncPointMap } } } + + // Reset tag + UPstream::msgType(oldTag); } @@ -387,6 +393,8 @@ void Foam::syncTools::syncEdgeMap // Do synchronisation without constructing globalEdge addressing // (since this constructs mesh edge addressing) + // Make sure we use unique message tag + const int oldTag = UPstream::incrMsgType(); // Swap proc patch info // ~~~~~~~~~~~~~~~~~~~~ @@ -734,6 +742,9 @@ void Foam::syncTools::syncEdgeMap ); } } + + // Reset tag + UPstream::msgType(oldTag); } @@ -1037,6 +1048,9 @@ void Foam::syncTools::syncBoundaryFaceList const polyBoundaryMesh& patches = mesh.boundaryMesh(); + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + if (parRun && UPstream::parRun()) { // Avoid mesh.globalData() - possible race condition @@ -1252,6 +1266,9 @@ void Foam::syncTools::syncBoundaryFaceList } } } + + // Reset tag + UPstream::msgType(oldTag); } @@ -1283,6 +1300,9 @@ void Foam::syncTools::syncFaceList const polyBoundaryMesh& patches = mesh.boundaryMesh(); + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + if (parRun && UPstream::parRun()) { const label startRequest = UPstream::nRequests(); @@ -1403,6 +1423,9 @@ void Foam::syncTools::syncFaceList } } } + + // Reset tag + UPstream::msgType(oldTag); } diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C index a95239a9b67b050106f5793fa130d15cfdbca65c..0a5da32f09f5f52cd4406233c0a3f750e0406572 100644 --- a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C +++ b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C @@ -611,6 +611,8 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate scalarRecvBufs_ ); } + + this->updatedMatrix(false); } @@ -679,6 +681,8 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix } this->addToInternalField(result, !add, faceCells, coeffs, pnf); + + this->updatedMatrix(true); } @@ -735,6 +739,8 @@ void Foam::cyclicACMIFvPatchField<Type>::initInterfaceMatrixUpdate recvBufs_ ); } + + this->updatedMatrix(false); } @@ -792,6 +798,8 @@ void Foam::cyclicACMIFvPatchField<Type>::updateInterfaceMatrix } this->addToInternalField(result, !add, faceCells, coeffs, pnf); + + this->updatedMatrix(true); } diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C index 8a8ad6e3cd8db421a1b78750a49fc7502ce9b39d..4e2acbd0233487948855764de8920fbac7fa7c17 100644 --- a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C +++ b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C @@ -616,6 +616,8 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate scalarRecvBufs_ ); } + + this->updatedMatrix(false); } @@ -693,6 +695,8 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix // Multiply the field by coefficients and add into the result this->addToInternalField(result, !add, faceCells, coeffs, pnf); + + this->updatedMatrix(true); } @@ -751,6 +755,8 @@ void Foam::cyclicAMIFvPatchField<Type>::initInterfaceMatrixUpdate recvBufs_ ); } + + this->updatedMatrix(false); } @@ -827,6 +833,8 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix // Multiply the field by coefficients and add into the result this->addToInternalField(result, !add, faceCells, coeffs, pnf); + + this->updatedMatrix(true); } diff --git a/src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationTemplates.C b/src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationTemplates.C index 535b35ca563f5a687cdb8136fa58fb4baa9245c5..bb76d2b4a3b80a42b21c2d76914aad55a430e5bb 100644 --- a/src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationTemplates.C +++ b/src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolationTemplates.C @@ -145,7 +145,8 @@ void Foam::AMIInterpolation::interpolateToTarget if (distributed()) { const mapDistribute& map = srcMapPtr_(); - work = fld; // deep copy + work.resize_nocopy(map.constructSize()); + SubList<Type>(work, fld.size()) = fld; // deep copy map.distribute(work); } @@ -205,7 +206,8 @@ void Foam::AMIInterpolation::interpolateToSource if (distributed()) { const mapDistribute& map = tgtMapPtr_(); - work = fld; // deep copy + work.resize_nocopy(map.constructSize()); + SubList<Type>(work, fld.size()) = fld; // deep copy map.distribute(work); } diff --git a/src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI/faceAreaWeightAMI.C b/src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI/faceAreaWeightAMI.C index 0c29c9b0c2c8b494753e50c0ce559c2c4c4a2b55..38948dfeb68125f97d4064e6d85dbfa6f006a785 100644 --- a/src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI/faceAreaWeightAMI.C +++ b/src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI/faceAreaWeightAMI.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2013-2016 OpenFOAM Foundation - Copyright (C) 2018-2021 OpenCFD Ltd. + Copyright (C) 2018-2024 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -714,6 +714,8 @@ bool Foam::faceAreaWeightAMI::calculate if (distributed()) { const label myRank = UPstream::myProcNo(comm_); + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); const primitivePatch& srcPatch0 = this->srcPatch0(); const primitivePatch& tgtPatch0 = this->tgtPatch0(); @@ -751,7 +753,7 @@ bool Foam::faceAreaWeightAMI::calculate labelList(), ListOps::appendEqOp<label>(), flipOp(), // flip operation - UPstream::msgType(), + UPstream::msgType()+77431, comm_ ); @@ -768,7 +770,7 @@ bool Foam::faceAreaWeightAMI::calculate scalarList(), ListOps::appendEqOp<scalar>(), flipOp(), - UPstream::msgType(), + UPstream::msgType()+77432, comm_ ); @@ -784,7 +786,7 @@ bool Foam::faceAreaWeightAMI::calculate globalSrcFaces, tgtAddress_, cMapSrc, - UPstream::msgType(), + UPstream::msgType()+77433, comm_ ) ); @@ -797,10 +799,13 @@ bool Foam::faceAreaWeightAMI::calculate globalTgtFaces, srcAddress_, cMapTgt, - UPstream::msgType(), + UPstream::msgType()+77434, comm_ ) ); + + // Reset tag + UPstream::msgType(oldTag); } // Convert the weights from areas to normalised values diff --git a/src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI2D/faceAreaWeightAMI2D.C b/src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI2D/faceAreaWeightAMI2D.C index dea7e4f032dd3769ccd47b7a31a4fac40f7a8df5..bd476f9633a8867682423cd1711355ac0b505b76 100644 --- a/src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI2D/faceAreaWeightAMI2D.C +++ b/src/meshTools/AMIInterpolation/AMIInterpolation/faceAreaWeightAMI2D/faceAreaWeightAMI2D.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2020,2022 OpenCFD Ltd. + Copyright (C) 2020,2022,2024 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -428,6 +428,8 @@ bool Foam::faceAreaWeightAMI2D::calculate if (distributed()) { const label myRank = UPstream::myProcNo(comm_); + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); const primitivePatch& srcPatch0 = this->srcPatch0(); const primitivePatch& tgtPatch0 = this->tgtPatch0(); @@ -465,7 +467,7 @@ bool Foam::faceAreaWeightAMI2D::calculate labelList(), ListOps::appendEqOp<label>(), flipOp(), // flip operation - UPstream::msgType(), + UPstream::msgType()+77431, comm_ ); @@ -482,7 +484,7 @@ bool Foam::faceAreaWeightAMI2D::calculate scalarList(), ListOps::appendEqOp<scalar>(), flipOp(), // flip operation - UPstream::msgType(), + UPstream::msgType()+77432, comm_ ); @@ -498,7 +500,7 @@ bool Foam::faceAreaWeightAMI2D::calculate globalSrcFaces, tgtAddress_, cMapSrc, - UPstream::msgType(), + UPstream::msgType()+77433, comm_ ) ); @@ -511,10 +513,13 @@ bool Foam::faceAreaWeightAMI2D::calculate globalTgtFaces, srcAddress_, cMapTgt, - UPstream::msgType(), + UPstream::msgType()+77434, comm_ ) ); + + // Reset tag + UPstream::msgType(oldTag); } // Convert the weights from areas to normalised values diff --git a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C index 2d4491ca778b993c80e679823c80c144c4e4b6e7..1bc66d590b278efe14fbc191ce75eeca1553e938 100644 --- a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C +++ b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C @@ -238,10 +238,13 @@ void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate sendRequests_, scalarSendBufs_, recvRequests_, - scalarRecvBufs_ + scalarRecvBufs_, + 19462+cyclicACMIInterface_.index() // unique offset + patch index ); UPstream::commWarn(oldWarnComm); } + + this->updatedMatrix(false); } @@ -286,7 +289,13 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix // Receive (= copy) data from buffers into work. TBD: receive directly // into slices of work. solveScalarField work; - map.receive(recvRequests_, scalarRecvBufs_, work); + map.receive + ( + recvRequests_, + scalarRecvBufs_, + work, + 19462+cyclicACMIInterface_.index() // unique offset + patch index + ); // Receive requests all handled by last function call recvRequests_.clear(); @@ -327,6 +336,8 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix this->addToInternalField(result, !add, faceCells, coeffs, pnf); } + + this->updatedMatrix(true); } diff --git a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C index 68d19e6c1d7c4a2c43be959dd0fe2b1c8c139cd4..f6bcc50d4e69f413c32a2c6c31c1a5865b634500 100644 --- a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C +++ b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C @@ -239,10 +239,13 @@ void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate sendRequests_, scalarSendBufs_, recvRequests_, - scalarRecvBufs_ + scalarRecvBufs_, + 19462+cyclicAMIInterface_.index() // unique offset + patch index ); UPstream::commWarn(oldWarnComm); } + + this->updatedMatrix(false); } @@ -300,7 +303,13 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix // Receive (= copy) data from buffers into work. TBD: receive directly // into slices of work. solveScalarField work; - map.receive(recvRequests_, scalarRecvBufs_, work); + map.receive + ( + recvRequests_, + scalarRecvBufs_, + work, + 19462+cyclicAMIInterface_.index() // unique offset + patch index + ); // Receive requests all handled by last function call recvRequests_.clear(); @@ -340,6 +349,8 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix // Add result using coefficients this->addToInternalField(result, !add, faceCells, coeffs, pnf); } + + this->updatedMatrix(true); } diff --git a/src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTemplates.C b/src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTemplates.C index 636c77f01aa9b934d706156d231dc70a4abf3ae9..036737e15c3082f3b418ab55de6fdf14fa3420aa 100644 --- a/src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTemplates.C +++ b/src/meshTools/AMIInterpolation/patches/cyclicAMI/cyclicAMIPolyPatch/cyclicAMIPolyPatchTemplates.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2016 OpenFOAM Foundation - Copyright (C) 2021-2023 OpenCFD Ltd. + Copyright (C) 2021-2024 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -175,7 +175,15 @@ void Foam::cyclicAMIPolyPatch::initInterpolateUntransformed const auto& map = (owner() ? AMI.tgtMap() : AMI.srcMap()); // Insert send/receive requests (non-blocking) - map.send(fld, sendRequests, sendBuffers, recvRequests, recvBuffers); + map.send + ( + fld, + sendRequests, + sendBuffers, + recvRequests, + recvBuffers, + 3894+this->index() // unique offset + patch index + ); } } @@ -256,7 +264,13 @@ Foam::tmp<Foam::Field<Type>> Foam::cyclicAMIPolyPatch::interpolate { // Receive (= copy) data from buffers into work. TBD: receive directly // into slices of work. - map.receive(requests, recvBuffers, work); + map.receive + ( + requests, + recvBuffers, + work, + 3894+this->index() // unique offset + patch index + ); } const Field<Type>& fld = (AMI.distributed() ? work : localFld); diff --git a/src/meshTools/polyTopoChange/polyTopoChange.C b/src/meshTools/polyTopoChange/polyTopoChange.C index ff2cc6f9846b25a5a110387f32550bf64c8f65fb..a749722f0efc787dec44155d658d42a78e0bc560 100644 --- a/src/meshTools/polyTopoChange/polyTopoChange.C +++ b/src/meshTools/polyTopoChange/polyTopoChange.C @@ -2026,6 +2026,9 @@ void Foam::polyTopoChange::reorderCoupledFaces // Rotation on new faces. labelList rotation(faces_.size(), Zero); + // Allocate unique tag for all comms + const int oldTag = UPstream::incrMsgType(); + PstreamBuffers pBufs; // Send ordering @@ -2131,6 +2134,9 @@ void Foam::polyTopoChange::reorderCoupledFaces } } } + + // Reset tag + UPstream::msgType(oldTag); }