diff --git a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixUpdateMatrixInterfaces.C b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixUpdateMatrixInterfaces.C index 1e36a8208676165af43b2f6ca3dae68ca403a9b5..5376a1947ba3d0f431f938740b8645cae94ddbde 100644 --- a/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixUpdateMatrixInterfaces.C +++ b/src/OpenFOAM/matrices/LduMatrix/LduMatrix/LduMatrixUpdateMatrixInterfaces.C @@ -115,23 +115,70 @@ void Foam::LduMatrix<Type, DType, LUType>::updateMatrixInterfaces { const UPstream::commsTypes commsType = UPstream::defaultCommsType; - // Block until sends/receives have finished - if (commsType == UPstream::commsTypes::nonBlocking) + if + ( + commsType == UPstream::commsTypes::nonBlocking + && UPstream::nPollProcInterfaces + ) { - UPstream::waitRequests(startRequest); + // Wait for some interface requests to become available and + // consume them. No guarantee that the finished requests actually + // correspond to any particular interface, but it is reasonably + // probable that some interfaces will be able to start consumption + // without waiting for all requests. + + DynamicList<int> indices; // (work array) + while + ( + UPstream::nPollProcInterfaces < 0 + && UPstream::waitSomeRequests(startRequest, &indices) + ) + { + forAll(interfaces_, interfacei) + { + auto* intf = interfaces_.get(interfacei); + + if (intf && !intf->updatedMatrix() && intf->ready()) + { + intf->updateInterfaceMatrix + ( + result, + add, + lduMesh_.lduAddr(), + interfacei, + psiif, + interfaceCoeffs[interfacei], + commsType + ); + } + } + } } + if ( commsType == UPstream::commsTypes::blocking || commsType == UPstream::commsTypes::nonBlocking ) { + // Wait until sends/receives have finished. + // - effectively a no-op (without waiting) if already completed. + if (commsType == UPstream::commsTypes::nonBlocking) + { + UPstream::waitRequests(startRequest); + } + + // Check/no-check for updatedMatrix() ? + const bool noCheck = (commsType == UPstream::commsTypes::blocking); + forAll(interfaces_, interfacei) { - if (interfaces_.set(interfacei)) + auto* intf = interfaces_.get(interfacei); + + if (intf && (noCheck || !intf->updatedMatrix())) { - interfaces_[interfacei].updateInterfaceMatrix + intf->updateInterfaceMatrix ( result, add, diff --git a/src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C b/src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C index d2e78b4cba457144c33d5aec8b11244fa8a5b385..1d20ed70237ec4283cd694e19dfcefbd1f7e497c 100644 --- a/src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C +++ b/src/OpenFOAM/matrices/lduMatrix/lduMatrix/lduMatrixUpdateMatrixInterfaces.C @@ -118,46 +118,68 @@ void Foam::lduMatrix::updateMatrixInterfaces { const UPstream::commsTypes commsType = UPstream::defaultCommsType; - if (commsType == UPstream::commsTypes::blocking) + if + ( + commsType == UPstream::commsTypes::nonBlocking + && UPstream::nPollProcInterfaces + ) { - forAll(interfaces, interfacei) + // Wait for some interface requests to become available and + // consume them. No guarantee that the finished requests actually + // correspond to any particular interface, but it is reasonably + // probable that some interfaces will be able to start consumption + // without waiting for all requests. + + DynamicList<int> indices; // (work array) + while + ( + UPstream::nPollProcInterfaces < 0 + && UPstream::waitSomeRequests(startRequest, &indices) + ) { - if (interfaces.set(interfacei)) + forAll(interfaces, interfacei) { - interfaces[interfacei].updateInterfaceMatrix - ( - result, - add, - mesh().lduAddr(), - interfacei, - psiif, - coupleCoeffs[interfacei], - cmpt, - commsType - ); + auto* intf = interfaces.get(interfacei); + + if (intf && !intf->updatedMatrix() && intf->ready()) + { + intf->updateInterfaceMatrix + ( + result, + add, + mesh().lduAddr(), + interfacei, + psiif, + coupleCoeffs[interfacei], + cmpt, + commsType + ); + } } } - } - else if (commsType == UPstream::commsTypes::nonBlocking) - { - // Try and consume interfaces as they become available - bool allUpdated = false; - for (label i=0; i<UPstream::nPollProcInterfaces; i++) + // [OLDER] + // Alternative for consuming interfaces as they become available. + // Within the loop, the ready() calls an MPI_Test + // that can trigger progression. However, a bit less reliably + // (and less efficient) since it is implies multiple calls to + // MPI_Test to progress the MPI transfer, but no guarantee that + // any of them will actually complete within nPollProcInterfaces + // calls. + + for (label i = 0; i < UPstream::nPollProcInterfaces; ++i) { - allUpdated = true; + bool allUpdated = true; forAll(interfaces, interfacei) { - if - ( - interfaces.set(interfacei) - && !interfaces[interfacei].updatedMatrix() - ) + auto* intf = interfaces.get(interfacei); + + if (intf && !intf->updatedMatrix()) { - if (interfaces[interfacei].ready()) + if (intf->ready()) { - interfaces[interfacei].updateInterfaceMatrix + intf->updateInterfaceMatrix ( result, add, @@ -178,35 +200,36 @@ void Foam::lduMatrix::updateMatrixInterfaces if (allUpdated) { - break; + break; // Early exit } } + } - // Block for everything - if (Pstream::parRun()) + + if + ( + commsType == UPstream::commsTypes::blocking + || commsType == UPstream::commsTypes::nonBlocking + ) + { + // Wait until sends/receives have finished. + // - effectively a no-op (without waiting) if already completed. + if (commsType == UPstream::commsTypes::nonBlocking) { - if (allUpdated) - { - // All received. Just remove all outstanding requests - UPstream::resetRequests(startRequest); - } - else - { - // Block for all requests and remove storage - UPstream::waitRequests(startRequest); - } + UPstream::waitRequests(startRequest); } - // Consume + // Check/no-check for updatedMatrix() ? + const bool noCheck = (commsType == UPstream::commsTypes::blocking); + + // Consume anything still outstanding forAll(interfaces, interfacei) { - if - ( - interfaces.set(interfacei) - && !interfaces[interfacei].updatedMatrix() - ) + auto* intf = interfaces.get(interfacei); + + if (intf && (noCheck || !intf->updatedMatrix())) { - interfaces[interfacei].updateInterfaceMatrix + intf->updateInterfaceMatrix ( result, add, @@ -282,7 +305,7 @@ void Foam::lduMatrix::updateMatrixInterfaces psiif, coupleCoeffs[interfacei], cmpt, - Pstream::commsTypes::blocking + UPstream::commsTypes::blocking ); } } diff --git a/tutorials/incompressible/simpleFoam/windAroundBuildings/Allrun-parallel b/tutorials/incompressible/simpleFoam/windAroundBuildings/Allrun-parallel index c391fe6cca25cd46d1bbb99397b29dc6ac88fb76..96d81ea377e6f58891c2810b0c2215a0d3d427d3 100755 --- a/tutorials/incompressible/simpleFoam/windAroundBuildings/Allrun-parallel +++ b/tutorials/incompressible/simpleFoam/windAroundBuildings/Allrun-parallel @@ -9,7 +9,8 @@ runApplication decomposePar restore0Dir -processor -runParallel $(getApplication) +# Test polling interfaces +runParallel $(getApplication) -opt-switch nPollProcInterfaces=-1 runApplication reconstructPar