From 5191f8433bdb05d2232dea2b8d3c5add7415a278 Mon Sep 17 00:00:00 2001
From: Stefano Zampini <stefano.zampini@gmail.com>
Date: Thu, 18 Jun 2020 17:24:18 +0300
Subject: [PATCH] Use the proper communicator for the matrix
---
src/petsc4Foam/Make/options | 1 +
src/petsc4Foam/solvers/petscSolver.C | 16 ++++++++++++++--
2 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/src/petsc4Foam/Make/options b/src/petsc4Foam/Make/options
index a4d0a73..22de33e 100644
--- a/src/petsc4Foam/Make/options
+++ b/src/petsc4Foam/Make/options
@@ -13,6 +13,7 @@ EXE_INC = \
-Wno-old-style-cast \
-I$(LIB_SRC)/finiteVolume/lnInclude \
-I$(LIB_SRC)/meshTools/lnInclude \
+ -I$(LIB_SRC)/Pstream/mpi \
$(foreach dir,$(PETSC_INC_DIR),-I$(dir))
LIB_LIBS = \
diff --git a/src/petsc4Foam/solvers/petscSolver.C b/src/petsc4Foam/solvers/petscSolver.C
index 8b4f4d7..77c372f 100644
--- a/src/petsc4Foam/solvers/petscSolver.C
+++ b/src/petsc4Foam/solvers/petscSolver.C
@@ -33,6 +33,7 @@ License
#include "cyclicLduInterface.H"
#include "cyclicAMILduInterface.H"
#include "addToRunTimeSelectionTable.H"
+#include "PstreamGlobals.H"
#include "petscSolver.H"
#include "petscControls.H"
@@ -354,7 +355,7 @@ void Foam::petscSolver::buildKsp
) const
{
// Create parallel solver context
- KSPCreate(PETSC_COMM_WORLD, &ksp);
+ KSPCreate(PetscObjectComm((PetscObject)Amat), &ksp);
// Set the prefix for the options db (e.g. -eqn_p_)
KSPSetOptionsPrefix(ksp, prefix_.c_str());
@@ -384,6 +385,17 @@ void Foam::petscSolver::computeMatAllocation
label& maxLowNonZeroPerRow
) const
{
+ // communicator for the matrix
+ // PETSc internally duplicates the communicator to
+ // avoid tag/attributes clashing
+
+ const auto comm =
+ (
+ PstreamGlobals::MPICommunicators_.empty()
+ ? PETSC_COMM_SELF
+ : PstreamGlobals::MPICommunicators_[matrix_.mesh().comm()]
+ );
+
// traversal-based algorithm
typedef List<PetscInt> integerList;
@@ -398,7 +410,7 @@ void Foam::petscSolver::computeMatAllocation
const label nrows_ = lduAddr.size();
// Create a petsc matrix
- MatCreate(PETSC_COMM_WORLD, &Amat);
+ MatCreate(comm, &Amat);
MatSetSizes(Amat, nrows_, nrows_, PETSC_DECIDE, PETSC_DECIDE);
// Set the prefix for the options db (e.g. -eqn_p_)
--
GitLab